0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-10-29 08:58:01 -04:00
denoland-deno/cli/tools/registry/mod.rs
2024-01-31 16:10:31 +01:00

860 lines
24 KiB
Rust

// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashMap;
use std::io::IsTerminal;
use std::rc::Rc;
use std::sync::Arc;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_config::ConfigFile;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures::FutureExt;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::unsync::JoinSet;
use deno_runtime::colors;
use deno_runtime::deno_fetch::reqwest;
use import_map::ImportMap;
use lsp_types::Url;
use serde::Serialize;
use sha2::Digest;
use crate::args::deno_registry_api_url;
use crate::args::deno_registry_url;
use crate::args::CliOptions;
use crate::args::Flags;
use crate::args::PublishFlags;
use crate::cache::ParsedSourceCache;
use crate::factory::CliFactory;
use crate::graph_util::ModuleGraphBuilder;
use crate::http_util::HttpClient;
use crate::tools::check::CheckOptions;
use crate::tools::registry::diagnostics::PublishDiagnosticsCollector;
use crate::tools::registry::graph::collect_fast_check_type_graph_diagnostics;
use crate::tools::registry::graph::collect_invalid_external_imports;
use crate::tools::registry::graph::get_workspace_member_roots;
use crate::tools::registry::graph::resolve_config_file_roots_from_exports;
use crate::tools::registry::graph::MemberRoots;
use crate::util::display::human_size;
use crate::util::import_map::ImportMapUnfurler;
mod api;
mod auth;
mod diagnostics;
mod graph;
mod paths;
mod publish_order;
mod tar;
use auth::get_auth_method;
use auth::AuthMethod;
use publish_order::PublishOrderGraph;
use super::check::TypeChecker;
use self::tar::PublishableTarball;
fn ring_bell() {
// ASCII code for the bell character.
print!("\x07");
}
struct PreparedPublishPackage {
scope: String,
package: String,
version: String,
tarball: PublishableTarball,
}
impl PreparedPublishPackage {
pub fn display_name(&self) -> String {
format!("@{}/{}@{}", self.scope, self.package, self.version)
}
}
static SUGGESTED_ENTRYPOINTS: [&str; 4] =
["mod.ts", "mod.js", "index.ts", "index.js"];
fn get_deno_json_package_name(
deno_json: &ConfigFile,
) -> Result<String, AnyError> {
match deno_json.json.name.clone() {
Some(name) => Ok(name),
None => bail!("{} is missing 'name' field", deno_json.specifier),
}
}
async fn prepare_publish(
deno_json: &ConfigFile,
source_cache: Arc<ParsedSourceCache>,
import_map: Arc<ImportMap>,
diagnostics_collector: &PublishDiagnosticsCollector,
) -> Result<Rc<PreparedPublishPackage>, AnyError> {
let config_path = deno_json.specifier.to_file_path().unwrap();
let dir_path = config_path.parent().unwrap().to_path_buf();
let Some(version) = deno_json.json.version.clone() else {
bail!("{} is missing 'version' field", deno_json.specifier);
};
let name = get_deno_json_package_name(deno_json)?;
if deno_json.json.exports.is_none() {
let mut suggested_entrypoint = None;
for entrypoint in SUGGESTED_ENTRYPOINTS {
if dir_path.join(entrypoint).exists() {
suggested_entrypoint = Some(entrypoint);
break;
}
}
let exports_content = format!(
r#"{{
"name": "{}",
"version": "{}",
"exports": "{}"
}}"#,
name,
version,
suggested_entrypoint.unwrap_or("<path_to_entrypoint>")
);
bail!(
"You did not specify an entrypoint to \"{}\" package in {}. Add `exports` mapping in the configuration file, eg:\n{}",
name,
deno_json.specifier,
exports_content
);
}
let Some(name) = name.strip_prefix('@') else {
bail!("Invalid package name, use '@<scope_name>/<package_name> format");
};
let Some((scope, package_name)) = name.split_once('/') else {
bail!("Invalid package name, use '@<scope_name>/<package_name> format");
};
let file_patterns = deno_json.to_publish_config()?.map(|c| c.files);
let diagnostics_collector = diagnostics_collector.clone();
let tarball = deno_core::unsync::spawn_blocking(move || {
let unfurler = ImportMapUnfurler::new(&import_map);
tar::create_gzipped_tarball(
&dir_path,
&*source_cache,
&diagnostics_collector,
&unfurler,
file_patterns,
)
.context("Failed to create a tarball")
})
.await??;
log::debug!("Tarball size ({}): {}", name, tarball.bytes.len());
Ok(Rc::new(PreparedPublishPackage {
scope: scope.to_string(),
package: package_name.to_string(),
version: version.to_string(),
tarball,
}))
}
#[derive(Serialize)]
#[serde(tag = "permission")]
pub enum Permission<'s> {
#[serde(rename = "package/publish", rename_all = "camelCase")]
VersionPublish {
scope: &'s str,
package: &'s str,
version: &'s str,
tarball_hash: &'s str,
},
}
async fn get_auth_headers(
client: &reqwest::Client,
registry_url: String,
packages: Vec<Rc<PreparedPublishPackage>>,
auth_method: AuthMethod,
) -> Result<HashMap<(String, String, String), Rc<str>>, AnyError> {
let permissions = packages
.iter()
.map(|package| Permission::VersionPublish {
scope: &package.scope,
package: &package.package,
version: &package.version,
tarball_hash: &package.tarball.hash,
})
.collect::<Vec<_>>();
let mut authorizations = HashMap::with_capacity(packages.len());
match auth_method {
AuthMethod::Interactive => {
let verifier = uuid::Uuid::new_v4().to_string();
let challenge = BASE64_STANDARD.encode(sha2::Sha256::digest(&verifier));
let response = client
.post(format!("{}authorizations", registry_url))
.json(&serde_json::json!({
"challenge": challenge,
"permissions": permissions,
}))
.send()
.await
.context("Failed to create interactive authorization")?;
let auth =
api::parse_response::<api::CreateAuthorizationResponse>(response)
.await
.context("Failed to create interactive authorization")?;
print!(
"Visit {} to authorize publishing of",
colors::cyan(format!("{}?code={}", auth.verification_url, auth.code))
);
if packages.len() > 1 {
println!(" {} packages", packages.len());
} else {
println!(" @{}/{}", packages[0].scope, packages[0].package);
}
ring_bell();
println!("{}", colors::gray("Waiting..."));
let interval = std::time::Duration::from_secs(auth.poll_interval);
loop {
tokio::time::sleep(interval).await;
let response = client
.post(format!("{}authorizations/exchange", registry_url))
.json(&serde_json::json!({
"exchangeToken": auth.exchange_token,
"verifier": verifier,
}))
.send()
.await
.context("Failed to exchange authorization")?;
let res =
api::parse_response::<api::ExchangeAuthorizationResponse>(response)
.await;
match res {
Ok(res) => {
println!(
"{} {} {}",
colors::green("Authorization successful."),
colors::gray("Authenticated as"),
colors::cyan(res.user.name)
);
let authorization: Rc<str> = format!("Bearer {}", res.token).into();
for pkg in &packages {
authorizations.insert(
(pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()),
authorization.clone(),
);
}
break;
}
Err(err) => {
if err.code == "authorizationPending" {
continue;
} else {
return Err(err).context("Failed to exchange authorization");
}
}
}
}
}
AuthMethod::Token(token) => {
let authorization: Rc<str> = format!("Bearer {}", token).into();
for pkg in &packages {
authorizations.insert(
(pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()),
authorization.clone(),
);
}
}
AuthMethod::Oidc(oidc_config) => {
let mut chunked_packages = packages.chunks(16);
for permissions in permissions.chunks(16) {
let audience = json!({ "permissions": permissions }).to_string();
let url = format!(
"{}&audience={}",
oidc_config.url,
percent_encoding::percent_encode(
audience.as_bytes(),
percent_encoding::NON_ALPHANUMERIC
)
);
let response = client
.get(url)
.bearer_auth(&oidc_config.token)
.send()
.await
.context("Failed to get OIDC token")?;
let status = response.status();
let text = response.text().await.with_context(|| {
format!("Failed to get OIDC token: status {}", status)
})?;
if !status.is_success() {
bail!(
"Failed to get OIDC token: status {}, response: '{}'",
status,
text
);
}
let api::OidcTokenResponse { value } = serde_json::from_str(&text)
.with_context(|| {
format!(
"Failed to parse OIDC token: '{}' (status {})",
text, status
)
})?;
let authorization: Rc<str> = format!("githuboidc {}", value).into();
for pkg in chunked_packages.next().unwrap() {
authorizations.insert(
(pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()),
authorization.clone(),
);
}
}
}
};
Ok(authorizations)
}
/// Check if both `scope` and `package` already exist, if not return
/// a URL to the management panel to create them.
async fn check_if_scope_and_package_exist(
client: &reqwest::Client,
registry_api_url: &str,
registry_manage_url: &str,
scope: &str,
package: &str,
) -> Result<Option<String>, AnyError> {
let mut needs_scope = false;
let mut needs_package = false;
let response = api::get_scope(client, registry_api_url, scope).await?;
if response.status() == 404 {
needs_scope = true;
}
let response =
api::get_package(client, registry_api_url, scope, package).await?;
if response.status() == 404 {
needs_package = true;
}
if needs_scope || needs_package {
let create_url = format!(
"{}new?scope={}&package={}&from=cli",
registry_manage_url, scope, package
);
return Ok(Some(create_url));
}
Ok(None)
}
async fn ensure_scopes_and_packages_exist(
client: &reqwest::Client,
registry_api_url: String,
registry_manage_url: String,
packages: Vec<Rc<PreparedPublishPackage>>,
) -> Result<(), AnyError> {
if !std::io::stdin().is_terminal() {
let mut missing_packages_lines = vec![];
for package in packages {
let maybe_create_package_url = check_if_scope_and_package_exist(
client,
&registry_api_url,
&registry_manage_url,
&package.scope,
&package.package,
)
.await?;
if let Some(create_package_url) = maybe_create_package_url {
missing_packages_lines.push(format!(" - {}", create_package_url));
}
}
if !missing_packages_lines.is_empty() {
bail!(
"Following packages don't exist, follow the links and create them:\n{}",
missing_packages_lines.join("\n")
);
}
return Ok(());
}
for package in packages {
let maybe_create_package_url = check_if_scope_and_package_exist(
client,
&registry_api_url,
&registry_manage_url,
&package.scope,
&package.package,
)
.await?;
let Some(create_package_url) = maybe_create_package_url else {
continue;
};
ring_bell();
println!(
"'@{}/{}' doesn't exist yet. Visit {} to create the package",
&package.scope,
&package.package,
colors::cyan_with_underline(create_package_url)
);
println!("{}", colors::gray("Waiting..."));
let package_api_url = api::get_package_api_url(
&registry_api_url,
&package.scope,
&package.package,
);
loop {
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
let response = client.get(&package_api_url).send().await?;
if response.status() == 200 {
let name = format!("@{}/{}", package.scope, package.package);
println!("Package {} created", colors::green(name));
break;
}
}
}
Ok(())
}
async fn perform_publish(
http_client: &Arc<HttpClient>,
mut publish_order_graph: PublishOrderGraph,
mut prepared_package_by_name: HashMap<String, Rc<PreparedPublishPackage>>,
auth_method: AuthMethod,
) -> Result<(), AnyError> {
let client = http_client.client()?;
let registry_api_url = deno_registry_api_url().to_string();
let registry_url = deno_registry_url().to_string();
let packages = prepared_package_by_name
.values()
.cloned()
.collect::<Vec<_>>();
ensure_scopes_and_packages_exist(
client,
registry_api_url.clone(),
registry_url.clone(),
packages.clone(),
)
.await?;
let mut authorizations =
get_auth_headers(client, registry_api_url.clone(), packages, auth_method)
.await?;
assert_eq!(prepared_package_by_name.len(), authorizations.len());
let mut futures: JoinSet<Result<String, AnyError>> = JoinSet::default();
loop {
let next_batch = publish_order_graph.next();
for package_name in next_batch {
let package = prepared_package_by_name.remove(&package_name).unwrap();
// todo(dsherret): output something that looks better than this even not in debug
if log::log_enabled!(log::Level::Debug) {
log::debug!("Publishing {}", package.display_name());
for file in &package.tarball.files {
log::debug!(
" Tarball file {} {}",
human_size(file.size as f64),
file.specifier
);
}
}
let authorization = authorizations
.remove(&(
package.scope.clone(),
package.package.clone(),
package.version.clone(),
))
.unwrap();
let registry_api_url = registry_api_url.clone();
let registry_url = registry_url.clone();
let http_client = http_client.clone();
futures.spawn(async move {
let display_name = package.display_name();
publish_package(
&http_client,
package,
&registry_api_url,
&registry_url,
&authorization,
)
.await
.with_context(|| format!("Failed to publish {}", display_name))?;
Ok(package_name)
});
}
let Some(result) = futures.join_next().await else {
// done, ensure no circular dependency
publish_order_graph.ensure_no_pending()?;
break;
};
let package_name = result??;
publish_order_graph.finish_package(&package_name);
}
Ok(())
}
async fn publish_package(
http_client: &HttpClient,
package: Rc<PreparedPublishPackage>,
registry_api_url: &str,
registry_url: &str,
authorization: &str,
) -> Result<(), AnyError> {
let client = http_client.client()?;
println!(
"{} @{}/{}@{} ...",
colors::intense_blue("Publishing"),
package.scope,
package.package,
package.version
);
let url = format!(
"{}scopes/{}/packages/{}/versions/{}",
registry_api_url, package.scope, package.package, package.version
);
let response = client
.post(url)
.header(reqwest::header::AUTHORIZATION, authorization)
.header(reqwest::header::CONTENT_ENCODING, "gzip")
.body(package.tarball.bytes.clone())
.send()
.await?;
let res = api::parse_response::<api::PublishingTask>(response).await;
let mut task = match res {
Ok(task) => task,
Err(mut err) if err.code == "duplicateVersionPublish" => {
let task = serde_json::from_value::<api::PublishingTask>(
err.data.get_mut("task").unwrap().take(),
)
.unwrap();
if task.status == "success" {
println!(
"{} @{}/{}@{}",
colors::green("Skipping, already published"),
package.scope,
package.package,
package.version
);
return Ok(());
}
println!(
"{} @{}/{}@{}",
colors::yellow("Already uploaded, waiting for publishing"),
package.scope,
package.package,
package.version
);
task
}
Err(err) => {
return Err(err).with_context(|| {
format!(
"Failed to publish @{}/{} at {}",
package.scope, package.package, package.version
)
})
}
};
let interval = std::time::Duration::from_secs(2);
while task.status != "success" && task.status != "failure" {
tokio::time::sleep(interval).await;
let resp = client
.get(format!("{}publish_status/{}", registry_api_url, task.id))
.send()
.await
.with_context(|| {
format!(
"Failed to get publishing status for @{}/{} at {}",
package.scope, package.package, package.version
)
})?;
task = api::parse_response::<api::PublishingTask>(resp)
.await
.with_context(|| {
format!(
"Failed to get publishing status for @{}/{} at {}",
package.scope, package.package, package.version
)
})?;
}
if let Some(error) = task.error {
bail!(
"{} @{}/{} at {}: {}",
colors::red("Failed to publish"),
package.scope,
package.package,
package.version,
error.message
);
}
println!(
"{} @{}/{}@{}",
colors::green("Successfully published"),
package.scope,
package.package,
package.version
);
println!(
"{}",
colors::gray(format!(
"Visit {}@{}/{}@{} for details",
registry_url, package.scope, package.package, package.version
))
);
Ok(())
}
async fn prepare_packages_for_publishing(
cli_factory: &CliFactory,
no_fast_check: bool,
diagnostics_collector: &PublishDiagnosticsCollector,
deno_json: ConfigFile,
import_map: Arc<ImportMap>,
) -> Result<
(
PublishOrderGraph,
HashMap<String, Rc<PreparedPublishPackage>>,
),
AnyError,
> {
let maybe_workspace_config = deno_json.to_workspace_config()?;
let module_graph_builder = cli_factory.module_graph_builder().await?.as_ref();
let source_cache = cli_factory.parsed_source_cache();
let type_checker = cli_factory.type_checker().await?;
let cli_options = cli_factory.cli_options();
let Some(workspace_config) = maybe_workspace_config else {
let roots = resolve_config_file_roots_from_exports(&deno_json)?;
build_and_check_graph_for_publish(
module_graph_builder,
type_checker,
cli_options,
no_fast_check,
diagnostics_collector,
&[MemberRoots {
name: get_deno_json_package_name(&deno_json)?,
dir_url: deno_json.specifier.join("./").unwrap().clone(),
exports: roots,
}],
)
.await?;
let mut prepared_package_by_name = HashMap::with_capacity(1);
let package = prepare_publish(
&deno_json,
source_cache.clone(),
import_map,
diagnostics_collector,
)
.await?;
let package_name = format!("@{}/{}", package.scope, package.package);
let publish_order_graph =
PublishOrderGraph::new_single(package_name.clone());
prepared_package_by_name.insert(package_name, package);
return Ok((publish_order_graph, prepared_package_by_name));
};
println!("Publishing a workspace...");
// create the module graph
let roots = get_workspace_member_roots(&workspace_config)?;
let graph = build_and_check_graph_for_publish(
module_graph_builder,
type_checker,
cli_options,
no_fast_check,
diagnostics_collector,
&roots,
)
.await?;
let mut prepared_package_by_name =
HashMap::with_capacity(workspace_config.members.len());
let publish_order_graph =
publish_order::build_publish_order_graph(&graph, &roots)?;
let results = workspace_config
.members
.iter()
.cloned()
.map(|member| {
let import_map = import_map.clone();
async move {
let package = prepare_publish(
&member.config_file,
source_cache.clone(),
import_map.clone(),
diagnostics_collector,
)
.await
.with_context(|| {
format!("Failed preparing '{}'.", member.package_name)
})?;
Ok::<_, AnyError>((member.package_name, package))
}
.boxed()
})
.collect::<Vec<_>>();
let results = deno_core::futures::future::join_all(results).await;
for result in results {
let (package_name, package) = result?;
prepared_package_by_name.insert(package_name, package);
}
Ok((publish_order_graph, prepared_package_by_name))
}
async fn build_and_check_graph_for_publish(
module_graph_builder: &ModuleGraphBuilder,
type_checker: &TypeChecker,
cli_options: &CliOptions,
no_fast_check: bool,
diagnostics_collector: &PublishDiagnosticsCollector,
packages: &[MemberRoots],
) -> Result<Arc<deno_graph::ModuleGraph>, deno_core::anyhow::Error> {
let graph = Arc::new(
module_graph_builder
.create_graph_with_options(crate::graph_util::CreateGraphOptions {
// All because we're going to use this same graph to determine the publish order later
graph_kind: deno_graph::GraphKind::All,
roots: packages
.iter()
.flat_map(|r| r.exports.iter())
.cloned()
.collect(),
workspace_fast_check: true,
loader: None,
})
.await?,
);
graph.valid()?;
collect_invalid_external_imports(&graph, diagnostics_collector);
let mut has_fast_check_diagnostics = false;
if !no_fast_check {
log::info!("Checking fast check type graph for errors...");
has_fast_check_diagnostics = collect_fast_check_type_graph_diagnostics(
&graph,
packages,
diagnostics_collector,
);
}
if !has_fast_check_diagnostics {
log::info!("Ensuring type checks...");
let diagnostics = type_checker
.check_diagnostics(
graph.clone(),
CheckOptions {
lib: cli_options.ts_type_lib_window(),
log_ignored_options: false,
reload: cli_options.reload_flag(),
},
)
.await?;
if !diagnostics.is_empty() {
bail!(
concat!(
"{:#}\n\n",
"You may have discovered a bug in Deno's fast check implementation. ",
"Fast check is still early days and we would appreciate if you log a ",
"bug if you believe this is one: https://github.com/denoland/deno/issues/"
),
diagnostics
);
}
}
Ok(graph)
}
pub async fn publish(
flags: Flags,
publish_flags: PublishFlags,
) -> Result<(), AnyError> {
let cli_factory = CliFactory::from_flags(flags).await?;
let auth_method = get_auth_method(publish_flags.token)?;
let import_map = cli_factory
.maybe_import_map()
.await?
.clone()
.unwrap_or_else(|| {
Arc::new(ImportMap::new(Url::parse("file:///dev/null").unwrap()))
});
let directory_path = cli_factory.cli_options().initial_cwd();
let cli_options = cli_factory.cli_options();
let Some(config_file) = cli_options.maybe_config_file() else {
bail!(
"Couldn't find a deno.json or a deno.jsonc configuration file in {}.",
directory_path.display()
);
};
let diagnostics_collector = PublishDiagnosticsCollector::default();
let (publish_order_graph, prepared_package_by_name) =
prepare_packages_for_publishing(
&cli_factory,
publish_flags.no_fast_check,
&diagnostics_collector,
config_file.clone(),
import_map,
)
.await?;
diagnostics_collector
.print_and_error(&**cli_factory.parsed_source_cache())?;
if prepared_package_by_name.is_empty() {
bail!("No packages to publish");
}
if publish_flags.dry_run {
log::warn!(
"{} Aborting due to --dry-run",
crate::colors::yellow("Warning")
);
return Ok(());
}
perform_publish(
cli_factory.http_client(),
publish_order_graph,
prepared_package_by_name,
auth_method,
)
.await
}