// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::collections::HashMap; use std::io::IsTerminal; use std::rc::Rc; use std::sync::Arc; use base64::prelude::BASE64_STANDARD; use base64::Engine; use deno_config::ConfigFile; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::unsync::JoinSet; use deno_runtime::colors; use deno_runtime::deno_fetch::reqwest; use import_map::ImportMap; use lsp_types::Url; use serde::Serialize; use sha2::Digest; use crate::args::deno_registry_api_url; use crate::args::deno_registry_url; use crate::args::CliOptions; use crate::args::Flags; use crate::args::PublishFlags; use crate::cache::ParsedSourceCache; use crate::factory::CliFactory; use crate::graph_util::ModuleGraphBuilder; use crate::http_util::HttpClient; use crate::tools::check::CheckOptions; use crate::tools::registry::diagnostics::PublishDiagnosticsCollector; use crate::tools::registry::graph::collect_fast_check_type_graph_diagnostics; use crate::tools::registry::graph::collect_invalid_external_imports; use crate::tools::registry::graph::get_workspace_member_roots; use crate::tools::registry::graph::resolve_config_file_roots_from_exports; use crate::tools::registry::graph::MemberRoots; use crate::util::display::human_size; use crate::util::import_map::ImportMapUnfurler; mod api; mod auth; mod diagnostics; mod graph; mod paths; mod publish_order; mod tar; use auth::get_auth_method; use auth::AuthMethod; use publish_order::PublishOrderGraph; use super::check::TypeChecker; use self::tar::PublishableTarball; fn ring_bell() { // ASCII code for the bell character. print!("\x07"); } struct PreparedPublishPackage { scope: String, package: String, version: String, tarball: PublishableTarball, } impl PreparedPublishPackage { pub fn display_name(&self) -> String { format!("@{}/{}@{}", self.scope, self.package, self.version) } } static SUGGESTED_ENTRYPOINTS: [&str; 4] = ["mod.ts", "mod.js", "index.ts", "index.js"]; fn get_deno_json_package_name( deno_json: &ConfigFile, ) -> Result { match deno_json.json.name.clone() { Some(name) => Ok(name), None => bail!("{} is missing 'name' field", deno_json.specifier), } } async fn prepare_publish( deno_json: &ConfigFile, source_cache: Arc, import_map: Arc, diagnostics_collector: &PublishDiagnosticsCollector, ) -> Result, AnyError> { let config_path = deno_json.specifier.to_file_path().unwrap(); let dir_path = config_path.parent().unwrap().to_path_buf(); let Some(version) = deno_json.json.version.clone() else { bail!("{} is missing 'version' field", deno_json.specifier); }; let name = get_deno_json_package_name(deno_json)?; if deno_json.json.exports.is_none() { let mut suggested_entrypoint = None; for entrypoint in SUGGESTED_ENTRYPOINTS { if dir_path.join(entrypoint).exists() { suggested_entrypoint = Some(entrypoint); break; } } let exports_content = format!( r#"{{ "name": "{}", "version": "{}", "exports": "{}" }}"#, name, version, suggested_entrypoint.unwrap_or("") ); bail!( "You did not specify an entrypoint to \"{}\" package in {}. Add `exports` mapping in the configuration file, eg:\n{}", name, deno_json.specifier, exports_content ); } let Some(name) = name.strip_prefix('@') else { bail!("Invalid package name, use '@/ format"); }; let Some((scope, package_name)) = name.split_once('/') else { bail!("Invalid package name, use '@/ format"); }; let file_patterns = deno_json.to_publish_config()?.map(|c| c.files); let diagnostics_collector = diagnostics_collector.clone(); let tarball = deno_core::unsync::spawn_blocking(move || { let unfurler = ImportMapUnfurler::new(&import_map); tar::create_gzipped_tarball( &dir_path, &*source_cache, &diagnostics_collector, &unfurler, file_patterns, ) .context("Failed to create a tarball") }) .await??; log::debug!("Tarball size ({}): {}", name, tarball.bytes.len()); Ok(Rc::new(PreparedPublishPackage { scope: scope.to_string(), package: package_name.to_string(), version: version.to_string(), tarball, })) } #[derive(Serialize)] #[serde(tag = "permission")] pub enum Permission<'s> { #[serde(rename = "package/publish", rename_all = "camelCase")] VersionPublish { scope: &'s str, package: &'s str, version: &'s str, tarball_hash: &'s str, }, } async fn get_auth_headers( client: &reqwest::Client, registry_url: String, packages: Vec>, auth_method: AuthMethod, ) -> Result>, AnyError> { let permissions = packages .iter() .map(|package| Permission::VersionPublish { scope: &package.scope, package: &package.package, version: &package.version, tarball_hash: &package.tarball.hash, }) .collect::>(); let mut authorizations = HashMap::with_capacity(packages.len()); match auth_method { AuthMethod::Interactive => { let verifier = uuid::Uuid::new_v4().to_string(); let challenge = BASE64_STANDARD.encode(sha2::Sha256::digest(&verifier)); let response = client .post(format!("{}authorizations", registry_url)) .json(&serde_json::json!({ "challenge": challenge, "permissions": permissions, })) .send() .await .context("Failed to create interactive authorization")?; let auth = api::parse_response::(response) .await .context("Failed to create interactive authorization")?; print!( "Visit {} to authorize publishing of", colors::cyan(format!("{}?code={}", auth.verification_url, auth.code)) ); if packages.len() > 1 { println!(" {} packages", packages.len()); } else { println!(" @{}/{}", packages[0].scope, packages[0].package); } ring_bell(); println!("{}", colors::gray("Waiting...")); let interval = std::time::Duration::from_secs(auth.poll_interval); loop { tokio::time::sleep(interval).await; let response = client .post(format!("{}authorizations/exchange", registry_url)) .json(&serde_json::json!({ "exchangeToken": auth.exchange_token, "verifier": verifier, })) .send() .await .context("Failed to exchange authorization")?; let res = api::parse_response::(response) .await; match res { Ok(res) => { println!( "{} {} {}", colors::green("Authorization successful."), colors::gray("Authenticated as"), colors::cyan(res.user.name) ); let authorization: Rc = format!("Bearer {}", res.token).into(); for pkg in &packages { authorizations.insert( (pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()), authorization.clone(), ); } break; } Err(err) => { if err.code == "authorizationPending" { continue; } else { return Err(err).context("Failed to exchange authorization"); } } } } } AuthMethod::Token(token) => { let authorization: Rc = format!("Bearer {}", token).into(); for pkg in &packages { authorizations.insert( (pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()), authorization.clone(), ); } } AuthMethod::Oidc(oidc_config) => { let mut chunked_packages = packages.chunks(16); for permissions in permissions.chunks(16) { let audience = json!({ "permissions": permissions }).to_string(); let url = format!( "{}&audience={}", oidc_config.url, percent_encoding::percent_encode( audience.as_bytes(), percent_encoding::NON_ALPHANUMERIC ) ); let response = client .get(url) .bearer_auth(&oidc_config.token) .send() .await .context("Failed to get OIDC token")?; let status = response.status(); let text = response.text().await.with_context(|| { format!("Failed to get OIDC token: status {}", status) })?; if !status.is_success() { bail!( "Failed to get OIDC token: status {}, response: '{}'", status, text ); } let api::OidcTokenResponse { value } = serde_json::from_str(&text) .with_context(|| { format!( "Failed to parse OIDC token: '{}' (status {})", text, status ) })?; let authorization: Rc = format!("githuboidc {}", value).into(); for pkg in chunked_packages.next().unwrap() { authorizations.insert( (pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()), authorization.clone(), ); } } } }; Ok(authorizations) } /// Check if both `scope` and `package` already exist, if not return /// a URL to the management panel to create them. async fn check_if_scope_and_package_exist( client: &reqwest::Client, registry_api_url: &str, registry_manage_url: &str, scope: &str, package: &str, ) -> Result, AnyError> { let mut needs_scope = false; let mut needs_package = false; let response = api::get_scope(client, registry_api_url, scope).await?; if response.status() == 404 { needs_scope = true; } let response = api::get_package(client, registry_api_url, scope, package).await?; if response.status() == 404 { needs_package = true; } if needs_scope || needs_package { let create_url = format!( "{}new?scope={}&package={}&from=cli", registry_manage_url, scope, package ); return Ok(Some(create_url)); } Ok(None) } async fn ensure_scopes_and_packages_exist( client: &reqwest::Client, registry_api_url: String, registry_manage_url: String, packages: Vec>, ) -> Result<(), AnyError> { if !std::io::stdin().is_terminal() { let mut missing_packages_lines = vec![]; for package in packages { let maybe_create_package_url = check_if_scope_and_package_exist( client, ®istry_api_url, ®istry_manage_url, &package.scope, &package.package, ) .await?; if let Some(create_package_url) = maybe_create_package_url { missing_packages_lines.push(format!(" - {}", create_package_url)); } } if !missing_packages_lines.is_empty() { bail!( "Following packages don't exist, follow the links and create them:\n{}", missing_packages_lines.join("\n") ); } return Ok(()); } for package in packages { let maybe_create_package_url = check_if_scope_and_package_exist( client, ®istry_api_url, ®istry_manage_url, &package.scope, &package.package, ) .await?; let Some(create_package_url) = maybe_create_package_url else { continue; }; ring_bell(); println!( "'@{}/{}' doesn't exist yet. Visit {} to create the package", &package.scope, &package.package, colors::cyan_with_underline(create_package_url) ); println!("{}", colors::gray("Waiting...")); let package_api_url = api::get_package_api_url( ®istry_api_url, &package.scope, &package.package, ); loop { tokio::time::sleep(std::time::Duration::from_secs(3)).await; let response = client.get(&package_api_url).send().await?; if response.status() == 200 { let name = format!("@{}/{}", package.scope, package.package); println!("Package {} created", colors::green(name)); break; } } } Ok(()) } async fn perform_publish( http_client: &Arc, mut publish_order_graph: PublishOrderGraph, mut prepared_package_by_name: HashMap>, auth_method: AuthMethod, ) -> Result<(), AnyError> { let client = http_client.client()?; let registry_api_url = deno_registry_api_url().to_string(); let registry_url = deno_registry_url().to_string(); let packages = prepared_package_by_name .values() .cloned() .collect::>(); ensure_scopes_and_packages_exist( client, registry_api_url.clone(), registry_url.clone(), packages.clone(), ) .await?; let mut authorizations = get_auth_headers(client, registry_api_url.clone(), packages, auth_method) .await?; assert_eq!(prepared_package_by_name.len(), authorizations.len()); let mut futures: JoinSet> = JoinSet::default(); loop { let next_batch = publish_order_graph.next(); for package_name in next_batch { let package = prepared_package_by_name.remove(&package_name).unwrap(); // todo(dsherret): output something that looks better than this even not in debug if log::log_enabled!(log::Level::Debug) { log::debug!("Publishing {}", package.display_name()); for file in &package.tarball.files { log::debug!( " Tarball file {} {}", human_size(file.size as f64), file.specifier ); } } let authorization = authorizations .remove(&( package.scope.clone(), package.package.clone(), package.version.clone(), )) .unwrap(); let registry_api_url = registry_api_url.clone(); let registry_url = registry_url.clone(); let http_client = http_client.clone(); futures.spawn(async move { let display_name = package.display_name(); publish_package( &http_client, package, ®istry_api_url, ®istry_url, &authorization, ) .await .with_context(|| format!("Failed to publish {}", display_name))?; Ok(package_name) }); } let Some(result) = futures.join_next().await else { // done, ensure no circular dependency publish_order_graph.ensure_no_pending()?; break; }; let package_name = result??; publish_order_graph.finish_package(&package_name); } Ok(()) } async fn publish_package( http_client: &HttpClient, package: Rc, registry_api_url: &str, registry_url: &str, authorization: &str, ) -> Result<(), AnyError> { let client = http_client.client()?; println!( "{} @{}/{}@{} ...", colors::intense_blue("Publishing"), package.scope, package.package, package.version ); let url = format!( "{}scopes/{}/packages/{}/versions/{}", registry_api_url, package.scope, package.package, package.version ); let response = client .post(url) .header(reqwest::header::AUTHORIZATION, authorization) .header(reqwest::header::CONTENT_ENCODING, "gzip") .body(package.tarball.bytes.clone()) .send() .await?; let res = api::parse_response::(response).await; let mut task = match res { Ok(task) => task, Err(mut err) if err.code == "duplicateVersionPublish" => { let task = serde_json::from_value::( err.data.get_mut("task").unwrap().take(), ) .unwrap(); if task.status == "success" { println!( "{} @{}/{}@{}", colors::green("Skipping, already published"), package.scope, package.package, package.version ); return Ok(()); } println!( "{} @{}/{}@{}", colors::yellow("Already uploaded, waiting for publishing"), package.scope, package.package, package.version ); task } Err(err) => { return Err(err).with_context(|| { format!( "Failed to publish @{}/{} at {}", package.scope, package.package, package.version ) }) } }; let interval = std::time::Duration::from_secs(2); while task.status != "success" && task.status != "failure" { tokio::time::sleep(interval).await; let resp = client .get(format!("{}publish_status/{}", registry_api_url, task.id)) .send() .await .with_context(|| { format!( "Failed to get publishing status for @{}/{} at {}", package.scope, package.package, package.version ) })?; task = api::parse_response::(resp) .await .with_context(|| { format!( "Failed to get publishing status for @{}/{} at {}", package.scope, package.package, package.version ) })?; } if let Some(error) = task.error { bail!( "{} @{}/{} at {}: {}", colors::red("Failed to publish"), package.scope, package.package, package.version, error.message ); } println!( "{} @{}/{}@{}", colors::green("Successfully published"), package.scope, package.package, package.version ); println!( "{}", colors::gray(format!( "Visit {}@{}/{}@{} for details", registry_url, package.scope, package.package, package.version )) ); Ok(()) } async fn prepare_packages_for_publishing( cli_factory: &CliFactory, no_fast_check: bool, diagnostics_collector: &PublishDiagnosticsCollector, deno_json: ConfigFile, import_map: Arc, ) -> Result< ( PublishOrderGraph, HashMap>, ), AnyError, > { let maybe_workspace_config = deno_json.to_workspace_config()?; let module_graph_builder = cli_factory.module_graph_builder().await?.as_ref(); let source_cache = cli_factory.parsed_source_cache(); let type_checker = cli_factory.type_checker().await?; let cli_options = cli_factory.cli_options(); let Some(workspace_config) = maybe_workspace_config else { let roots = resolve_config_file_roots_from_exports(&deno_json)?; build_and_check_graph_for_publish( module_graph_builder, type_checker, cli_options, no_fast_check, diagnostics_collector, &[MemberRoots { name: get_deno_json_package_name(&deno_json)?, dir_url: deno_json.specifier.join("./").unwrap().clone(), exports: roots, }], ) .await?; let mut prepared_package_by_name = HashMap::with_capacity(1); let package = prepare_publish( &deno_json, source_cache.clone(), import_map, diagnostics_collector, ) .await?; let package_name = format!("@{}/{}", package.scope, package.package); let publish_order_graph = PublishOrderGraph::new_single(package_name.clone()); prepared_package_by_name.insert(package_name, package); return Ok((publish_order_graph, prepared_package_by_name)); }; println!("Publishing a workspace..."); // create the module graph let roots = get_workspace_member_roots(&workspace_config)?; let graph = build_and_check_graph_for_publish( module_graph_builder, type_checker, cli_options, no_fast_check, diagnostics_collector, &roots, ) .await?; let mut prepared_package_by_name = HashMap::with_capacity(workspace_config.members.len()); let publish_order_graph = publish_order::build_publish_order_graph(&graph, &roots)?; let results = workspace_config .members .iter() .cloned() .map(|member| { let import_map = import_map.clone(); async move { let package = prepare_publish( &member.config_file, source_cache.clone(), import_map.clone(), diagnostics_collector, ) .await .with_context(|| { format!("Failed preparing '{}'.", member.package_name) })?; Ok::<_, AnyError>((member.package_name, package)) } .boxed() }) .collect::>(); let results = deno_core::futures::future::join_all(results).await; for result in results { let (package_name, package) = result?; prepared_package_by_name.insert(package_name, package); } Ok((publish_order_graph, prepared_package_by_name)) } async fn build_and_check_graph_for_publish( module_graph_builder: &ModuleGraphBuilder, type_checker: &TypeChecker, cli_options: &CliOptions, no_fast_check: bool, diagnostics_collector: &PublishDiagnosticsCollector, packages: &[MemberRoots], ) -> Result, deno_core::anyhow::Error> { let graph = Arc::new( module_graph_builder .create_graph_with_options(crate::graph_util::CreateGraphOptions { // All because we're going to use this same graph to determine the publish order later graph_kind: deno_graph::GraphKind::All, roots: packages .iter() .flat_map(|r| r.exports.iter()) .cloned() .collect(), workspace_fast_check: true, loader: None, }) .await?, ); graph.valid()?; collect_invalid_external_imports(&graph, diagnostics_collector); let mut has_fast_check_diagnostics = false; if !no_fast_check { log::info!("Checking fast check type graph for errors..."); has_fast_check_diagnostics = collect_fast_check_type_graph_diagnostics( &graph, packages, diagnostics_collector, ); } if !has_fast_check_diagnostics { log::info!("Ensuring type checks..."); let diagnostics = type_checker .check_diagnostics( graph.clone(), CheckOptions { lib: cli_options.ts_type_lib_window(), log_ignored_options: false, reload: cli_options.reload_flag(), }, ) .await?; if !diagnostics.is_empty() { bail!( concat!( "{:#}\n\n", "You may have discovered a bug in Deno's fast check implementation. ", "Fast check is still early days and we would appreciate if you log a ", "bug if you believe this is one: https://github.com/denoland/deno/issues/" ), diagnostics ); } } Ok(graph) } pub async fn publish( flags: Flags, publish_flags: PublishFlags, ) -> Result<(), AnyError> { let cli_factory = CliFactory::from_flags(flags).await?; let auth_method = get_auth_method(publish_flags.token)?; let import_map = cli_factory .maybe_import_map() .await? .clone() .unwrap_or_else(|| { Arc::new(ImportMap::new(Url::parse("file:///dev/null").unwrap())) }); let directory_path = cli_factory.cli_options().initial_cwd(); let cli_options = cli_factory.cli_options(); let Some(config_file) = cli_options.maybe_config_file() else { bail!( "Couldn't find a deno.json or a deno.jsonc configuration file in {}.", directory_path.display() ); }; let diagnostics_collector = PublishDiagnosticsCollector::default(); let (publish_order_graph, prepared_package_by_name) = prepare_packages_for_publishing( &cli_factory, publish_flags.no_fast_check, &diagnostics_collector, config_file.clone(), import_map, ) .await?; diagnostics_collector .print_and_error(&**cli_factory.parsed_source_cache())?; if prepared_package_by_name.is_empty() { bail!("No packages to publish"); } if publish_flags.dry_run { log::warn!( "{} Aborting due to --dry-run", crate::colors::yellow("Warning") ); return Ok(()); } perform_publish( cli_factory.http_client(), publish_order_graph, prepared_package_by_name, auth_method, ) .await }