From 8072722f5c4bf854fe1e9d81cf4d7caaae610c94 Mon Sep 17 00:00:00 2001 From: Julian Ramirez Ruiseco Date: Tue, 18 Aug 2026 11:30:29 -0400 Subject: [PATCH] feat(release): application release pipeline core The release contract logic moves here from fsl_libs' .github/scripts (bash), per the doctrine that fslabscli owns CI logic; the fsl_libs workflows become thin invocations of a pinned fslabscli (fslabs/fsl_libs#4230). This also makes fslabscli the single owner of the library-vs-application release boundary: `release classify` shares the publish- tag-prefix constant with the publish path, so a library release this tool creates is skipped by the app pipeline by construction. This PR is the pipeline CORE - everything a release cannot ship without; the operational periphery (resolve, record, healthcheck, cleanup-drafts) stacks on top in a follow-up PR. - classify: library skip vs validated - app release, keyed on the TAG (this tool leaves library release names null); apps are discovered from [package.metadata.fslabs.release] across every git-tracked manifest, INCLUDING excluded workspaces (fdk_apps holds the only shipped app). - verify-production: revision resolved from the tag itself (never target_commitish, which is a branch name), ancestry of main, workspace version binding, green check runs. - probe-store: the conditional-write go/no-go gate against a deployed store. Object lock plus versioning stops deletion but NOT overwrites; only conditional writes make published objects immutable. - publish: completeness against configured targets (missing OR extra artifacts fail; every linux artifact needs its .asc), signature re-verification against the bytes (osslsigncode per MSI, gpg against the PUBLISHED key for linux; DMG is stapler-validated on the mac host), digesting, monotonicity derived from COMMITTED MANIFESTS (never index.json), artifact writes read back and digest-compared, the manifest written LAST as the atomic commit point, index compare-and-swap. A refused overwrite propagates as-is: the version is spent. - promote: the only writer of channels.json. Production-manifest gate ("previews never"), per-target artifact presence, selected-artifact digest re-verification, and a pure apply_move whose backward gate re-runs INSIDE the CAS retry loop; acknowledged backward moves (the rollback procedure) record backward:true plus the reason in a capped provenance log. - bundle-linux: cargo-deb plus a bundle-nothing AppImage (pinned appimagetool by sha256) with the glibc floor asserted from the binary's versioned symbols; no versioned glibc symbols at all is an error, not a pass. - sign-linux: temp GNUPGHOME, detach-sign, self-verify, fingerprint out. types.rs is the single source of truth for the manifest / index / channels.json shapes; store.rs holds the storage semantics on the existing opendal dependency; http.rs and keyring.rs are the one HTTP client and one gpg keyring every command shares. New deps: semver, thiserror. All commands run on Linux runners, so the existing linux-musl release asset remains the only distributed binary. Proven end to end with the built binary against a throwaway MinIO (publish 4 artifacts + manifest-last; re-publish refused; backward move refused then acknowledged with provenance) and against the real fsl_libs tree (classify discovers spatial_engine from metadata in the excluded workspace). Integration defects found by those live runs, invisible to the unit tests: toml 0.9 document-vs-value parsing, clap's propagated auto --version flag colliding with --version arguments, and opendal expressing S3 If-None-Match:* as if_not_exists. Known wart: main.rs requires a git root before dispatch; a follow-up. The legacy publish.binary installer/channel model in check_workspace/binary.rs is superseded by this family; deprecation is follow-up after the fsl_libs cutover. cargo test: 311 passed. clippy: zero diagnostics. --- Cargo.lock | 7 +- Cargo.toml | 2 + src/commands/mod.rs | 1 + src/commands/release/bundle_linux.rs | 457 ++++++++++++ src/commands/release/classify.rs | 375 ++++++++++ src/commands/release/http.rs | 73 ++ src/commands/release/keyring.rs | 71 ++ src/commands/release/mod.rs | 120 ++++ src/commands/release/probe_store.rs | 163 +++++ src/commands/release/promote.rs | 409 +++++++++++ src/commands/release/publish.rs | 810 ++++++++++++++++++++++ src/commands/release/sign_linux.rs | 213 ++++++ src/commands/release/store.rs | 246 +++++++ src/commands/release/types.rs | 219 ++++++ src/commands/release/verify_production.rs | 213 ++++++ src/main.rs | 6 + 16 files changed, 3383 insertions(+), 2 deletions(-) create mode 100644 src/commands/release/bundle_linux.rs create mode 100644 src/commands/release/classify.rs create mode 100644 src/commands/release/http.rs create mode 100644 src/commands/release/keyring.rs create mode 100644 src/commands/release/mod.rs create mode 100644 src/commands/release/probe_store.rs create mode 100644 src/commands/release/promote.rs create mode 100644 src/commands/release/publish.rs create mode 100644 src/commands/release/sign_linux.rs create mode 100644 src/commands/release/store.rs create mode 100644 src/commands/release/types.rs create mode 100644 src/commands/release/verify_production.rs diff --git a/Cargo.lock b/Cargo.lock index 013eb504d..4253257e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,6 +817,7 @@ dependencies = [ "rust-toolchain-file", "rustls", "self_update", + "semver", "serde", "serde_json", "serde_with", @@ -827,6 +828,7 @@ dependencies = [ "temp-dir", "tempfile", "testcontainers", + "thiserror 2.0.18", "tokio", "toml", "toml_edit 0.23.7", @@ -4550,11 +4552,12 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", + "serde_core", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b3e80bc58..f32ab86d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,8 @@ toml = "0.9.8" clap_complete = "4.5.60" clap_mangen = "0.2.31" regex = "1.12.2" +semver = "1.0.27" +thiserror = "2.0.17" sha2 = "0.10.9" toml_edit = "0.23.7" walkdir = "2.5.0" diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 7ac57c895..a55ed515b 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -8,6 +8,7 @@ pub mod generate_wix; pub mod generate_workflow; pub mod github_app_token; pub mod publish; +pub mod release; pub mod release_utils; #[cfg(test)] pub mod release_utils_tests; diff --git a/src/commands/release/bundle_linux.rs b/src/commands/release/bundle_linux.rs new file mode 100644 index 000000000..30b4d60de --- /dev/null +++ b/src/commands/release/bundle_linux.rs @@ -0,0 +1,457 @@ +//! Build the Linux release artifacts for an app: a .deb from the package's +//! `[package.metadata.deb]` table (cargo-deb; the Depends line is authored +//! by hand because Vulkan/EGL/X11 are dlopened and absent from DT_NEEDED) +//! and an AppImage for distributions the .deb does not cover. +//! +//! Runs inside the pinned build container (debian bullseye), whose glibc is +//! the declared compatibility floor; the floor is asserted against the built +//! binary's versioned symbols (objdump -T), so a toolchain or container +//! change that raises it fails HERE instead of on a customer machine. +//! +//! The AppImage deliberately bundles NOTHING but the binary, desktop entry +//! and icon: the Vulkan loader, GL, X11 and GTK must come from the host, +//! because bundling a Vulkan loader against a host ICD is the standard +//! AppImage failure mode. appimagetool is fetched pinned by URL and sha256 +//! and run with APPIMAGE_EXTRACT_AND_RUN=1 (no FUSE in a container). + +use std::fmt::{Display, Formatter}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; +use clap::Parser; +use serde::Serialize; +use walkdir::WalkDir; + +use super::store::sha256_hex; +use super::types::TARGET_LINUX; +use crate::PrettyPrintable; + +pub const APPIMAGETOOL_URL: &str = + "https://github.com/AppImage/appimagetool/releases/download/1.9.1/appimagetool-x86_64.AppImage"; +pub const APPIMAGETOOL_SHA256: &str = + "ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0"; + +#[derive(Debug, Parser, Clone)] +#[command( + about = "Build the Linux .deb and AppImage in the pinned floor container", + disable_version_flag = true +)] +pub struct Options { + /// Cargo package name (e.g. spatial_engine). + #[arg(long)] + pub package: String, + /// Binary/product name (e.g. SpatialEngine). + #[arg(long)] + pub binary_name: String, + #[arg(long)] + pub version: String, + /// Package directory relative to the repo root. + #[arg(long)] + pub package_dir: PathBuf, + /// Icon file installed as the AppImage/desktop icon. + #[arg(long)] + pub icon: PathBuf, + #[arg(long, default_value = "2.31")] + pub glibc_floor: String, + /// Skip the cargo build (the binary already exists in the target dir). + #[arg(long, default_value_t = false)] + pub no_build: bool, +} + +#[derive(Debug, Serialize, Clone)] +pub struct BundleLinuxResult { + pub deb: PathBuf, + pub appimage: PathBuf, + pub max_glibc_symbol: String, +} + +impl Display for BundleLinuxResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + writeln!(f, "deb={}", self.deb.display())?; + writeln!(f, "appimage={}", self.appimage.display())?; + write!(f, "highest glibc symbol: {}", self.max_glibc_symbol) + } +} + +impl PrettyPrintable for BundleLinuxResult { + fn pretty_print(&self) -> String { + self.to_string() + } +} + +/// Files that must NEVER ship inside the AppDir: graphics stacks the host +/// provides. Bundling any of them against a host driver is the standard +/// AppImage failure mode. +const FORBIDDEN_LIB_PREFIXES: [&str; 4] = ["libvulkan", "libGL", "libEGL", "libX11"]; + +/// Parse "2.31" or "2.2.5" into numeric segments. String or `sort` +/// comparison gets 2.9 vs 2.31 wrong, which is the whole point of doing +/// this numerically. +fn parse_numeric_version(version: &str) -> anyhow::Result> { + version + .split('.') + .map(|segment| { + segment + .parse::() + .with_context(|| format!("unparseable version segment '{segment}' in '{version}'")) + }) + .collect() +} + +/// The highest GLIBC_x.y[.z] versioned symbol in `objdump -T` output, by +/// numeric segment comparison. Non-numeric suffixes (GLIBC_PRIVATE) are +/// ignored. `None` when no versioned glibc symbol appears at all. +fn max_glibc_symbol(objdump_output: &str) -> Option { + let mut best: Option> = None; + for (idx, _) in objdump_output.match_indices("GLIBC_") { + let rest = &objdump_output[idx + "GLIBC_".len()..]; + let end = rest + .find(|c: char| !c.is_ascii_digit() && c != '.') + .unwrap_or(rest.len()); + let version = rest[..end].trim_end_matches('.'); + if version.is_empty() { + continue; + } + let Ok(parsed) = parse_numeric_version(version) else { + continue; + }; + if best.as_ref().is_none_or(|b| parsed > *b) { + best = Some(parsed); + } + } + best.map(|v| v.iter().map(u32::to_string).collect::>().join(".")) +} + +/// True when the binary's highest glibc symbol exceeds the declared floor. +fn exceeds_floor(max_symbol: &str, floor: &str) -> anyhow::Result { + Ok(parse_numeric_version(max_symbol)? > parse_numeric_version(floor)?) +} + +/// The `Icon=` value of a desktop entry; it names the icon file installed +/// into the AppDir. +fn desktop_icon_name(desktop_contents: &str) -> Option { + desktop_contents + .lines() + .find_map(|line| line.trim().strip_prefix("Icon=")) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +async fn run_step( + program: &str, + args: &[&str], + dir: &Path, + envs: &[(&str, &str)], +) -> anyhow::Result<()> { + let mut command = tokio::process::Command::new(program); + command.args(args).current_dir(dir); + for (name, value) in envs { + command.env(name, value); + } + let status = command + .status() + .await + .with_context(|| format!("failed to run {program}"))?; + if !status.success() { + bail!("{program} {} failed with {status}", args.join(" ")); + } + Ok(()) +} + +async fn capture_stdout(program: &str, args: &[&str]) -> anyhow::Result { + let output = tokio::process::Command::new(program) + .args(args) + .output() + .await + .with_context(|| format!("failed to run {program}"))?; + if !output.status.success() { + bail!( + "{program} {} failed with {}: {}", + args.join(" "), + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// Download a URL (GitHub release downloads redirect to object storage). +async fn download(url: &str) -> anyhow::Result> { + super::http::get_bytes(&super::http::client()?, url).await +} + +fn make_executable(path: &Path) -> anyhow::Result<()> { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)) + .with_context(|| format!("cannot chmod {}", path.display())) +} + +pub async fn run( + options: &Options, + working_directory: PathBuf, +) -> anyhow::Result { + let package_dir = if options.package_dir.is_absolute() { + options.package_dir.clone() + } else { + working_directory.join(&options.package_dir) + }; + let icon = if options.icon.is_absolute() { + options.icon.clone() + } else { + working_directory.join(&options.icon) + }; + + if !options.no_build { + run_step( + "cargo", + &[ + "build", + "--release", + "--locked", + "--package", + &options.package, + "--target", + TARGET_LINUX, + ], + &package_dir, + &[], + ) + .await?; + } + + let target_dir = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| package_dir.join("target")); + let binary = target_dir + .join(TARGET_LINUX) + .join("release") + .join(&options.binary_name); + if !binary.is_file() { + bail!("built binary not found at {}", binary.display()); + } + + // Assert the glibc floor against the binary's versioned symbols. + let objdump_output = capture_stdout("objdump", &["-T", &binary.to_string_lossy()]).await?; + let max_glibc = max_glibc_symbol(&objdump_output).with_context(|| { + format!( + "objdump -T {} shows no GLIBC_ versioned symbols; cannot assert the floor", + binary.display() + ) + })?; + if exceeds_floor(&max_glibc, &options.glibc_floor)? { + bail!( + "binary requires glibc {max_glibc}, above the declared floor {}; the build container \ + no longer sets the floor it claims", + options.glibc_floor + ); + } + + // Package the .deb from the already-built binary. + run_step( + "cargo", + &[ + "deb", + "--package", + &options.package, + "--no-build", + "--target", + TARGET_LINUX, + "--deb-version", + &options.version, + ], + &package_dir, + &[], + ) + .await?; + let debian_dir = target_dir.join(TARGET_LINUX).join("debian"); + let mut debs: Vec = std::fs::read_dir(&debian_dir) + .with_context(|| format!("cannot list {}", debian_dir.display()))? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "deb")) + .collect(); + debs.sort(); + let Some(deb) = debs.into_iter().next() else { + bail!("cargo deb produced no .deb under {}", debian_dir.display()); + }; + + // Assemble the AppDir: binary, desktop entry, icon, AppRun. Nothing else. + let out = target_dir.join(TARGET_LINUX).join("appimage"); + if out.exists() { + std::fs::remove_dir_all(&out).with_context(|| format!("cannot clear {}", out.display()))?; + } + let appdir = out.join("AppDir"); + for dir in [ + appdir.join("usr/bin"), + appdir.join("usr/share/applications"), + appdir.join("usr/share/icons/hicolor/128x128/apps"), + ] { + std::fs::create_dir_all(&dir) + .with_context(|| format!("cannot create {}", dir.display()))?; + } + std::fs::copy(&binary, appdir.join("usr/bin").join(&options.binary_name)) + .context("cannot copy the binary into the AppDir")?; + + let desktop_file = format!("{}.desktop", options.binary_name); + let desktop_src = package_dir.join("packaging").join(&desktop_file); + let desktop_contents = std::fs::read_to_string(&desktop_src) + .with_context(|| format!("no desktop entry at {}", desktop_src.display()))?; + std::fs::copy(&desktop_src, appdir.join(&desktop_file))?; + std::fs::copy( + &desktop_src, + appdir.join("usr/share/applications").join(&desktop_file), + )?; + + // The desktop entry's Icon= names the icon file appimagetool looks for. + let icon_name = desktop_icon_name(&desktop_contents).with_context(|| { + format!( + "{} has no Icon= line; the AppImage needs one", + desktop_src.display() + ) + })?; + let icon_file = format!("{icon_name}.png"); + std::fs::copy(&icon, appdir.join(&icon_file)) + .with_context(|| format!("cannot copy the icon from {}", icon.display()))?; + std::fs::copy( + &icon, + appdir + .join("usr/share/icons/hicolor/128x128/apps") + .join(&icon_file), + )?; + + let apprun = appdir.join("AppRun"); + std::fs::write( + &apprun, + format!( + "#!/bin/sh\nHERE=\"$(dirname \"$(readlink -f \"$0\")\")\"\nexec \"$HERE/usr/bin/{}\" \"$@\"\n", + options.binary_name + ), + )?; + make_executable(&apprun)?; + + // Fetch appimagetool pinned by URL and digest. + let tool_bytes = download(APPIMAGETOOL_URL).await?; + let tool_digest = sha256_hex(&tool_bytes); + if tool_digest != APPIMAGETOOL_SHA256 { + bail!( + "appimagetool digest mismatch: expected {APPIMAGETOOL_SHA256}, downloaded {tool_digest}" + ); + } + let tool = out.join("appimagetool"); + std::fs::write(&tool, &tool_bytes)?; + make_executable(&tool)?; + + let appimage = out.join(format!( + "{}-{}-x86_64.AppImage", + options.binary_name, options.version + )); + run_step( + &tool.to_string_lossy(), + &[ + "--no-appstream", + &appdir.to_string_lossy(), + &appimage.to_string_lossy(), + ], + &out, + // No FUSE inside a container; extract-and-run instead. + &[("APPIMAGE_EXTRACT_AND_RUN", "1"), ("ARCH", "x86_64")], + ) + .await?; + if !appimage.is_file() { + bail!( + "appimagetool produced no AppImage at {}", + appimage.display() + ); + } + + // Prove the bundle-nothing rule held. + let contraband: Vec = WalkDir::new(&appdir) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| { + let name = entry.file_name().to_string_lossy(); + FORBIDDEN_LIB_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + }) + .map(|entry| entry.path().display().to_string()) + .collect(); + if !contraband.is_empty() { + bail!( + "AppDir contains graphics libraries that must come from the host: {}", + contraband.join(", ") + ); + } + + Ok(BundleLinuxResult { + deb, + appimage, + max_glibc_symbol: max_glibc, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const OBJDUMP_231: &str = "\ +DYNAMIC SYMBOL TABLE: +0000000000000000 DF *UND*\t0000000000000000 (GLIBC_2.2.5) __libc_start_main +0000000000000000 DF *UND*\t0000000000000000 (GLIBC_2.3.4) __printf_chk +0000000000000000 DF *UND*\t0000000000000000 (GLIBC_2.31) pthread_cond_clockwait +0000000000000000 DF *UND*\t0000000000000000 GLIBC_PRIVATE __libc_dlopen_mode +"; + + const OBJDUMP_234: &str = "\ +DYNAMIC SYMBOL TABLE: +0000000000000000 DF *UND*\t0000000000000000 (GLIBC_2.2.5) __libc_start_main +0000000000000000 DF *UND*\t0000000000000000 (GLIBC_2.34) pthread_create +"; + + #[test] + fn max_symbol_is_extracted_numerically() { + assert_eq!(max_glibc_symbol(OBJDUMP_231).as_deref(), Some("2.31")); + assert_eq!(max_glibc_symbol(OBJDUMP_234).as_deref(), Some("2.34")); + assert_eq!(max_glibc_symbol("no versioned symbols here"), None); + // GLIBC_PRIVATE alone is not a version. + assert_eq!(max_glibc_symbol("GLIBC_PRIVATE"), None); + } + + #[test] + fn floor_2_31_passes_a_2_31_binary_and_fails_a_2_34_one() { + let max = max_glibc_symbol(OBJDUMP_231).unwrap(); + assert!(!exceeds_floor(&max, "2.31").unwrap()); + let max = max_glibc_symbol(OBJDUMP_234).unwrap(); + assert!(exceeds_floor(&max, "2.31").unwrap()); + } + + #[test] + fn version_comparison_is_numeric_not_lexical() { + // String comparison would call 2.9 the bigger one. + assert!(!exceeds_floor("2.9", "2.31").unwrap()); + assert!(exceeds_floor("2.31", "2.9").unwrap()); + assert_eq!( + max_glibc_symbol("(GLIBC_2.9) a (GLIBC_2.31) b").as_deref(), + Some("2.31") + ); + assert!(exceeds_floor("2.2.6", "2.2.5").unwrap()); + assert!(exceeds_floor("x.y", "2.31").is_err()); + } + + #[test] + fn desktop_icon_line_is_parsed() { + assert_eq!( + desktop_icon_name( + "[Desktop Entry]\nName=SpatialEngine\nIcon=spatialengine\nExec=SpatialEngine\n" + ) + .as_deref(), + Some("spatialengine") + ); + assert_eq!( + desktop_icon_name(" Icon= spaced \n").as_deref(), + Some("spaced") + ); + assert_eq!(desktop_icon_name("[Desktop Entry]\nName=X\n"), None); + assert_eq!(desktop_icon_name("Icon=\n"), None); + } +} diff --git a/src/commands/release/classify.rs b/src/commands/release/classify.rs new file mode 100644 index 000000000..9cf29a234 --- /dev/null +++ b/src/commands/release/classify.rs @@ -0,0 +1,375 @@ +//! Classify a published GitHub Release for the application-release pipeline. +//! +//! Classification keys on the release TAG, never the name: this same tool's +//! `publish` command creates library releases named from the tag or with no +//! name at all (measured in fsl_libs: 29 of 652 published releases carry a +//! null name), so a name-keyed rule would fail a third of library publishes +//! instead of skipping them. The library/application boundary therefore has +//! exactly one owner: [`LIBRARY_TAG_PREFIX`], shared with the publish path. +//! +//! An application release must carry equal tag and name matching +//! `-`, where `` names a package that declares +//! `[package.metadata.fslabs.release]`. The app segment forbids a dash, so +//! it can never swallow part of a version or collide with `publish-*`. An +//! empty name is a hard failure rather than a tag fallback: a fallback would +//! silently reinstate the tag as the contract. + +use std::collections::BTreeMap; +use std::fmt::{Display, Formatter}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, bail}; +use clap::Parser; +use serde::Serialize; + +use super::types::AppReleaseConfig; +use crate::PrettyPrintable; + +/// The tag namespace owned by library crate publishing (`fslabscli publish` +/// under Prow). Application release tags must NEVER start with this, or they +/// route to the library flow, nothing builds, and nothing says why. +pub const LIBRARY_TAG_PREFIX: &str = "publish-"; + +#[derive(Debug, Parser, Clone)] +#[command( + about = "Classify a published GitHub Release as a library skip or a validated application release" +)] +pub struct Options { + /// The release tag (github.event.release.tag_name); always present. + #[arg(long, env = "RELEASE_TAG")] + pub tag: String, + /// The release name/title (github.event.release.name); may be empty. + #[arg(long, env = "RELEASE_NAME", default_value = "")] + pub name: String, + /// The pre-release checkbox (github.event.release.prerelease). + #[arg(long, env = "RELEASE_PRERELEASE")] + pub prerelease: bool, +} + +#[derive(Debug, Serialize, Clone, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + /// A library release created for a publish-* tag; skip cleanly. + LibrarySkip, + /// A valid application release; build it. + Proceed, +} + +#[derive(Debug, Serialize, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +#[serde(rename_all = "snake_case")] +pub enum Destination { + Preview, + Production, +} + +impl Display for Destination { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Destination::Preview => write!(f, "preview"), + Destination::Production => write!(f, "production"), + } + } +} + +#[derive(Debug, Serialize, Clone)] +pub struct ClassifyResult { + pub outcome: Outcome, + #[serde(skip_serializing_if = "Option::is_none")] + pub app: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub destination: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub config: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub package_directory: Option, +} + +impl Display for ClassifyResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.outcome { + Outcome::LibrarySkip => write!(f, "library release; nothing to bundle"), + Outcome::Proceed => write!( + f, + "{} {} -> {}", + self.app.as_deref().unwrap_or("?"), + self.version.as_deref().unwrap_or("?"), + self.destination.map(|d| d.to_string()).unwrap_or_default() + ), + } + } +} + +impl PrettyPrintable for ClassifyResult { + fn pretty_print(&self) -> String { + self.to_string() + } +} + +/// Split `-`: the app segment is `[a-z0-9_]+` (no dash), the +/// remainder must parse as a semantic version. +pub fn parse_release_name(name: &str) -> Option<(String, semver::Version)> { + // The app segment cannot contain '-', so the FIRST dash is the split. + let (app, version) = name.split_once('-')?; + if app.is_empty() + || !app + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') + { + return None; + } + let version = semver::Version::parse(version).ok()?; + Some((app.to_string(), version)) +} + +/// Discover applications: every git-tracked Cargo.toml declaring +/// `[package.metadata.fslabs.release]`, INCLUDING workspaces excluded from +/// the root one (fdk_apps is excluded by design and holds the only shipped +/// app), which is why this walks `git ls-files` rather than cargo metadata. +pub fn discover_apps( + repo_root: &Path, +) -> anyhow::Result> { + let output = Command::new("git") + .arg("-C") + .arg(repo_root) + .args(["ls-files", "*Cargo.toml", "**/Cargo.toml"]) + .output() + .context("git ls-files failed")?; + if !output.status.success() { + bail!( + "git ls-files failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + let mut apps = BTreeMap::new(); + for line in String::from_utf8(output.stdout)?.lines() { + let manifest_path = repo_root.join(line); + let Ok(contents) = std::fs::read_to_string(&manifest_path) else { + continue; + }; + // Cheap pre-filter before parsing every manifest in the tree. + if !contents.contains("[package.metadata.fslabs.release]") { + continue; + } + let value: toml::Value = toml::from_str(&contents) + .with_context(|| format!("unparseable manifest {}", manifest_path.display()))?; + let Some(package) = value.get("package") else { + continue; + }; + let Some(name) = package.get("name").and_then(|n| n.as_str()) else { + continue; + }; + let Some(release) = package + .get("metadata") + .and_then(|m| m.get("fslabs")) + .and_then(|f| f.get("release")) + else { + continue; + }; + let config: AppReleaseConfig = release.clone().try_into().with_context(|| { + format!( + "invalid [package.metadata.fslabs.release] in {}", + manifest_path.display() + ) + })?; + let dir = manifest_path + .parent() + .map(Path::to_path_buf) + .unwrap_or_default(); + if apps.insert(name.to_string(), (dir, config)).is_some() { + bail!("two packages named {name} declare [package.metadata.fslabs.release]"); + } + } + Ok(apps) +} + +pub fn classify( + options: &Options, + apps: &BTreeMap, + repo_root: &Path, +) -> anyhow::Result { + if options.tag.is_empty() { + bail!("release has no tag"); + } + if options.tag.starts_with(LIBRARY_TAG_PREFIX) { + return Ok(ClassifyResult { + outcome: Outcome::LibrarySkip, + app: None, + version: None, + destination: None, + config: None, + package_directory: None, + }); + } + + // From here on this claims to be an application release, so every defect + // is a hard failure. + if options.name.is_empty() || options.name == "null" { + bail!( + "release name must be set to - (got an empty name; set the release title)" + ); + } + if options.name != options.tag { + bail!( + "release name ({}) must equal the release tag ({})", + options.name, + options.tag + ); + } + let Some((app, version)) = parse_release_name(&options.name) else { + bail!( + "release name must be - with a semantic version; got '{}'", + options.name + ); + }; + let Some((dir, config)) = apps.get(&app) else { + bail!( + "unknown application '{app}'; configured applications (packages with [package.metadata.fslabs.release]): {}", + apps.keys().cloned().collect::>().join(", ") + ); + }; + + let destination = if options.prerelease { + Destination::Preview + } else { + if !version.pre.is_empty() || !version.build.is_empty() { + bail!( + "production versions must be plain X.Y.Z; '{version}' has a prerelease or build \ + component. Tick the pre-release checkbox for one-off builds." + ); + } + Destination::Production + }; + + let package_directory = dir + .strip_prefix(repo_root) + .unwrap_or(dir) + .to_string_lossy() + .to_string(); + + Ok(ClassifyResult { + outcome: Outcome::Proceed, + app: Some(app), + version: Some(version.to_string()), + destination: Some(destination), + config: Some(config.clone()), + package_directory: Some(package_directory), + }) +} + +pub async fn run(options: &Options, repo_root: PathBuf) -> anyhow::Result { + let apps = discover_apps(&repo_root)?; + classify(options, &apps, &repo_root) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn apps() -> BTreeMap { + let mut m = BTreeMap::new(); + m.insert( + "spatial_engine".to_string(), + ( + PathBuf::from("/repo/fdk_apps/spatial_engine"), + AppReleaseConfig { + verbose_name: "SpatialEngine".into(), + targets: vec![super::super::types::TARGET_WINDOWS.into()], + license_macos: None, + }, + ), + ); + m + } + + fn opts(tag: &str, name: &str, prerelease: bool) -> Options { + Options { + tag: tag.into(), + name: name.into(), + prerelease, + } + } + + fn run(tag: &str, name: &str, prerelease: bool) -> anyhow::Result { + classify(&opts(tag, name, prerelease), &apps(), Path::new("/repo")) + } + + #[test] + fn plain_production_release_proceeds() { + let r = run("spatial_engine-33.2.1", "spatial_engine-33.2.1", false).unwrap(); + assert_eq!(r.outcome, Outcome::Proceed); + assert_eq!(r.version.as_deref(), Some("33.2.1")); + assert_eq!(r.destination, Some(Destination::Production)); + assert_eq!( + r.package_directory.as_deref(), + Some("fdk_apps/spatial_engine") + ); + } + + #[test] + fn prerelease_routes_to_preview() { + let r = run( + "spatial_engine-33.2.1-rc.1", + "spatial_engine-33.2.1-rc.1", + true, + ) + .unwrap(); + assert_eq!(r.destination, Some(Destination::Preview)); + } + + #[test] + fn library_tags_skip_regardless_of_name_shape() { + for (tag, name) in [ + ("publish-fdk-33.0.0", "publish-fdk-33.0.0"), + ("publish-dagger_cli-32.0.0", "null"), + ("publish-fse_state-0.14.2", ""), + ] { + let r = run(tag, name, false).unwrap(); + assert_eq!(r.outcome, Outcome::LibrarySkip, "tag {tag}"); + } + } + + #[test] + fn production_with_prerelease_version_fails() { + assert!( + run( + "spatial_engine-33.2.1-rc.1", + "spatial_engine-33.2.1-rc.1", + false + ) + .is_err() + ); + } + + #[test] + fn invalid_shapes_fail() { + // Uppercase app segment. + assert!(run("SpatialEngine-33.2.1", "SpatialEngine-33.2.1", false).is_err()); + // Two-component version. + assert!(run("spatial_engine-33.2", "spatial_engine-33.2", false).is_err()); + // v-prefixed version. + assert!(run("spatial_engine-v33.2.1", "spatial_engine-v33.2.1", false).is_err()); + // Unknown application (matches the grammar, is not configured). + assert!(run("unknown_app-1.0.0", "unknown_app-1.0.0", false).is_err()); + // Crate-shaped name that mismatches the tag. + assert!(run("fse_state-0.14.2", "fse_state-0.14.3", false).is_err()); + // No tag at all. + assert!(run("", "spatial_engine-33.2.1", false).is_err()); + } + + #[test] + fn empty_name_is_a_hard_failure_never_a_tag_fallback() { + assert!(run("spatial_engine-33.2.1", "", false).is_err()); + assert!(run("spatial_engine-33.2.1", "null", false).is_err()); + } + + #[test] + fn unknown_app_error_lists_configured_apps() { + let err = run("unknown_app-1.0.0", "unknown_app-1.0.0", false) + .unwrap_err() + .to_string(); + assert!(err.contains("spatial_engine"), "{err}"); + } +} diff --git a/src/commands/release/http.rs b/src/commands/release/http.rs new file mode 100644 index 000000000..bfa4fb80a --- /dev/null +++ b/src/commands/release/http.rs @@ -0,0 +1,73 @@ +//! The one HTTP client for the release commands, over the repository's +//! established hyper-rustls stack (reqwest is a dev-dependency only, and the +//! release surface needs exactly plain requests: credential-less GET/HEAD of +//! the published objects, streamed downloads, and record's small JSON API +//! calls). One module means one TLS, redirect, and error behaviour to +//! review. + +use anyhow::{Context, bail}; +use http_body_util::{BodyExt, Full}; +use hyper::body::Bytes; +use hyper::{Method, Request}; +use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; +use hyper_util::client::legacy::Client as HyperClient; +use hyper_util::client::legacy::connect::HttpConnector; +use hyper_util::rt::TokioExecutor; + +pub(crate) type Client = HyperClient, Full>; + +pub(crate) const MAX_REDIRECTS: usize = 5; + +pub(crate) fn client() -> anyhow::Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let tls_config = rustls::ClientConfig::builder() + .with_native_roots() + .context("no native TLS roots available")? + .with_no_client_auth(); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_tls_config(tls_config) + .https_or_http() + .enable_http1() + .build(); + Ok(HyperClient::builder(TokioExecutor::new()).build(https)) +} + +/// GET following up to [`MAX_REDIRECTS`] redirects (GitHub release downloads +/// 302 to object storage), succeeding only on a 2xx. +async fn get_response( + client: &Client, + url: &str, +) -> anyhow::Result> { + let mut current = url::Url::parse(url).with_context(|| format!("invalid url {url}"))?; + for _ in 0..=MAX_REDIRECTS { + let req = Request::builder() + .method(Method::GET) + .uri(current.as_str()) + .body(Full::new(Bytes::new()))?; + let res = client + .request(req) + .await + .with_context(|| format!("GET {current} failed"))?; + if res.status().is_redirection() { + let location = res + .headers() + .get(hyper::header::LOCATION) + .and_then(|v| v.to_str().ok()) + .with_context(|| format!("GET {current}: redirect without a Location header"))?; + current = current + .join(location) + .with_context(|| format!("GET {current}: unusable redirect to {location}"))?; + continue; + } + if !res.status().is_success() { + bail!("GET {current} returned {}", res.status()); + } + return Ok(res); + } + bail!("GET {url}: more than {MAX_REDIRECTS} redirects") +} + +pub(crate) async fn get_bytes(client: &Client, url: &str) -> anyhow::Result> { + let response = get_response(client, url).await?; + Ok(response.into_body().collect().await?.to_bytes().to_vec()) +} diff --git a/src/commands/release/keyring.rs b/src/commands/release/keyring.rs new file mode 100644 index 000000000..f82933339 --- /dev/null +++ b/src/commands/release/keyring.rs @@ -0,0 +1,71 @@ +//! A gpg keyring in a throwaway GNUPGHOME holding exactly the published +//! release key, so verification can never be satisfied by an unrelated key +//! already on the host. Used by publish (re-verifying detached signatures +//! before anything is written) and resolve (client-side verification). + +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +use anyhow::{Context, bail}; + +pub(crate) struct TempKeyring { + home: tempfile::TempDir, +} + +impl TempKeyring { + pub(crate) fn import(key: &[u8]) -> anyhow::Result { + let home = tempfile::tempdir().context("cannot create a temporary GNUPGHOME")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(home.path(), std::fs::Permissions::from_mode(0o700))?; + } + let keyring = Self { home }; + let mut child = keyring + .gpg() + .args(["--batch", "--quiet", "--import"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .context("cannot run gpg")?; + child + .stdin + .as_mut() + .expect("gpg stdin is piped") + .write_all(key)?; + let output = child.wait_with_output()?; + if !output.status.success() { + bail!( + "gpg --import failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(keyring) + } + + fn gpg(&self) -> Command { + let mut command = Command::new("gpg"); + command.env("GNUPGHOME", self.home.path()); + command + } + + pub(crate) fn verify_detached(&self, signature: &Path, file: &Path) -> anyhow::Result<()> { + let output = self + .gpg() + .arg("--verify") + .arg(signature) + .arg(file) + .output() + .context("cannot run gpg")?; + if !output.status.success() { + bail!( + "gpg --verify failed for {}: {}", + file.display(), + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) + } +} diff --git a/src/commands/release/mod.rs b/src/commands/release/mod.rs new file mode 100644 index 000000000..ff7ea4a3d --- /dev/null +++ b/src/commands/release/mod.rs @@ -0,0 +1,120 @@ +//! Application-release pipeline: the logic behind fsl_libs' +//! `bundle_and_sign.yaml`, `promote_release.yaml` and +//! `release_healthcheck.yaml`, which are thin invocations of these +//! subcommands. The contract (manifest/index/channels shapes, the +//! library-vs-application tag boundary, storage semantics) lives HERE, in one +//! versioned place, and the workflows pin a released fslabscli binary. +//! +//! Storage rules in brief: production objects are immutable via conditional +//! writes (`probe-store` is the go/no-go gate against a deployed MinIO); the +//! manifest is written last and its existence is the atomic commit point; +//! monotonicity is derived from committed manifests, never index.json; +//! channel pointers move only through `promote`, whose backward-move gate +//! re-runs inside the compare-and-swap retry loop. + +use std::fmt::{Display, Formatter}; +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; +use serde::Serialize; + +use crate::PrettyPrintable; + +pub mod bundle_linux; +pub mod classify; +pub(crate) mod http; +pub(crate) mod keyring; +pub mod probe_store; +pub mod promote; +pub mod publish; +pub mod sign_linux; +pub mod store; +pub mod types; +pub mod verify_production; + +#[derive(Debug, Parser)] +#[command(about = "Application release pipeline: classify, publish, promote, resolve, verify")] +pub struct Options { + #[command(subcommand)] + command: ReleaseCommands, +} + +#[derive(Debug, Subcommand)] +enum ReleaseCommands { + /// Classify a published GitHub Release: library skip or validated app release + Classify(Box), + /// Verify a release commit is eligible for production publication + VerifyProduction(Box), + /// Probe the object store for conditional-write support (go/no-go gate) + ProbeStore(Box), + /// Atomically publish signed artifacts: digests, manifest-last, index CAS + Publish(Box), + /// Move a channel pointer in channels.json (the only writer) + Promote(Box), + /// Build the Linux .deb and AppImage in the pinned floor container + BundleLinux(Box), + /// Detach-sign Linux artifacts with the org OpenPGP key + SignLinux(Box), +} + +#[derive(Serialize)] +#[serde(untagged)] +pub enum ReleaseResult { + Classify(classify::ClassifyResult), + VerifyProduction(verify_production::VerifyProductionResult), + ProbeStore(probe_store::ProbeStoreResult), + Publish(publish::PublishResult), + Promote(promote::PromoteResult), + BundleLinux(bundle_linux::BundleLinuxResult), + SignLinux(sign_linux::SignLinuxResult), +} + +impl Display for ReleaseResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ReleaseResult::Classify(r) => r.fmt(f), + ReleaseResult::VerifyProduction(r) => r.fmt(f), + ReleaseResult::ProbeStore(r) => r.fmt(f), + ReleaseResult::Publish(r) => r.fmt(f), + ReleaseResult::Promote(r) => r.fmt(f), + ReleaseResult::BundleLinux(r) => r.fmt(f), + ReleaseResult::SignLinux(r) => r.fmt(f), + } + } +} + +impl PrettyPrintable for ReleaseResult { + fn pretty_print(&self) -> String { + match self { + ReleaseResult::Classify(r) => r.pretty_print(), + ReleaseResult::VerifyProduction(r) => r.pretty_print(), + ReleaseResult::ProbeStore(r) => r.pretty_print(), + ReleaseResult::Publish(r) => r.pretty_print(), + ReleaseResult::Promote(r) => r.pretty_print(), + ReleaseResult::BundleLinux(r) => r.pretty_print(), + ReleaseResult::SignLinux(r) => r.pretty_print(), + } + } +} + +pub async fn release( + options: Box, + working_directory: PathBuf, + repo_root: PathBuf, +) -> anyhow::Result { + match options.command { + ReleaseCommands::Classify(o) => classify::run(&o, repo_root) + .await + .map(ReleaseResult::Classify), + ReleaseCommands::VerifyProduction(o) => verify_production::run(&o, repo_root) + .await + .map(ReleaseResult::VerifyProduction), + ReleaseCommands::ProbeStore(o) => probe_store::run(&o).await.map(ReleaseResult::ProbeStore), + ReleaseCommands::Publish(o) => publish::run(&o).await.map(ReleaseResult::Publish), + ReleaseCommands::Promote(o) => promote::run(&o).await.map(ReleaseResult::Promote), + ReleaseCommands::BundleLinux(o) => bundle_linux::run(&o, working_directory) + .await + .map(ReleaseResult::BundleLinux), + ReleaseCommands::SignLinux(o) => sign_linux::run(&o).await.map(ReleaseResult::SignLinux), + } +} diff --git a/src/commands/release/probe_store.rs b/src/commands/release/probe_store.rs new file mode 100644 index 000000000..9c0e0a1f2 --- /dev/null +++ b/src/commands/release/probe_store.rs @@ -0,0 +1,163 @@ +//! Probe whether the deployed object store supports S3 conditional writes. +//! +//! Conditional writes are the only mechanism that makes a published object +//! immutable: object lock plus versioning stops deletion, but a plain re-PUT +//! still succeeds and becomes the new current version. Run this once against +//! the real endpoint with the publisher credential before `release publish` +//! may be trusted, and paste the output into the PR that enables +//! publication. Any FAIL means publication must not be enabled until the +//! store is upgraded. + +use std::fmt::{Display, Formatter}; + +use clap::Parser; +use serde::Serialize; + +use super::store::{ReleaseStore, sha256_hex}; +use crate::PrettyPrintable; + +#[derive(Debug, Parser, Clone)] +#[command(about = "Probe the object store for conditional-write support (go/no-go gate)")] +pub struct Options { + /// Bucket to probe (use the preview bucket, never production). + #[arg(long)] + pub bucket: String, + /// Key prefix for the probe objects (they are left behind; the preview + /// bucket's lifecycle expiry cleans them). + #[arg(long, default_value = "_probe")] + pub prefix: String, +} + +#[derive(Debug, Serialize, Clone)] +pub struct ProbeCase { + pub description: String, + pub passed: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Debug, Serialize, Clone)] +pub struct ProbeStoreResult { + pub bucket: String, + pub cases: Vec, + pub passed: bool, +} + +impl Display for ProbeStoreResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + writeln!(f, "conditional-write probe against bucket {}", self.bucket)?; + for case in &self.cases { + writeln!( + f, + "{}: {}{}", + if case.passed { "PASS" } else { "FAIL" }, + case.description, + case.detail + .as_deref() + .map(|d| format!(" ({d})")) + .unwrap_or_default() + )?; + } + write!( + f, + "{}", + if self.passed { + "conditional writes supported; publication may rely on immutability" + } else { + "CONDITIONAL WRITES UNSUPPORTED; do not enable publication against this store" + } + ) + } +} + +impl PrettyPrintable for ProbeStoreResult { + fn pretty_print(&self) -> String { + self.to_string() + } +} + +pub async fn run(options: &Options) -> anyhow::Result { + let store = ReleaseStore::from_env(&options.bucket)?; + // A per-invocation key: the probe must always exercise the fresh-key case. + let nonce = sha256_hex( + format!( + "{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() + ) + .as_bytes(), + ); + let key = format!("{}/{}", options.prefix, &nonce[..16]); + let body_one = b"probe-body-1".to_vec(); + let body_two = b"probe-body-2".to_vec(); + + let mut cases = Vec::new(); + let mut case = |description: &str, passed: bool, detail: Option| { + cases.push(ProbeCase { + description: description.to_string(), + passed, + detail, + }); + }; + + let first = store.put_immutable(&key, body_one.clone()).await; + case( + "If-None-Match:* on a fresh key succeeds", + first.is_ok(), + first.err().map(|e| format!("{e:#}")), + ); + + let second = store.put_immutable(&key, body_two.clone()).await; + let refused = second + .as_ref() + .err() + .and_then(|e| e.downcast_ref::()) + .is_some(); + case( + "If-None-Match:* on an existing key is refused", + refused, + (!refused).then(|| { + second + .err() + .map(|e| format!("{e:#}")) + .unwrap_or_else(|| "write succeeded".into()) + }), + ); + + // If-Match is exercised through the CAS path on a second, JSON-seeded + // key: create via If-None-Match, then update via If-Match. + let cas_key = format!("{key}-cas"); + store + .put_immutable( + &cas_key, + serde_json::to_vec(&serde_json::json!({"probe": "seed"}))?, + ) + .await?; + let cas: anyhow::Result = store + .cas_update(&cas_key, None, |_current: serde_json::Value| { + Ok(serde_json::json!({"probe": "updated"})) + }) + .await; + let cas_ok = cas.is_ok(); + case( + "If-Match with the current ETag succeeds (CAS update)", + cas_ok, + cas.err().map(|e| format!("{e:#}")), + ); + + let readback = store.read(&key).await?; + case( + "refused overwrite left the original bytes", + readback == body_one, + None, + ); + + let passed = cases.iter().all(|c| c.passed); + Ok(ProbeStoreResult { + bucket: options.bucket.clone(), + cases, + passed, + }) +} diff --git a/src/commands/release/promote.rs b/src/commands/release/promote.rs new file mode 100644 index 000000000..02b8da45d --- /dev/null +++ b/src/commands/release/promote.rs @@ -0,0 +1,409 @@ +//! Move a channel pointer in channels.json. This command is the ONLY writer +//! of channel state, runs with the promotion credential (which cannot write +//! artifacts; the publication credential cannot write channels.json), and is +//! also the rollback procedure: a backward move with `--allow-backward` and +//! a `--reason`. +//! +//! Validation before any write, each with a distinct message: the manifest +//! must exist in the PRODUCTION bucket (a preview can never be promoted), it +//! must list an artifact for every selected target, and each selected +//! artifact is re-read and digest-compared, so promotion re-verifies rather +//! than trusting a record. The write is a compare-and-swap whose transform +//! AND backward gate re-run against the freshly-read document on every +//! retry: a promotion that loses a race can never apply a decision made +//! against a stale pointer. + +use std::collections::BTreeMap; +use std::fmt::{Display, Formatter}; + +use anyhow::{Context, bail}; +use clap::Parser; +use serde::Serialize; + +use super::store::{ReleaseStore, semver_gt}; +use super::types::{ + Channel, ChannelPointers, Channels, Manifest, PROVENANCE_CAP, ProvenanceEntry, SCHEMA_VERSION, +}; +use crate::PrettyPrintable; + +#[derive(Debug, Parser, Clone)] +#[command( + about = "Move a channel pointer in channels.json (the only writer)", + disable_version_flag = true +)] +pub struct Options { + #[arg(long)] + pub app: String, + /// Published PRODUCTION version to point at. + #[arg(long)] + pub version: String, + #[arg(long, value_enum)] + pub channel: Channel, + /// Comma-separated target triples whose pointers move; unselected + /// targets are untouched. + #[arg(long, value_delimiter = ',')] + pub targets: Vec, + /// Acknowledge moving a pointer to an OLDER version (rollback). + #[arg(long, default_value_t = false)] + pub allow_backward: bool, + /// Required when --allow-backward is set. + #[arg(long)] + pub reason: Option, + #[arg(long, default_value = "")] + pub moved_by: String, + #[arg(long, default_value = "")] + pub run_url: String, + #[arg(long, default_value = "fsl-releases-channels")] + pub channels_bucket: String, + #[arg(long, default_value = "fsl-releases")] + pub releases_bucket: String, + #[arg( + long, + env = "RELEASE_PUBLIC_BASE_URL", + default_value = "https://api.s3.fsl.dev" + )] + pub base_url: String, + /// Skip the per-artifact digest re-verification (tests only). + #[arg(long, default_value_t = false)] + pub skip_digest_verification: bool, +} + +#[derive(Debug, Serialize, Clone)] +pub struct PromoteResult { + pub channel: Channel, + pub version: String, + pub targets: Vec, + pub backward: bool, + /// The document as written. + pub channels: Channels, +} + +impl Display for PromoteResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "moved {} to {} for {}{}", + self.channel, + self.version, + self.targets.join(", "), + if self.backward { " (BACKWARD)" } else { "" } + ) + } +} + +impl PrettyPrintable for PromoteResult { + fn pretty_print(&self) -> String { + self.to_string() + } +} + +/// Apply one pointer move to a channels document: the backward gate and the +/// provenance append. Pure, so the gate is unit-tested and the CAS loop +/// re-runs it on every retry. Errors with a "backward move refused" message +/// when a selected target currently points at a newer version and +/// `allow_backward` is false. +pub fn apply_move( + mut doc: Channels, + options: &Options, + now: &str, +) -> anyhow::Result<(Channels, bool)> { + let pointers = doc.channels.get(options.channel); + let mut backward_targets = Vec::new(); + for target in &options.targets { + if let Some(current) = pointers.get(target) + && semver_gt(current, &options.version)? + { + backward_targets.push((target.clone(), current.clone())); + } + } + if !backward_targets.is_empty() && !options.allow_backward { + let refusals: Vec = backward_targets + .iter() + .map(|(target, current)| { + format!( + "backward move refused: {}/{target} currently points at {current}, \ + which is newer than {}; re-run with --allow-backward and a --reason \ + if this is an intentional rollback", + options.channel, options.version + ) + }) + .collect(); + bail!("{}", refusals.join("\n")); + } + let backward = !backward_targets.is_empty(); + + let from: BTreeMap> = options + .targets + .iter() + .map(|target| (target.clone(), pointers.get(target).cloned())) + .collect(); + let pointers = doc.channels.get_mut(options.channel); + for target in &options.targets { + pointers.insert(target.clone(), options.version.clone()); + } + doc.updated_at = now.to_string(); + doc.provenance.insert( + 0, + ProvenanceEntry { + channel: options.channel, + targets: options.targets.clone(), + from, + to: options.version.clone(), + backward, + reason: options.reason.clone(), + moved_at: now.to_string(), + moved_by: options.moved_by.clone(), + run: options.run_url.clone(), + }, + ); + doc.provenance.truncate(PROVENANCE_CAP); + Ok((doc, backward)) +} + +pub async fn run(options: &Options) -> anyhow::Result { + // Validation precedes ALL network I/O. + if options.allow_backward && options.reason.as_deref().is_none_or(str::is_empty) { + bail!("--allow-backward requires --reason"); + } + if options.targets.is_empty() { + bail!("at least one target must be selected"); + } + + // The manifest must exist in the PRODUCTION bucket; previews never do. + let releases = ReleaseStore::from_env(&options.releases_bucket)?; + let manifest_key = format!("{}/{}/manifest.json", options.app, options.version); + if !releases.exists(&manifest_key).await? { + bail!( + "no production manifest for {} {}: only published production versions can be promoted (previews never)", + options.app, + options.version + ); + } + let manifest: Manifest = serde_json::from_slice(&releases.read(&manifest_key).await?) + .with_context(|| { + format!( + "s3://{}/{manifest_key} is not a valid manifest", + options.releases_bucket + ) + })?; + for target in &options.targets { + if !manifest.artifacts.iter().any(|a| a.target == *target) { + bail!( + "manifest for {} {} lists no artifact for target {target}", + options.app, + options.version + ); + } + } + + // Re-read and digest-compare every selected artifact: promotion + // re-verifies rather than trusting a record. + if !options.skip_digest_verification { + for artifact in manifest + .artifacts + .iter() + .filter(|a| options.targets.contains(&a.target)) + { + let key = releases.key_from_url(&artifact.url)?; + releases + .verify_key(key, &artifact.sha256) + .await + .with_context(|| { + format!( + "artifact {key} no longer matches its recorded digest; refusing to promote" + ) + })?; + } + } + + let channels_store = ReleaseStore::from_env(&options.channels_bucket)?; + let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let initial = Channels { + schema_version: SCHEMA_VERSION, + app: options.app.clone(), + updated_at: now.clone(), + manifest_base: format!("{}/{}", options.base_url, options.releases_bucket), + channels: ChannelPointers::default(), + provenance: Vec::new(), + }; + let backward = std::cell::Cell::new(false); + let doc = channels_store + .cas_update( + &format!("{}/channels.json", options.app), + Some(initial), + |doc: Channels| { + // Re-applied to the FRESHLY READ document on every retry, so + // the backward gate can never act on a stale pointer. Its + // refusal propagates out of the CAS loop. + let (next, moved_backward) = apply_move(doc, options, &now)?; + backward.set(moved_backward); + Ok(next) + }, + ) + .await?; + + Ok(PromoteResult { + channel: options.channel, + version: options.version.clone(), + targets: options.targets.clone(), + backward: backward.get(), + channels: doc, + }) +} + +#[cfg(test)] +mod tests { + use super::super::types::{TARGET_LINUX, TARGET_MACOS, TARGET_WINDOWS}; + use super::*; + + fn opts(version: &str, channel: Channel, targets: &[&str]) -> Options { + Options { + app: "app".into(), + version: version.into(), + channel, + targets: targets.iter().map(|t| t.to_string()).collect(), + allow_backward: false, + reason: None, + moved_by: "tester".into(), + run_url: "http://run".into(), + channels_bucket: "channels".into(), + releases_bucket: "releases".into(), + base_url: "https://api.s3.fsl.dev".into(), + skip_digest_verification: true, + } + } + + fn doc() -> Channels { + let mut latest = BTreeMap::new(); + latest.insert(TARGET_WINDOWS.to_string(), "1.1.0".to_string()); + latest.insert(TARGET_MACOS.to_string(), "1.0.0".to_string()); + Channels { + schema_version: SCHEMA_VERSION, + app: "app".into(), + updated_at: "t0".into(), + manifest_base: "https://api.s3.fsl.dev/releases".into(), + channels: ChannelPointers { + latest, + stable: BTreeMap::new(), + }, + provenance: Vec::new(), + } + } + + #[test] + fn forward_move_touches_only_selected_targets() { + let options = opts("1.2.0", Channel::Latest, &[TARGET_WINDOWS]); + let (next, backward) = apply_move(doc(), &options, "t1").unwrap(); + assert!(!backward); + assert_eq!(next.channels.latest.get(TARGET_WINDOWS).unwrap(), "1.2.0"); + assert_eq!( + next.channels.latest.get(TARGET_MACOS).unwrap(), + "1.0.0", + "unselected target moved" + ); + assert!(next.channels.stable.is_empty(), "other channel touched"); + assert_eq!(next.updated_at, "t1"); + let entry = &next.provenance[0]; + assert_eq!(entry.to, "1.2.0"); + assert!(!entry.backward); + assert_eq!(entry.moved_at, "t1"); + assert_eq!(entry.moved_by, "tester"); + assert_eq!(entry.run, "http://run"); + } + + #[test] + fn stable_move_leaves_latest_untouched() { + let options = opts("1.0.0", Channel::Stable, &[TARGET_WINDOWS]); + let (next, backward) = apply_move(doc(), &options, "t1").unwrap(); + assert!(!backward); + assert_eq!(next.channels.stable.get(TARGET_WINDOWS).unwrap(), "1.0.0"); + assert_eq!(next.channels.latest.get(TARGET_WINDOWS).unwrap(), "1.1.0"); + } + + #[test] + fn backward_move_is_refused_without_the_acknowledgement() { + let options = opts("1.0.0", Channel::Latest, &[TARGET_WINDOWS]); + let err = apply_move(doc(), &options, "t1").unwrap_err().to_string(); + assert!(err.contains("backward move refused"), "{err}"); + assert!(err.contains(&format!("latest/{TARGET_WINDOWS}")), "{err}"); + assert!(err.contains("1.1.0"), "{err}"); + assert!(err.contains("--allow-backward"), "{err}"); + assert!(err.contains("--reason"), "{err}"); + } + + #[test] + fn one_backward_target_refuses_the_whole_move() { + // WIN is at 1.1.0 (backward), MAC at 1.0.0 (forward): still refused. + let options = opts("1.0.5", Channel::Latest, &[TARGET_WINDOWS, TARGET_MACOS]); + let err = apply_move(doc(), &options, "t1").unwrap_err().to_string(); + assert!(err.contains("backward move refused"), "{err}"); + } + + #[test] + fn acknowledged_backward_move_records_backward_and_reason() { + let mut options = opts("1.0.0", Channel::Latest, &[TARGET_WINDOWS]); + options.allow_backward = true; + options.reason = Some("1.1.0 crashes on start".into()); + let (next, backward) = apply_move(doc(), &options, "t1").unwrap(); + assert!(backward); + assert_eq!(next.channels.latest.get(TARGET_WINDOWS).unwrap(), "1.0.0"); + let entry = &next.provenance[0]; + assert!(entry.backward); + assert_eq!(entry.reason.as_deref(), Some("1.1.0 crashes on start")); + } + + #[test] + fn same_version_is_not_a_backward_move() { + let options = opts("1.1.0", Channel::Latest, &[TARGET_WINDOWS]); + let (next, backward) = apply_move(doc(), &options, "t1").unwrap(); + assert!(!backward); + assert_eq!(next.channels.latest.get(TARGET_WINDOWS).unwrap(), "1.1.0"); + } + + #[test] + fn from_records_previous_pointers_including_absent_ones() { + let options = opts("1.2.0", Channel::Latest, &[TARGET_WINDOWS, TARGET_LINUX]); + let (next, _) = apply_move(doc(), &options, "t1").unwrap(); + let from = &next.provenance[0].from; + assert_eq!(from.get(TARGET_WINDOWS).unwrap().as_deref(), Some("1.1.0")); + assert_eq!(from.get(TARGET_LINUX).unwrap(), &None); + assert_eq!(next.channels.latest.get(TARGET_LINUX).unwrap(), "1.2.0"); + } + + #[test] + fn provenance_is_capped() { + let mut current = doc(); + for i in 0..(PROVENANCE_CAP + 10) { + let options = opts( + &format!("1.1.{}", i + 1), + Channel::Latest, + &[TARGET_WINDOWS], + ); + (current, _) = apply_move(current, &options, "t1").unwrap(); + } + assert_eq!(current.provenance.len(), PROVENANCE_CAP); + // Newest first. + assert_eq!( + current.provenance[0].to, + format!("1.1.{}", PROVENANCE_CAP + 10) + ); + } + + #[tokio::test] + async fn allow_backward_without_reason_fails_before_any_io() { + for reason in [None, Some(String::new())] { + let mut options = opts("1.0.0", Channel::Latest, &[TARGET_WINDOWS]); + options.allow_backward = true; + options.reason = reason; + let err = run(&options).await.unwrap_err().to_string(); + assert!(err.contains("--allow-backward requires --reason"), "{err}"); + } + } + + #[tokio::test] + async fn empty_target_selection_fails_before_any_io() { + let options = opts("1.0.0", Channel::Latest, &[]); + let err = run(&options).await.unwrap_err().to_string(); + assert!(err.contains("at least one target"), "{err}"); + } +} diff --git a/src/commands/release/publish.rs b/src/commands/release/publish.rs new file mode 100644 index 000000000..11bc098cd --- /dev/null +++ b/src/commands/release/publish.rs @@ -0,0 +1,810 @@ +//! Atomic publication of a signed artifact set, ordered so nothing broken +//! can ever become published-and-immutable: +//! +//! 1. Collect the artifact directory and assert completeness against the +//! app's configured targets (windows: msi, macos: dmg, linux: deb AND +//! appimage, each linux artifact with a sibling `.asc`). +//! 2. Re-verify signatures against the bytes: `osslsigncode verify` for the +//! MSI, `gpg --verify` against the PUBLISHED key (fetched from +//! `keys/fsl-release-linux.asc` in the production bucket) for the Linux +//! detached signatures. DMG notarization is stapler-validated on the +//! macOS build host; a Linux publisher cannot re-check it. +//! 3. Digest everything and build the [`Manifest`]. +//! 4. Production only: [`ReleaseStore::assert_monotonic`] over committed +//! manifests. +//! 5. `put_immutable` every artifact under `///`, +//! then read each back and digest-compare (`verify_key`). +//! 6. `put_immutable` the manifest LAST: its existence is the atomic commit +//! point. A failed production publication permanently spends its version +//! number; the retry is a new version. +//! 7. `cas_update` the derived, repairable index (append entry, dedupe by +//! version, sort descending by semver). +//! +//! GitHub-release attachment of convenience copies stays in the workflow +//! (a `gh release upload` step); this command owns only the bucket. + +use std::collections::BTreeMap; +use std::fmt::{Display, Formatter}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, bail}; +use clap::Parser; +use serde::Serialize; + +use super::classify::Destination; +use super::http; +use super::keyring::TempKeyring; +use super::store::{ReleaseStore, sha256_hex}; +use super::types::{ + ArtifactFormat, ArtifactSignature, Index, IndexEntry, Manifest, ManifestArtifact, + SCHEMA_VERSION, TARGET_LINUX, TARGET_MACOS, TARGET_WINDOWS, +}; +use crate::PrettyPrintable; + +#[derive(Debug, Parser, Clone)] +#[command( + about = "Atomically publish signed artifacts: digests, manifest-last, index CAS", + disable_version_flag = true +)] +pub struct Options { + #[arg(long)] + pub app: String, + #[arg(long)] + pub version: String, + #[arg(long, value_enum)] + pub destination: Destination, + /// 40-hex commit the artifacts were built from. + #[arg(long)] + pub source_revision: String, + #[arg(long)] + pub release_name: String, + #[arg(long)] + pub release_id: Option, + #[arg(long)] + pub run_url: Option, + /// Directory holding every signed artifact (flat). + #[arg(long)] + pub artifacts_dir: PathBuf, + /// Comma-separated target triples the app is configured to ship + /// (completeness is asserted against this, not inferred from the dir). + #[arg(long, value_delimiter = ',')] + pub targets: Vec, + /// Public base URL recorded in manifest artifact URLs. + #[arg( + long, + env = "RELEASE_PUBLIC_BASE_URL", + default_value = "https://api.s3.fsl.dev" + )] + pub base_url: String, + #[arg(long, default_value = "fsl-releases")] + pub prod_bucket: String, + #[arg(long, default_value = "fsl-releases-preview")] + pub preview_bucket: String, + /// Fingerprint of the Linux signing key (required when Linux artifacts + /// are present; recorded in each detached-signature entry). + #[arg(long)] + pub linux_fingerprint: Option, + #[arg(long, default_value = "2.31")] + pub glibc_floor: String, + /// Skip the signature shell-outs (tests and stores without the key). + #[arg(long, default_value_t = false)] + pub skip_signature_verification: bool, +} + +#[derive(Debug, Serialize, Clone)] +pub struct PublishResult { + pub bucket: String, + pub manifest_key: String, + pub artifact_count: usize, + /// Per-artifact object keys that were written and read back. + pub written: Vec, + /// Versions that already had committed manifests (production only). + pub previously_published: Vec, + /// Targets present in the manifest, for the auto-latest promotion. + pub manifest_targets: Vec, + /// Local path of the manifest copy for GH-release attachment. + pub manifest_path: String, +} + +impl Display for PublishResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "published {} artifact(s); commit point s3://{}/{}", + self.artifact_count, self.bucket, self.manifest_key + )?; + for key in &self.written { + writeln!(f, " {key}")?; + } + Ok(()) + } +} + +impl PrettyPrintable for PublishResult { + fn pretty_print(&self) -> String { + self.to_string() + } +} + +/// Map a directory of signed artifacts to (file, target, format) triples and +/// assert completeness against the configured targets. Pure; unit-tested. +pub fn collect_artifacts( + dir: &std::path::Path, + targets: &[String], +) -> anyhow::Result> { + let mut paths: Vec = std::fs::read_dir(dir) + .with_context(|| format!("cannot read artifact directory {}", dir.display()))? + .map(|entry| Ok(entry?.path())) + .collect::>()?; + paths.sort(); + + let mut artifacts = Vec::new(); + for path in paths { + let name = path + .file_name() + .and_then(|n| n.to_str()) + .with_context(|| format!("non-UTF-8 artifact name: {}", path.display()))? + .to_string(); + let (target, format) = if name.ends_with(".msi") { + (TARGET_WINDOWS, ArtifactFormat::Msi) + } else if name.ends_with(".dmg") { + (TARGET_MACOS, ArtifactFormat::Dmg) + } else if name.ends_with(".deb") { + (TARGET_LINUX, ArtifactFormat::Deb) + } else if name.ends_with(".AppImage") { + (TARGET_LINUX, ArtifactFormat::Appimage) + } else if name.ends_with(".asc") { + // Recorded alongside the artifact they sign, below. + continue; + } else { + bail!("unrecognised artifact {name}"); + }; + if target == TARGET_LINUX && !asc_sibling(&path).is_file() { + bail!("missing detached signature {name}.asc"); + } + artifacts.push((path, target.to_string(), format)); + } + if artifacts.is_empty() { + bail!("no artifacts found in {}", dir.display()); + } + + // Completeness is asserted against the CONFIGURED target list: fail-fast + // is off in the build matrix and a re-run can hand over a partial set, so + // presence is never inferred from the directory. + for target in targets { + let formats: Vec = artifacts + .iter() + .filter(|(_, t, _)| t == target) + .map(|&(_, _, format)| format) + .collect(); + if target == TARGET_LINUX { + if formats.len() != 2 + || !formats.contains(&ArtifactFormat::Deb) + || !formats.contains(&ArtifactFormat::Appimage) + { + bail!( + "target {target} requires exactly one .deb and one .AppImage; found {} artifact(s)", + formats.len() + ); + } + } else if formats.len() != 1 { + bail!( + "target {target} requires exactly one artifact; found {}", + formats.len() + ); + } + } + for (path, target, _) in &artifacts { + if !targets.iter().any(|t| t == target) { + bail!( + "artifact {} is for target {target}, which the app is not configured to ship", + path.file_name().unwrap_or_default().to_string_lossy() + ); + } + } + Ok(artifacts) +} + +fn asc_sibling(path: &Path) -> PathBuf { + let mut asc = path.to_path_buf().into_os_string(); + asc.push(".asc"); + PathBuf::from(asc) +} + +/// Build the manifest and per-file object keys. Pure; unit-tested. +#[allow(clippy::too_many_arguments)] +pub fn build_manifest( + options: &Options, + bucket: &str, + artifacts: &[(PathBuf, String, super::types::ArtifactFormat)], + digests: &BTreeMap, +) -> anyhow::Result<(super::types::Manifest, Vec<(PathBuf, String)>)> { + if options.source_revision.len() != 40 + || !options + .source_revision + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + bail!( + "source revision must be 40 hex, got '{}'", + options.source_revision + ); + } + if artifacts.is_empty() { + bail!("no artifacts to publish"); + } + + let mut uploads = Vec::new(); + let mut manifest_artifacts = Vec::new(); + for (path, target, format) in artifacts { + let filename = path + .file_name() + .and_then(|n| n.to_str()) + .with_context(|| format!("non-UTF-8 artifact name: {}", path.display()))? + .to_string(); + let (sha256, size_bytes) = digests + .get(path) + .with_context(|| format!("no digest recorded for {}", path.display()))? + .clone(); + let key = format!("{}/{}/{target}/{filename}", options.app, options.version); + let url = format!("{}/{bucket}/{key}", options.base_url); + uploads.push((path.clone(), key)); + + let (signature, glibc_floor) = match format { + ArtifactFormat::Msi => (ArtifactSignature::Authenticode, None), + ArtifactFormat::Dmg => (ArtifactSignature::AppleNotarized, None), + ArtifactFormat::Deb | ArtifactFormat::Appimage => { + let Some(fingerprint) = options + .linux_fingerprint + .as_deref() + .filter(|f| !f.is_empty()) + else { + bail!("--linux-fingerprint is required when Linux artifacts are present"); + }; + let asc_key = format!( + "{}/{}/{target}/{filename}.asc", + options.app, options.version + ); + uploads.push((asc_sibling(path), asc_key.clone())); + ( + ArtifactSignature::OpenpgpDetached { + url: format!("{}/{bucket}/{asc_key}", options.base_url), + key_fingerprint: fingerprint.to_string(), + }, + Some(options.glibc_floor.clone()), + ) + } + }; + manifest_artifacts.push(ManifestArtifact { + target: target.clone(), + format: *format, + filename, + url, + size_bytes, + sha256, + signature, + glibc_floor, + }); + } + + let manifest = Manifest { + schema_version: SCHEMA_VERSION, + app: options.app.clone(), + version: options.version.clone(), + prerelease: matches!(options.destination, Destination::Preview), + source_revision: options.source_revision.clone(), + release_name: options.release_name.clone(), + release_id: options.release_id.clone(), + workflow_run: options.run_url.clone(), + published_at: now_utc_seconds(), + artifacts: manifest_artifacts, + }; + Ok((manifest, uploads)) +} + +fn now_utc_seconds() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +/// Unique targets present in the manifest, sorted. +fn manifest_targets(manifest: &Manifest) -> Vec { + let mut targets: Vec = manifest + .artifacts + .iter() + .map(|a| a.target.clone()) + .collect(); + targets.sort(); + targets.dedup(); + targets +} + +/// Fold one publication into the index: dedupe by version keeping the NEW +/// entry, sort descending with real semver precedence (preview versions may +/// carry prerelease suffixes; anything unparseable sorts last), and stamp +/// `updated_at`. Pure; runs inside the CAS retry loop. +fn index_with_entry(mut index: Index, entry: IndexEntry, now: &str) -> Index { + index + .versions + .retain(|existing| existing.version != entry.version); + index.versions.push(entry); + index.versions.sort_by(|a, b| { + use std::cmp::Ordering; + match ( + semver::Version::parse(&a.version).ok(), + semver::Version::parse(&b.version).ok(), + ) { + (Some(va), Some(vb)) => vb.cmp(&va), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => b.version.cmp(&a.version), + } + }); + index.updated_at = now.to_string(); + index +} + +/// Re-verify each signature against the artifact BYTES, never build-job +/// state: osslsigncode for the MSI, gpg against the PUBLISHED key for the +/// Linux detached signatures. The DMG is deliberately absent here: +/// notarization is stapler-validated on the macOS build host, and a Linux +/// publisher cannot re-check it. +async fn verify_signatures( + options: &Options, + artifacts: &[(PathBuf, String, ArtifactFormat)], +) -> anyhow::Result<()> { + for (path, _, format) in artifacts { + if *format != ArtifactFormat::Msi { + continue; + } + let output = Command::new("osslsigncode") + .arg("verify") + .arg("-in") + .arg(path) + .output() + .context("cannot run osslsigncode")?; + if !output.status.success() { + bail!( + "Authenticode verification failed for {}: {}{}", + path.file_name().unwrap_or_default().to_string_lossy(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + } + + let linux: Vec<&PathBuf> = artifacts + .iter() + .filter(|(_, _, format)| matches!(format, ArtifactFormat::Deb | ArtifactFormat::Appimage)) + .map(|(path, _, _)| path) + .collect(); + if linux.is_empty() { + return Ok(()); + } + let key_url = format!( + "{}/{}/keys/fsl-release-linux.asc", + options.base_url, options.prod_bucket + ); + let key = http::get_bytes(&http::client()?, &key_url).await.with_context(|| { + format!("cannot fetch the published Linux signing key from {key_url}; publish it before releasing") + })?; + let keyring = TempKeyring::import(&key) + .with_context(|| format!("cannot import the published Linux signing key from {key_url}"))?; + for path in linux { + keyring + .verify_detached(&asc_sibling(path), path) + .with_context(|| { + format!( + "detached signature verification failed for {}", + path.file_name().unwrap_or_default().to_string_lossy() + ) + })?; + } + Ok(()) +} + +pub async fn run(options: &Options) -> anyhow::Result { + let artifacts = collect_artifacts(&options.artifacts_dir, &options.targets)?; + + if !options.skip_signature_verification { + verify_signatures(options, &artifacts).await?; + } + + let mut digests = BTreeMap::new(); + for (path, _, _) in &artifacts { + let bytes = + std::fs::read(path).with_context(|| format!("cannot read {}", path.display()))?; + digests.insert(path.clone(), (sha256_hex(&bytes), bytes.len() as u64)); + } + + let bucket = match options.destination { + Destination::Production => options.prod_bucket.as_str(), + Destination::Preview => options.preview_bucket.as_str(), + }; + let (manifest, uploads) = build_manifest(options, bucket, &artifacts, &digests)?; + + let store = ReleaseStore::from_env(bucket)?; + // Production versions increase strictly over COMMITTED manifests; a + // preview may republish freely under new prerelease suffixes. + let previously_published = match options.destination { + Destination::Production => { + store + .assert_monotonic(&options.app, &options.version) + .await? + } + Destination::Preview => Vec::new(), + }; + + let mut written = Vec::new(); + for (path, key) in &uploads { + let bytes = + std::fs::read(path).with_context(|| format!("cannot read {}", path.display()))?; + store.put_immutable(key, bytes).await?; + written.push(key.clone()); + } + // Read back and digest-compare every manifest artifact while no manifest + // exists yet, so nothing broken can become published-and-immutable. + for artifact in &manifest.artifacts { + let key = store.key_from_url(&artifact.url)?; + store.verify_key(key, &artifact.sha256).await?; + } + + // The manifest is written LAST: its existence is the atomic commit + // point. An AlreadyExists from the store propagates as-is; that version + // number is spent. + let manifest_key = format!("{}/{}/manifest.json", options.app, options.version); + let manifest_bytes = serde_json::to_vec_pretty(&manifest)?; + store + .put_immutable(&manifest_key, manifest_bytes.clone()) + .await?; + + // The index is derived and repairable; the committed manifests stay the + // authority. + let now = now_utc_seconds(); + let entry = IndexEntry { + version: manifest.version.clone(), + published_at: manifest.published_at.clone(), + source_revision: manifest.source_revision.clone(), + manifest: format!("{}/{bucket}/{manifest_key}", options.base_url), + targets: manifest_targets(&manifest), + }; + let initial = Index { + schema_version: SCHEMA_VERSION, + app: options.app.clone(), + updated_at: now.clone(), + versions: Vec::new(), + }; + store + .cas_update( + &format!("{}/index.json", options.app), + Some(initial), + |index: Index| Ok(index_with_entry(index, entry.clone(), &now)), + ) + .await?; + + // A local copy for the workflow to attach to the GitHub release + // (convenience only; the bucket stays authoritative). + let manifest_path = options.artifacts_dir.join("manifest.json"); + std::fs::write(&manifest_path, &manifest_bytes) + .with_context(|| format!("cannot write {}", manifest_path.display()))?; + + Ok(PublishResult { + bucket: bucket.to_string(), + manifest_key, + artifact_count: manifest.artifacts.len(), + written, + previously_published, + manifest_targets: manifest_targets(&manifest), + manifest_path: manifest_path.display().to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SHA_SRC: &str = "0123456789abcdef0123456789abcdef01234567"; + const FPR: &str = "593674FBF5A2221533824CA7E27819A5417615EE"; + + fn touch(dir: &Path, name: &str, contents: &str) { + std::fs::write(dir.join(name), contents).unwrap(); + } + + fn full_set(dir: &Path) { + touch(dir, "SpatialEngine-1.2.3.msi", "msi-bytes"); + touch(dir, "SpatialEngine-1.2.3.dmg", "dmg-bytes"); + touch(dir, "spatialengine_1.2.3_amd64.deb", "deb-bytes"); + touch(dir, "spatialengine_1.2.3_amd64.deb.asc", "deb-sig"); + touch(dir, "SpatialEngine-1.2.3-x86_64.AppImage", "appimage-bytes"); + touch( + dir, + "SpatialEngine-1.2.3-x86_64.AppImage.asc", + "appimage-sig", + ); + } + + fn all_targets() -> Vec { + vec![ + TARGET_WINDOWS.to_string(), + TARGET_MACOS.to_string(), + TARGET_LINUX.to_string(), + ] + } + + fn opts(dir: &Path, targets: Vec) -> Options { + Options { + app: "spatial_engine".into(), + version: "1.2.3".into(), + destination: Destination::Production, + source_revision: SHA_SRC.into(), + release_name: "spatial_engine-1.2.3".into(), + release_id: Some("42".into()), + run_url: Some("https://github.com/fslabs/fsl_libs/actions/runs/1".into()), + artifacts_dir: dir.to_path_buf(), + targets, + base_url: "https://api.s3.fsl.dev".into(), + prod_bucket: "fsl-releases".into(), + preview_bucket: "fsl-releases-preview".into(), + linux_fingerprint: Some(FPR.into()), + glibc_floor: "2.31".into(), + skip_signature_verification: true, + } + } + + fn digests_of( + artifacts: &[(PathBuf, String, ArtifactFormat)], + ) -> BTreeMap { + artifacts + .iter() + .map(|(path, _, _)| { + let bytes = std::fs::read(path).unwrap(); + (path.clone(), (sha256_hex(&bytes), bytes.len() as u64)) + }) + .collect() + } + + #[test] + fn complete_set_collects_four_artifacts() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + let artifacts = collect_artifacts(dir.path(), &all_targets()).unwrap(); + assert_eq!(artifacts.len(), 4); + let formats: Vec = artifacts.iter().map(|&(_, _, f)| f).collect(); + for format in [ + ArtifactFormat::Msi, + ArtifactFormat::Dmg, + ArtifactFormat::Deb, + ArtifactFormat::Appimage, + ] { + assert!(formats.contains(&format), "missing {format:?}"); + } + } + + #[test] + fn missing_asc_fails() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + std::fs::remove_file(dir.path().join("spatialengine_1.2.3_amd64.deb.asc")).unwrap(); + let err = collect_artifacts(dir.path(), &all_targets()) + .unwrap_err() + .to_string(); + assert!(err.contains("missing detached signature"), "{err}"); + } + + #[test] + fn unrecognised_file_fails() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + touch(dir.path(), "notes.txt", "stray"); + let err = collect_artifacts(dir.path(), &all_targets()) + .unwrap_err() + .to_string(); + assert!(err.contains("unrecognised artifact notes.txt"), "{err}"); + } + + #[test] + fn incomplete_set_for_configured_targets_fails() { + let dir = tempfile::tempdir().unwrap(); + touch(dir.path(), "SpatialEngine-1.2.3.msi", "msi-bytes"); + // Windows alone cannot satisfy a three-target configuration. + assert!(collect_artifacts(dir.path(), &all_targets()).is_err()); + // Linux with only a deb (even signed) is incomplete: deb AND appimage. + let dir = tempfile::tempdir().unwrap(); + touch(dir.path(), "a.deb", "deb"); + touch(dir.path(), "a.deb.asc", "sig"); + let err = collect_artifacts(dir.path(), &[TARGET_LINUX.to_string()]) + .unwrap_err() + .to_string(); + assert!(err.contains(".deb and one .AppImage"), "{err}"); + } + + #[test] + fn artifacts_for_unconfigured_targets_fail() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + let err = collect_artifacts(dir.path(), &[TARGET_WINDOWS.to_string()]) + .unwrap_err() + .to_string(); + assert!(err.contains("not configured to ship"), "{err}"); + } + + #[test] + fn empty_directory_fails() { + let dir = tempfile::tempdir().unwrap(); + assert!(collect_artifacts(dir.path(), &all_targets()).is_err()); + } + + #[test] + fn manifest_records_digests_keys_and_urls() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + let artifacts = collect_artifacts(dir.path(), &all_targets()).unwrap(); + let digests = digests_of(&artifacts); + let options = opts(dir.path(), all_targets()); + let (manifest, uploads) = + build_manifest(&options, "fsl-releases", &artifacts, &digests).unwrap(); + + assert_eq!(manifest.schema_version, SCHEMA_VERSION); + assert_eq!(manifest.artifacts.len(), 4); + assert!(!manifest.prerelease); + // 4 artifacts + 2 detached signatures. + assert_eq!(uploads.len(), 6); + + let msi = manifest + .artifacts + .iter() + .find(|a| a.format == ArtifactFormat::Msi) + .unwrap(); + let want_sha = + sha256_hex(&std::fs::read(dir.path().join("SpatialEngine-1.2.3.msi")).unwrap()); + assert_eq!(msi.sha256, want_sha); + assert_eq!(msi.size_bytes, "msi-bytes".len() as u64); + assert_eq!( + msi.url, + "https://api.s3.fsl.dev/fsl-releases/spatial_engine/1.2.3/x86_64-pc-windows-gnu/SpatialEngine-1.2.3.msi" + ); + assert!( + uploads.iter().any(|(_, key)| key + == "spatial_engine/1.2.3/x86_64-pc-windows-gnu/SpatialEngine-1.2.3.msi") + ); + assert!(matches!(msi.signature, ArtifactSignature::Authenticode)); + assert!(msi.glibc_floor.is_none()); + } + + #[test] + fn linux_entries_carry_fingerprint_floor_and_asc_uploads() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + let artifacts = collect_artifacts(dir.path(), &all_targets()).unwrap(); + let digests = digests_of(&artifacts); + let options = opts(dir.path(), all_targets()); + let (manifest, uploads) = + build_manifest(&options, "fsl-releases", &artifacts, &digests).unwrap(); + + let linux: Vec<&ManifestArtifact> = manifest + .artifacts + .iter() + .filter(|a| a.target == TARGET_LINUX) + .collect(); + assert_eq!(linux.len(), 2); + for artifact in linux { + assert_eq!(artifact.glibc_floor.as_deref(), Some("2.31")); + let ArtifactSignature::OpenpgpDetached { + url, + key_fingerprint, + } = &artifact.signature + else { + panic!("linux artifact without a detached signature record"); + }; + assert_eq!(key_fingerprint, FPR); + assert!(url.ends_with(".asc"), "{url}"); + assert_eq!(*url, format!("{}.asc", artifact.url)); + } + // Both .asc files are uploaded, but they are NOT manifest artifacts. + assert_eq!( + uploads + .iter() + .filter(|(_, key)| key.ends_with(".asc")) + .count(), + 2 + ); + } + + #[test] + fn missing_fingerprint_with_linux_artifacts_fails() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + let artifacts = collect_artifacts(dir.path(), &all_targets()).unwrap(); + let digests = digests_of(&artifacts); + for fingerprint in [None, Some(String::new())] { + let mut options = opts(dir.path(), all_targets()); + options.linux_fingerprint = fingerprint; + let err = build_manifest(&options, "fsl-releases", &artifacts, &digests) + .unwrap_err() + .to_string(); + assert!(err.contains("linux-fingerprint"), "{err}"); + } + } + + #[test] + fn malformed_source_revision_fails() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + let artifacts = collect_artifacts(dir.path(), &all_targets()).unwrap(); + let digests = digests_of(&artifacts); + for bad in [ + "short".to_string(), + SHA_SRC.to_uppercase(), + format!("{SHA_SRC}00"), + ] { + let mut options = opts(dir.path(), all_targets()); + options.source_revision = bad; + let err = build_manifest(&options, "fsl-releases", &artifacts, &digests) + .unwrap_err() + .to_string(); + assert!(err.contains("40 hex"), "{err}"); + } + } + + #[test] + fn preview_destination_marks_the_manifest_prerelease() { + let dir = tempfile::tempdir().unwrap(); + full_set(dir.path()); + let artifacts = collect_artifacts(dir.path(), &all_targets()).unwrap(); + let digests = digests_of(&artifacts); + let mut options = opts(dir.path(), all_targets()); + options.destination = Destination::Preview; + let (manifest, _) = + build_manifest(&options, "fsl-releases-preview", &artifacts, &digests).unwrap(); + assert!(manifest.prerelease); + } + + fn entry(version: &str) -> IndexEntry { + IndexEntry { + version: version.to_string(), + published_at: "2026-08-18T00:00:00Z".into(), + source_revision: SHA_SRC.into(), + manifest: format!("https://x/b/app/{version}/manifest.json"), + targets: vec![TARGET_WINDOWS.to_string()], + } + } + + fn index(versions: &[&str]) -> Index { + Index { + schema_version: SCHEMA_VERSION, + app: "app".into(), + updated_at: "t0".into(), + versions: versions.iter().map(|v| entry(v)).collect(), + } + } + + #[test] + fn index_sorts_descending_with_semver_precedence() { + let idx = index_with_entry(index(&["1.0.0", "1.10.0"]), entry("1.9.9"), "t1"); + let versions: Vec<&str> = idx.versions.iter().map(|e| e.version.as_str()).collect(); + assert_eq!(versions, ["1.10.0", "1.9.9", "1.0.0"]); + assert_eq!(idx.updated_at, "t1"); + + // Prerelease suffixes (preview) order below their plain version. + let idx = index_with_entry(index(&["1.0.0"]), entry("1.0.0-rc.1"), "t1"); + let versions: Vec<&str> = idx.versions.iter().map(|e| e.version.as_str()).collect(); + assert_eq!(versions, ["1.0.0", "1.0.0-rc.1"]); + } + + #[test] + fn index_dedupes_by_version_keeping_the_new_entry() { + let mut new_entry = entry("1.0.0"); + new_entry.manifest = "https://x/b/app/1.0.0/manifest.json?rewritten".into(); + let idx = index_with_entry(index(&["1.0.0", "1.1.0"]), new_entry.clone(), "t1"); + assert_eq!(idx.versions.len(), 2); + let kept = idx.versions.iter().find(|e| e.version == "1.0.0").unwrap(); + assert_eq!(kept.manifest, new_entry.manifest); + } + + #[test] + fn unparseable_versions_sort_last_not_error() { + let idx = index_with_entry(index(&["weird", "1.0.0"]), entry("1.1.0"), "t1"); + let versions: Vec<&str> = idx.versions.iter().map(|e| e.version.as_str()).collect(); + assert_eq!(versions, ["1.1.0", "1.0.0", "weird"]); + } +} diff --git a/src/commands/release/sign_linux.rs b/src/commands/release/sign_linux.rs new file mode 100644 index 000000000..41cc44903 --- /dev/null +++ b/src/commands/release/sign_linux.rs @@ -0,0 +1,213 @@ +//! Detach-sign the Linux release artifacts with the organisation's OpenPGP +//! key. Linux has no operating-system signature (Windows gets Authenticode, +//! macOS gets notarization), so the .deb and AppImage each get a detached +//! ASCII-armoured signature; the manifest's sha256 remains the primary trust +//! anchor and the public key is published at keys/fsl-release-linux.asc in +//! the production bucket. +//! +//! Runs on an ephemeral GitHub-hosted runner (signing-only per the runner +//! policy): the key lives in a job-scoped GNUPGHOME under a temp dir wiped +//! on drop, and every signature is self-verified before the command exits. + +use std::fmt::{Display, Formatter}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +use anyhow::{Context, bail}; +use clap::Parser; +use serde::Serialize; +use tokio::io::AsyncWriteExt; + +use crate::PrettyPrintable; + +#[derive(Debug, Parser, Clone)] +#[command(about = "Detach-sign Linux artifacts with the org OpenPGP key")] +pub struct Options { + /// Directory whose *.deb and *.AppImage files each get a .asc. + #[arg(long)] + pub dir: PathBuf, + /// ASCII-armoured private key. + #[arg(long, env = "LINUX_SIGNING_KEY", hide_env_values = true)] + pub key: String, + #[arg(long, env = "LINUX_SIGNING_PASSPHRASE", hide_env_values = true)] + pub passphrase: String, +} + +#[derive(Debug, Serialize, Clone)] +pub struct SignLinuxResult { + /// 40-hex fingerprint recorded in the manifest against the published key. + pub fingerprint: String, + pub signed: Vec, +} + +impl Display for SignLinuxResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + for s in &self.signed { + writeln!(f, "signed {s}")?; + } + write!(f, "fingerprint={}", self.fingerprint) + } +} + +impl PrettyPrintable for SignLinuxResult { + fn pretty_print(&self) -> String { + self.to_string() + } +} + +/// Extract the key id (field 5 of the first `sec` line) and fingerprint +/// (field 10 of the first `fpr` line) from +/// `gpg --list-secret-keys --with-colons` output. +fn parse_secret_key_listing(listing: &str) -> anyhow::Result<(String, String)> { + let field = |prefix: &str, index: usize| { + listing + .lines() + .find(|line| line.starts_with(prefix)) + .and_then(|line| line.split(':').nth(index)) + .filter(|value| !value.is_empty()) + .map(str::to_string) + }; + let key_id = field("sec", 4).context("no secret key imported")?; + let fingerprint = field("fpr", 9).context("imported secret key has no fingerprint line")?; + Ok((key_id, fingerprint)) +} + +async fn gpg(gnupg_home: &Path, args: &[&str]) -> anyhow::Result { + let output = tokio::process::Command::new("gpg") + .args(args) + .env("GNUPGHOME", gnupg_home) + .output() + .await + .context("failed to run gpg")?; + if !output.status.success() { + bail!( + "gpg {} failed with {}: {}", + args.join(" "), + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(output) +} + +pub async fn run(options: &Options) -> anyhow::Result { + // Job-scoped GNUPGHOME: created 0700, wiped when the TempDir drops. + let gnupg_home = tempfile::TempDir::new().context("cannot create a temporary GNUPGHOME")?; + std::fs::set_permissions(gnupg_home.path(), std::fs::Permissions::from_mode(0o700))?; + let home = gnupg_home.path(); + + // Import the key via stdin so it never touches the filesystem outside + // the GNUPGHOME. + let mut import = tokio::process::Command::new("gpg") + .args(["--batch", "--quiet", "--import"]) + .env("GNUPGHOME", home) + .stdin(Stdio::piped()) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .context("failed to spawn gpg for the key import")?; + let mut stdin = import.stdin.take().context("no stdin on the gpg import")?; + stdin.write_all(options.key.as_bytes()).await?; + stdin.write_all(b"\n").await?; + drop(stdin); + let import = import.wait_with_output().await?; + if !import.status.success() { + bail!( + "gpg --import failed with {}: {}", + import.status, + String::from_utf8_lossy(&import.stderr) + ); + } + + let listing = gpg(home, &["--list-secret-keys", "--with-colons"]).await?; + let (key_id, fingerprint) = + parse_secret_key_listing(&String::from_utf8_lossy(&listing.stdout))?; + + // Every *.deb and *.AppImage in the directory, deterministically ordered. + let mut artifacts: Vec = std::fs::read_dir(&options.dir) + .with_context(|| format!("cannot list {}", options.dir.display()))? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.is_file() + && path.file_name().is_some_and(|name| { + let name = name.to_string_lossy(); + name.ends_with(".deb") || name.ends_with(".AppImage") + }) + }) + .collect(); + artifacts.sort(); + if artifacts.is_empty() { + bail!("no .deb or .AppImage found in {}", options.dir.display()); + } + + let mut signed = Vec::new(); + for artifact in &artifacts { + let artifact_str = artifact.to_string_lossy().into_owned(); + let signature = format!("{artifact_str}.asc"); + gpg( + home, + &[ + "--batch", + "--yes", + "--pinentry-mode", + "loopback", + "--passphrase", + &options.passphrase, + "--local-user", + &key_id, + "--armor", + "--detach-sign", + "--output", + &signature, + &artifact_str, + ], + ) + .await + .with_context(|| format!("signing {} failed", artifact.display()))?; + // Self-verify before anything downstream trusts the .asc. + gpg(home, &["--verify", &signature, &artifact_str]) + .await + .with_context(|| format!("self-verification failed for {}", artifact.display()))?; + signed.push( + artifact + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or(artifact_str), + ); + } + + Ok(SignLinuxResult { + fingerprint, + signed, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const LISTING: &str = "\ +tru::1:1650000000:0:3:1:5 +sec:u:4096:1:0123456789ABCDEF:1650000000:::u:::scESC:::+:::23::0: +fpr:::::::::ABCDEF0123456789ABCDEF0123456789ABCDEF01: +grp:::::::::0000000000000000000000000000000000000000: +uid:u::::1650000000::AAAA::FSL Release ::::::::::0: +"; + + #[test] + fn key_id_and_fingerprint_come_from_the_colon_listing() { + let (key_id, fingerprint) = parse_secret_key_listing(LISTING).unwrap(); + assert_eq!(key_id, "0123456789ABCDEF"); + assert_eq!(fingerprint, "ABCDEF0123456789ABCDEF0123456789ABCDEF01"); + } + + #[test] + fn listing_without_a_secret_key_is_an_error() { + let err = parse_secret_key_listing("tru::1:1650000000:0:3:1:5\n") + .unwrap_err() + .to_string(); + assert!(err.contains("no secret key imported"), "{err}"); + } +} diff --git a/src/commands/release/store.rs b/src/commands/release/store.rs new file mode 100644 index 000000000..73e20c2c0 --- /dev/null +++ b/src/commands/release/store.rs @@ -0,0 +1,246 @@ +//! Storage rules for release publication, in one place. +//! +//! Immutability and lost-update protection come from S3 conditional writes: +//! [`ReleaseStore::put_immutable`] refuses to overwrite via If-None-Match:*, +//! and [`ReleaseStore::cas_update`] is an ETag If-Match compare-and-swap +//! whose transform re-runs against the freshly-read document on every retry, +//! so validation embedded in the transform (for example the backward-move +//! gate) is never evaluated against a stale document. Object lock and +//! write-only credentials guard against deletion, but only a conditional +//! write prevents an overwrite (a re-PUT on a versioned bucket succeeds and +//! becomes the new current version), which is why `release probe-store` must +//! pass against the deployed store before publication may rely on this +//! module. + +use anyhow::{Context, bail}; +use opendal::{ErrorKind, Operator, services::S3}; +use semver::Version; +use sha2::{Digest, Sha256}; + +pub const CAS_RETRIES: usize = 5; + +/// A refused overwrite of an existing object: the caller must not retry the +/// same key. Detect with `err.downcast_ref::()`. +#[derive(Debug, thiserror::Error)] +#[error("refusing to overwrite existing object {0}")] +pub struct AlreadyExists(pub String); + +pub struct ReleaseStore { + op: Operator, + bucket: String, +} + +/// Build an S3-backed operator for `bucket` from the environment: +/// `RELEASE_STORE_ENDPOINT` (optional), `AWS_ACCESS_KEY_ID`, +/// `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` (default us-east-1). +pub fn operator_from_env(bucket: &str) -> anyhow::Result { + let mut builder = S3::default() + .bucket(bucket) + .region(&std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".into())); + if let Ok(endpoint) = std::env::var("RELEASE_STORE_ENDPOINT") { + builder = builder.endpoint(&endpoint); + } + if let Ok(key) = std::env::var("AWS_ACCESS_KEY_ID") { + builder = builder.access_key_id(&key); + } + if let Ok(secret) = std::env::var("AWS_SECRET_ACCESS_KEY") { + builder = builder.secret_access_key(&secret); + } + Ok(Operator::new(builder)?.finish()) +} + +pub fn sha256_hex(data: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) +} + +/// Strict semver "greater than", with real prerelease precedence: 1.0.0 > +/// 1.0.0-rc.1, and 1.10.0 > 1.9.9. (`sort -V` and string comparison both get +/// these wrong, which is why this lives here and nowhere else.) +pub fn semver_gt(a: &str, b: &str) -> anyhow::Result { + let a = Version::parse(a).with_context(|| format!("unparseable version: {a}"))?; + let b = Version::parse(b).with_context(|| format!("unparseable version: {b}"))?; + Ok(a > b) +} + +impl ReleaseStore { + pub fn new(op: Operator, bucket: impl Into) -> Self { + Self { + op, + bucket: bucket.into(), + } + } + + pub fn from_env(bucket: &str) -> anyhow::Result { + Ok(Self::new(operator_from_env(bucket)?, bucket)) + } + + pub async fn read(&self, key: &str) -> anyhow::Result> { + Ok(self + .op + .read(key) + .await + .with_context(|| format!("cannot read s3://{}/{key}", self.bucket))? + .to_vec()) + } + + pub async fn exists(&self, key: &str) -> anyhow::Result { + Ok(self.op.exists(key).await?) + } + + /// Write a key that must not already exist. An existing key returns + /// [`AlreadyExists`]; a shipped artifact is never overwritten. + pub async fn put_immutable(&self, key: &str, data: Vec) -> anyhow::Result<()> { + // opendal expresses S3 If-None-Match:* as if_not_exists. + match self.op.write_with(key, data).if_not_exists(true).await { + Ok(_) => Ok(()), + Err(e) + if e.kind() == ErrorKind::ConditionNotMatch + || e.kind() == ErrorKind::AlreadyExists => + { + Err(AlreadyExists(format!("s3://{}/{key}", self.bucket)).into()) + } + Err(e) => Err(e).with_context(|| format!("put-immutable s3://{}/{key}", self.bucket)), + } + } + + /// Read-modify-write under If-Match, retrying with a fresh read when the + /// precondition fails. `transform` sees the CURRENT document (or + /// `initial` when the key does not exist yet) on every attempt; an error + /// from the transform aborts the whole update and propagates. + pub async fn cas_update( + &self, + key: &str, + initial: Option, + mut transform: F, + ) -> anyhow::Result + where + T: serde::Serialize + serde::de::DeserializeOwned, + F: FnMut(T) -> anyhow::Result, + { + for attempt in 1..=CAS_RETRIES { + let (current, etag): (T, Option) = match self.op.stat(key).await { + Ok(meta) => { + let etag = meta.etag().map(str::to_string); + let bytes = self.read(key).await?; + ( + serde_json::from_slice(&bytes).with_context(|| { + format!("s3://{}/{key} is not valid JSON", self.bucket) + })?, + etag, + ) + } + Err(e) if e.kind() == ErrorKind::NotFound => match &initial { + Some(init) => (serde_json::from_value(serde_json::to_value(init)?)?, None), + None => bail!( + "cas-update: s3://{}/{key} does not exist and no initial document was given", + self.bucket + ), + }, + Err(e) => return Err(e).context("stat failed"), + }; + + let next = transform(current)?; + let body = serde_json::to_vec_pretty(&next)?; + let write = match etag { + Some(ref etag) => self.op.write_with(key, body).if_match(etag).await, + None => self.op.write_with(key, body).if_not_exists(true).await, + }; + match write { + Ok(_) => return Ok(next), + Err(e) + if e.kind() == ErrorKind::ConditionNotMatch + || e.kind() == ErrorKind::AlreadyExists => + { + tracing::warn!( + "cas-update: lost the race on s3://{}/{key} (attempt {attempt}/{CAS_RETRIES}), re-reading", + self.bucket + ); + continue; + } + Err(e) => { + return Err(e) + .with_context(|| format!("cas-update s3://{}/{key}", self.bucket)); + } + } + } + bail!( + "cas-update: gave up on s3://{}/{key} after {CAS_RETRIES} attempts", + self.bucket + ) + } + + /// Fail unless `version` is strictly greater than every version that has + /// a COMMITTED MANIFEST under `/`. The listed manifests are the + /// authority, never index.json: an index update can fail after a manifest + /// landed, and a check that trusted the index would then admit a lower + /// version forever. A version prefix with artifacts but no manifest is + /// invisible, because the manifest is the commit point. + pub async fn assert_monotonic(&self, app: &str, version: &str) -> anyhow::Result> { + let mut published = Vec::new(); + let entries = self + .op + .list_with(&format!("{app}/")) + .recursive(true) + .await + .with_context(|| format!("cannot list s3://{}/{app}/", self.bucket))?; + for entry in entries { + let path = entry.path(); + if let Some(v) = path + .strip_prefix(&format!("{app}/")) + .and_then(|rest| rest.strip_suffix("/manifest.json")) + && !v.contains('/') + { + published.push(v.to_string()); + } + } + for existing in &published { + if !semver_gt(version, existing)? { + bail!( + "version {version} is not strictly greater than published {existing} for {app}; \ + production versions increase strictly and a failed publication's number is spent" + ); + } + } + Ok(published) + } + + /// Re-read one object and compare its digest. Used between the artifact + /// writes and the manifest write, before any manifest exists. + pub async fn verify_key(&self, key: &str, want_sha256: &str) -> anyhow::Result<()> { + let data = self.read(key).await?; + let got = sha256_hex(&data); + if got != want_sha256 { + bail!("digest mismatch for {key}: wanted {want_sha256}, object has {got}"); + } + Ok(()) + } + + /// Turn a manifest artifact URL back into this bucket's object key. + pub fn key_from_url<'a>(&self, url: &'a str) -> anyhow::Result<&'a str> { + let marker = format!("/{}/", self.bucket); + match url.find(&marker) { + Some(idx) => Ok(&url[idx + marker.len()..]), + None => bail!( + "artifact url {url} does not reference bucket {}", + self.bucket + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn semver_ordering_is_real_precedence() { + assert!(semver_gt("1.10.0", "1.9.9").unwrap()); + assert!(semver_gt("1.0.0", "1.0.0-rc.1").unwrap()); + assert!(!semver_gt("1.0.0-rc.1", "1.0.0").unwrap()); + assert!(!semver_gt("1.0.0", "1.0.0").unwrap()); + assert!(semver_gt("1.0.0-rc.2", "1.0.0-rc.1").unwrap()); + assert!(semver_gt("x.y.z", "1.0.0").is_err()); + } +} diff --git a/src/commands/release/types.rs b/src/commands/release/types.rs new file mode 100644 index 000000000..257e474a8 --- /dev/null +++ b/src/commands/release/types.rs @@ -0,0 +1,219 @@ +//! The application-release contract types: the manifest, the version index, +//! and the channel pointers. These serde definitions are the single source of +//! truth for the object shapes documented in fsl_libs' +//! `software_guide/content/docs/releases/`; clients gate on `schema_version` +//! and refuse unknown majors, so fields may be added freely within a major +//! but never repurposed or removed. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +pub const SCHEMA_VERSION: u64 = 1; + +pub const TARGET_WINDOWS: &str = "x86_64-pc-windows-gnu"; +pub const TARGET_MACOS: &str = "aarch64-apple-darwin"; +pub const TARGET_LINUX: &str = "x86_64-unknown-linux-gnu"; + +/// Immutable record of one published application version. Its existence under +/// `//manifest.json` is the atomic commit point of a +/// publication: it is written last, after every artifact object has been read +/// back and digest-verified, and it is never rewritten. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Manifest { + pub schema_version: u64, + pub app: String, + pub version: String, + pub prerelease: bool, + /// 40-hex commit the artifacts were built from, resolved from the release + /// tag itself (never from `target_commitish`, which is a branch name). + pub source_revision: String, + pub release_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub release_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workflow_run: Option, + pub published_at: String, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ManifestArtifact { + pub target: String, + pub format: ArtifactFormat, + pub filename: String, + pub url: String, + pub size_bytes: u64, + pub sha256: String, + pub signature: ArtifactSignature, + /// Linux artifacts record the glibc floor the build container fixed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub glibc_floor: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum ArtifactFormat { + Msi, + Dmg, + Deb, + Appimage, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case")] +pub enum ArtifactSignature { + /// Windows: the OS verifies on install; publication re-verifies with + /// osslsigncode. + Authenticode, + /// macOS: notarization is stapler-validated on the build host; a Linux + /// publisher cannot re-check it. + AppleNotarized, + /// Linux has no OS signature: a detached ASCII-armoured signature whose + /// key is published at `keys/fsl-release-linux.asc` in the production + /// bucket. The manifest sha256 stays the primary trust anchor. + OpenpgpDetached { + url: String, + key_fingerprint: String, + }, +} + +/// Derived list of published versions for one application. The set of +/// committed manifests is the authority; this document is a convenience for +/// humans and the healthcheck, updated by compare-and-swap and repairable at +/// any time. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Index { + pub schema_version: u64, + pub app: String, + pub updated_at: String, + pub versions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndexEntry { + pub version: String, + pub published_at: String, + pub source_revision: String, + pub manifest: String, + pub targets: Vec, +} + +/// One version pointer per channel per target for one application. Only the +/// promotion flow writes this document; the publication credential cannot. +/// Clients read exactly one field: `.channels..`, +/// and an absent key means "no pointer" with no fallback allowed. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Channels { + pub schema_version: u64, + pub app: String, + pub updated_at: String, + pub manifest_base: String, + pub channels: ChannelPointers, + #[serde(default)] + pub provenance: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ChannelPointers { + #[serde(default)] + pub latest: BTreeMap, + #[serde(default)] + pub stable: BTreeMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum Channel { + Latest, + Stable, +} + +impl std::fmt::Display for Channel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Channel::Latest => write!(f, "latest"), + Channel::Stable => write!(f, "stable"), + } + } +} + +impl ChannelPointers { + pub fn get_mut(&mut self, channel: Channel) -> &mut BTreeMap { + match channel { + Channel::Latest => &mut self.latest, + Channel::Stable => &mut self.stable, + } + } + + pub fn get(&self, channel: Channel) -> &BTreeMap { + match channel { + Channel::Latest => &self.latest, + Channel::Stable => &self.stable, + } + } +} + +/// Audit record of a pointer move, capped at the most recent 100 entries. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProvenanceEntry { + pub channel: Channel, + pub targets: Vec, + /// Previous pointer per target (absent target key = no previous pointer). + pub from: BTreeMap>, + pub to: String, + pub backward: bool, + pub reason: Option, + pub moved_at: String, + pub moved_by: String, + pub run: String, +} + +pub const PROVENANCE_CAP: usize = 100; + +/// Per-application release configuration, read from +/// `[package.metadata.fslabs.release]` in the app crate's Cargo.toml. The +/// presence of this table is what makes a package an application the release +/// pipeline knows about. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct AppReleaseConfig { + /// The binary/product name (e.g. "SpatialEngine"). + pub verbose_name: String, + /// Target triples this application ships for. + pub targets: Vec, + /// Path to the macOS EULA, relative to the package directory. + #[serde(default)] + pub license_macos: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn signature_serialization_shape_is_the_contract() { + let sig = ArtifactSignature::OpenpgpDetached { + url: "https://x/y.asc".into(), + key_fingerprint: "ABCD".into(), + }; + let v = serde_json::to_value(&sig).unwrap(); + assert_eq!(v["kind"], "openpgp-detached"); + assert_eq!(v["url"], "https://x/y.asc"); + let plain = serde_json::to_value(ArtifactSignature::Authenticode).unwrap(); + assert_eq!(plain["kind"], "authenticode"); + } + + #[test] + fn channels_roundtrip_preserves_unknown_free_shape() { + let json = serde_json::json!({ + "schema_version": 1, + "app": "spatial_engine", + "updated_at": "2026-08-18T00:00:00Z", + "manifest_base": "https://api.s3.fsl.dev/fsl-releases", + "channels": {"latest": {"aarch64-apple-darwin": "1.0.0"}, "stable": {}}, + }); + let ch: Channels = serde_json::from_value(json).unwrap(); + assert_eq!(ch.channels.latest.get(TARGET_MACOS).unwrap(), "1.0.0"); + assert!(ch.provenance.is_empty()); + } +} diff --git a/src/commands/release/verify_production.rs b/src/commands/release/verify_production.rs new file mode 100644 index 000000000..8c2c68f26 --- /dev/null +++ b/src/commands/release/verify_production.rs @@ -0,0 +1,213 @@ +//! Production-eligibility gate for an application release. +//! +//! The revision is resolved from the TAG ITSELF: the release payload's +//! `target_commitish` is a branch name for releases cut from an existing tag +//! (this tool's own publish path passes the literal string "main"), so it is +//! never trusted as a revision. The resolved commit must equal the revision +//! the workflow checked out, must be an ancestor of main (cherry-picked +//! hotfixes ship as pre-releases), must carry green check runs (production +//! ships verified main), and the human-typed version must equal the +//! workspace version in the tagged tree so a typo cannot burn a wrong number +//! into the immutable, monotonic production index. + +use std::fmt::{Display, Formatter}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, bail}; +use clap::Parser; +use serde::Serialize; + +use crate::PrettyPrintable; + +#[derive(Debug, Parser, Clone)] +#[command( + about = "Verify a release commit is eligible for production publication", + disable_version_flag = true +)] +pub struct Options { + /// The release tag to resolve. + #[arg(long, env = "RELEASE_TAG")] + pub tag: String, + /// The revision the workflow checked out (github.sha). + #[arg(long)] + pub sha: String, + /// The released version; verified against the workspace version file. + #[arg(long)] + pub version: Option, + /// Manifest whose `version` field binds the release version. + #[arg(long, default_value = "fdk_apps/Cargo.toml")] + pub workspace_version_file: PathBuf, + /// Branch the commit must be an ancestor of. + #[arg(long, default_value = "origin/main")] + pub main_ref: String, + /// Skip the check-run gate (only for environments with no API access). + #[arg(long, default_value_t = false)] + pub skip_check_runs: bool, +} + +#[derive(Debug, Serialize, Clone)] +pub struct VerifyProductionResult { + pub source_revision: String, + pub checks_verified: bool, +} + +impl Display for VerifyProductionResult { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "source_revision={} (checks {})", + self.source_revision, + if self.checks_verified { + "verified" + } else { + "SKIPPED" + } + ) + } +} + +impl PrettyPrintable for VerifyProductionResult { + fn pretty_print(&self) -> String { + self.to_string() + } +} + +fn git(repo_root: &Path, args: &[&str]) -> anyhow::Result { + Command::new("git") + .arg("-C") + .arg(repo_root) + .args(args) + .output() + .context("git invocation failed") +} + +pub async fn run(options: &Options, repo_root: PathBuf) -> anyhow::Result { + // Resolve the tag to a commit. Requires a full-history checkout with + // tags (fetch-depth: 0). + let out = git( + &repo_root, + &[ + "rev-list", + "-n1", + &format!("refs/tags/{}^{{commit}}", options.tag), + ], + )?; + if !out.status.success() { + bail!( + "tag {} does not resolve to a commit in this checkout (fetch tags with fetch-depth: 0)", + options.tag + ); + } + let tag_sha = String::from_utf8(out.stdout)?.trim().to_string(); + if tag_sha != options.sha { + bail!( + "tag {} points at {tag_sha} but this run checked out {}; refusing to publish a revision other than the tagged one", + options.tag, + options.sha + ); + } + + let ancestor = git( + &repo_root, + &["merge-base", "--is-ancestor", &tag_sha, &options.main_ref], + )?; + if !ancestor.status.success() { + bail!( + "commit {tag_sha} is not an ancestor of {}; production releases must ship main. Cherry-picked builds ship as pre-releases.", + options.main_ref + ); + } + + if let Some(version) = &options.version { + let manifest_path = repo_root.join(&options.workspace_version_file); + let contents = std::fs::read_to_string(&manifest_path) + .with_context(|| format!("cannot read {}", manifest_path.display()))?; + let value: toml::Value = toml::from_str(&contents)?; + let workspace_version = value + .get("workspace") + .and_then(|w| w.get("package")) + .and_then(|p| p.get("version")) + .or_else(|| value.get("package").and_then(|p| p.get("version"))) + .and_then(|v| v.as_str()) + .with_context(|| format!("no workspace version in {}", manifest_path.display()))?; + if version != workspace_version { + bail!( + "release version {version} does not match the {} workspace version {workspace_version}; bump the workspace version first", + options.workspace_version_file.display() + ); + } + } + + // Built from verified main: the tagged commit's check runs must be green. + let mut checks_verified = false; + if !options.skip_check_runs { + let (Ok(token), Ok(repository)) = ( + std::env::var("GH_TOKEN").or_else(|_| std::env::var("GITHUB_TOKEN")), + std::env::var("GITHUB_REPOSITORY"), + ) else { + bail!( + "GITHUB_TOKEN/GITHUB_REPOSITORY unavailable for the check-run gate; pass --skip-check-runs only where API access is impossible" + ); + }; + let (owner, repo) = repository + .split_once('/') + .context("GITHUB_REPOSITORY is not owner/repo")?; + let octocrab = octocrab::OctocrabBuilder::new() + .personal_token(token) + .build()?; + let mut bad = Vec::new(); + let mut total = 0usize; + let mut page = 1u32; + loop { + let response: serde_json::Value = octocrab + .get( + format!("/repos/{owner}/{repo}/commits/{tag_sha}/check-runs?per_page=100&page={page}"), + None::<&()>, + ) + .await + .context("check-runs query failed")?; + let runs = response["check_runs"] + .as_array() + .cloned() + .unwrap_or_default(); + if runs.is_empty() { + break; + } + for run in &runs { + total += 1; + let conclusion = run["conclusion"].as_str().unwrap_or(""); + if matches!( + conclusion, + "failure" | "timed_out" | "cancelled" | "action_required" + ) { + bad.push(format!( + "{} ({conclusion})", + run["name"].as_str().unwrap_or("?") + )); + } + } + if runs.len() < 100 { + break; + } + page += 1; + } + if !bad.is_empty() { + bail!( + "commit {tag_sha} has failed check run(s): {}; production releases ship verified main", + bad.join(", ") + ); + } + if total == 0 { + bail!( + "commit {tag_sha} has no check runs at all; production releases ship verified main" + ); + } + checks_verified = true; + } + + Ok(VerifyProductionResult { + source_revision: tag_sha, + checks_verified, + }) +} diff --git a/src/main.rs b/src/main.rs index b51c4cfd9..401550284 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ use crate::commands::generate_wix::{Options as GenerateWixOptions, generate_wix} use crate::commands::generate_workflow::{Options as GenerateWorkflowOptions, generate_workflow}; use crate::commands::github_app_token::{Options as GithubAppTokenOptions, github_app_token}; use crate::commands::publish::{Options as PublishOptions, publish}; +use crate::commands::release::{Options as ReleaseOptions, release}; use crate::commands::summaries::{Options as SummariesOptions, summaries}; use crate::commands::tests::{Options as TestsOptions, tests}; use crate::crate_graph::find_git_root; @@ -125,6 +126,8 @@ enum Commands { DockerBuildPush(Box), /// Create or update a draft GitHub release and upload artifacts DraftRelease(Box), + /// Application release pipeline: classify, publish, promote, resolve, verify + Release(Box), /// Post build findings (logs, JUnit XML) as GitHub check-run annotations Annotate(Box), @@ -503,6 +506,9 @@ async fn run() -> anyhow::Result { Commands::DraftRelease(options) => draft_release(options, working_directory) .await .map(|r| display_results(cli.json, cli.pretty_print, r)), + Commands::Release(options) => release(options, working_directory, repo_root) + .await + .map(|r| display_results(cli.json, cli.pretty_print, r)), // Repo root, not the working directory: annotation paths are anchored // at the repository root by GitHub, and the job may be invoked from a // subdirectory.