From 26a3969e41067039df8729513b9278181f0d9d72 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 11:00:17 +0200 Subject: [PATCH 01/26] feat(stream): implement declared event ingress --- INVARIANTS.md | 3 +- crates/agent-spec/src/declared.rs | 21 +- crates/agent-spec/src/discovery.rs | 5 +- crates/agent-spec/src/kdl_format.rs | 65 +- crates/agent-spec/src/lib.rs | 4 +- crates/agent-spec/src/spec.rs | 163 ++++- crates/agent-spec/tests/discovery.rs | 96 ++- crates/st2-wire/src/message.rs | 16 +- ...eam-subsystem-specified-not-implemented.md | 60 -- docs/vrs/02-agent-spec/spec.md | 31 + src/agent_author.rs | 528 +++++++++++++- src/catalog.rs | 30 +- src/catalog_transaction.rs | 26 +- src/codex_app_server.rs | 69 +- src/ding/mod.rs | 216 +++++- src/driver.rs | 6 +- src/eval_run.rs | 668 ++++++++++++++---- src/eval_spec.rs | 312 +++++--- src/event.rs | 367 ++++++++++ src/flapping.rs | 70 +- src/host_lock.rs | 18 +- src/isolate.rs | 24 +- src/lib.rs | 5 +- src/main.rs | 202 +++++- src/materialize.rs | 12 +- src/message.rs | 275 +++++-- src/park.rs | 74 +- src/pretrust.rs | 87 ++- src/reconcile.rs | 53 +- src/run.rs | 72 +- src/service.rs | 19 +- src/task_inventory.rs | 5 +- src/validate.rs | 3 + tests/agent_desired_state.rs | 81 ++- tests/catalog_apply.rs | 81 +-- tests/catalog_config.rs | 16 +- tests/codex_app_server.rs | 14 +- tests/doctor.rs | 27 +- tests/driver_expansion.rs | 16 +- tests/eval_run_e2e.rs | 554 ++++++++++++--- tests/eval_up.rs | 266 +++++-- tests/event_e2e.rs | 333 +++++++++ tests/hooks.rs | 24 +- tests/materialize.rs | 13 +- tests/message_cli.rs | 188 +++-- tests/parked_recovery.rs | 22 +- tests/reconcile.rs | 92 ++- tests/request_cli.rs | 4 +- tests/run.rs | 453 +++++++++++- tests/service.rs | 21 +- tests/stream_authoring_cli.rs | 166 +++++ tests/task_inventory_cli.rs | 99 ++- tests/transport_isolation.rs | 68 +- tests/transport_isolation_macos.rs | 51 +- tests/validate.rs | 7 +- 55 files changed, 5154 insertions(+), 1047 deletions(-) delete mode 100644 docs/vrs/.delta/DELTA-003-stream-subsystem-specified-not-implemented.md create mode 100644 src/event.rs create mode 100644 tests/event_e2e.rs create mode 100644 tests/stream_authoring_cli.rs diff --git a/INVARIANTS.md b/INVARIANTS.md index 97801e7b..bd053af9 100644 --- a/INVARIANTS.md +++ b/INVARIANTS.md @@ -11,9 +11,10 @@ materialization, messaging, DING, or presence must preserve them. | **Transport-decoupled lifecycle** | Each task is isolated from a supervisor/transport process-group or cgroup cascade. | `tests/transport_isolation.rs`; `tests/transport_isolation_macos.rs` | | **Clean exec teardown** | Killing an exec task reaps its whole process group. | `tests/exec_backend.rs::exec_kill_reaps_the_whole_process_group_not_just_the_leader` | | **Bounded restart diagnostics** | Relaunching an exec task preserves the just-finished log as one prior generation while bounding retained diagnostics to current plus prior. Final retirement removes the PID and both logs. | `tests/exec_backend.rs::exec_restart_reap_keeps_bounded_diagnostics_and_final_remove_cleans_them`; `tests/run.rs::up_once_finally_removes_dead_retired_tasks_without_restarting_them` | -| **Derived companion lifecycle** | A generated DING starts only with an eligible canonical agent, is suppressed or stopped while that target is held, suspended, unavailable, retired, or terminally parked, and remains coupled without changing explicit sibling task behavior. Healthy compact startup still launches both tasks in one pass, and targeted reconciliation holds a missing generated DING rather than broadening to its agent. | `tests/run.rs::fresh_compact_agent_launches_with_its_derived_ding`; `tests/run.rs::absent_adopt_only_compact_agent_holds_its_derived_ding`; `tests/run.rs::held_adopt_only_compact_agent_stops_its_live_derived_ding`; `tests/run.rs::failed_compact_agent_restart_stops_its_live_derived_ding`; `tests/run.rs::failed_compact_agent_reap_stops_its_live_derived_ding`; `tests/run.rs::parked_compact_agent_stops_its_live_derived_ding`; `tests/run.rs::parked_compact_agent_does_not_relaunch_its_exited_derived_ding`; `tests/run.rs::retired_compact_agent_stops_agent_and_derived_ding`; `tests/run.rs::suspend_and_resume_cover_derived_ding_sibling_continuity_and_inbox_retention`; `tests/run.rs::selected_missing_derived_ding_is_held_without_broadening_to_its_agent`; `tests/run.rs::up_once_collects_spawn_errors_without_aborting` | +| **Derived companion lifecycle** | A generated DING or launched stream starts only with an eligible canonical agent, is suppressed or stopped while that target is held, suspended, unavailable, retired, or terminally parked, and remains coupled without changing explicit sibling task behavior. A stream lowers its authored adapter launch directly, parks and surfaces independently, and neither makes an otherwise empty agent runnable nor claims a delivery transport. Stream authoring is serialized, authority-scoped, source-preserving, and fail-closed for Nix ownership and invalid declarations. Healthy compact startup launches companions in one pass, and targeted reconciliation holds a missing generated companion rather than broadening to its agent. | `crates/agent-spec/tests/discovery.rs::streams_are_typed_and_only_launched_streams_lower_to_derived_exec_tasks`; `crates/agent-spec/tests/discovery.rs::stream_names_launches_and_task_collisions_fail_closed`; `src/agent_author.rs::stream_add_supports_external_command_and_argv_and_remove_is_idempotent`; `src/agent_author.rs::stream_authoring_enforces_authority_nix_ownership_and_canonical_validation`; `tests/run.rs::fresh_compact_agent_launches_with_its_derived_ding`; `tests/run.rs::held_adopt_only_compact_agent_stops_its_live_derived_ding`; `tests/run.rs::parked_compact_agent_stops_its_live_derived_ding`; `tests/run.rs::retired_compact_agent_stops_agent_and_derived_ding`; `tests/run.rs::selected_missing_derived_ding_is_held_without_broadening_to_its_agent`; `tests/run.rs::fresh_compact_agent_launches_with_its_derived_stream`; `tests/run.rs::retired_compact_agent_stops_agent_and_derived_stream`; `tests/run.rs::suspended_compact_agent_stops_its_derived_stream_without_touching_a_sibling`; `tests/run.rs::held_adopt_only_compact_agent_stops_its_live_derived_stream`; `tests/run.rs::a_crash_looping_stream_parks_and_surfaces_without_disturbing_its_agent`; `tests/run.rs::parked_compact_agent_stops_its_live_derived_stream`; `tests/run.rs::selected_missing_derived_stream_is_held_without_broadening_to_its_agent`; `tests/run.rs::a_stream_alone_does_not_make_an_agent_runnable`; `tests/run.rs::a_stream_does_not_claim_a_delivery_transport` | | **Exactly-once-safe native bus** | Messages use stable `-.md` files. An archive filename is a durable receipt that shadows and cleans restored inbox replicas and makes repeated archive cleanup idempotent. | `src/message.rs::filename_grammar`; `src/message.rs::archive_receipt_suppresses_and_idempotently_cleans_a_restored_inbox_copy`; `tests/message.rs` | | **Idempotent service requests** | A declared non-agent service principal publishes one exact JSON request per caller-supplied idempotency key to a canonical Agent Spec inbox. Concurrent or crash-replayed publication reuses the reserved filename; conflicting key reuse fails. The typed reply routes to the principal's canonical inbox without an Agent Spec identity or orphan mailbox. | `tests/request_cli.rs::stable_request_key_atomically_deduplicates_one_canonical_agent_message`; `tests/request_cli.rs::concurrent_replays_publish_exactly_one_request`; `tests/request_cli.rs::typed_reply_routes_to_the_principal_and_status_is_a_tagged_json_union`; `tests/request_cli.rs::request_api_rejects_agent_impersonation_and_unknown_flat_principals` | +| **Bounded idempotent stream ingress** | A running agent accepts events only for a declared stream. Within the retained 128-receipt ring, concurrent or crash-replayed `(stream, event-id)` publication reuses one canonical filename, conflicting content fails, and supersession archives only the matching keyed predecessor or the stream-wide head through ordinary archive semantics. State remains bounded and honestly treats an identity evicted from the ring as new without searching inbox or archive history. Events do not write the Sent ledger and DING marks them as stream work. | `tests/event_e2e.rs::stable_event_identity_publishes_exactly_one_canonical_message`; `tests/event_e2e.rs::concurrent_replays_publish_exactly_one_event`; `tests/event_e2e.rs::conflicting_reuse_and_undeclared_or_suspended_ingress_fail_closed`; `tests/event_e2e.rs::supersede_collapses_only_the_matching_key_and_preserves_archive_receipts`; `tests/event_e2e.rs::keyless_supersede_replaces_the_stream_wide_head`; `tests/event_e2e.rs::crash_replay_honors_an_archive_receipt_and_never_restores_the_inbox_copy`; `tests/event_e2e.rs::subject_frontmatter_injection_is_refused_before_any_write`; `tests/event_e2e.rs::stream_state_is_bounded_and_forgets_only_beyond_its_honest_horizon`; `tests/event_e2e.rs::event_emit_cli_returns_a_stable_json_receipt_and_ding_marks_the_record`; `tests/stream_authoring_cli.rs::a_direct_adapter_launch_executes_the_exact_event_cli_contract`; `tests/run.rs::suspend_and_resume_relaunch_the_agent_and_stream_together` | | **Fail-closed observed native DING** | Each unread message becomes one normalized `[DING]` frame. Fresh delivery records ownership, then preserves the one combined bracketed-paste, 0.5 second delay, and Return transaction. PTY and Return success are transport only: `Delivered` additionally requires adapter classification of the expected notice text in a submitted-prompt or queued-message pattern while the lowest live composer is empty or an accepted idle placeholder. Retry never re-pastes and may send one bare Return only after two adjacent `RetainedSafe` observations. A maintained adapter's positive `NotRetained` observation releases only an already archived staged head; unread, blocked, timed-out, errored, unknown, and unrecognized states retain ownership and later FIFO work remains blocked. Ownership prevents duplicate paste across command failures, receipt ambiguity, archive races, and restart adoption without letting a vanished archived head block FIFO indefinitely. Startup backlog otherwise becomes one generic recovery DING; new arrivals remain FIFO; `busy` delivers immediately; only fresh `dnd` defers. | `src/ding/mod.rs::poke_text_normalizes_and_bounds_untrusted_fields`; `src/ding/mod.rs::malicious_controls_cannot_escape_the_single_paste_frame`; `src/ding/mod.rs::pty_delivery_uses_face607_delay_order_and_seconds`; `src/ding/mod.rs::maintained_composer_classifiers_require_exact_idle_state`; `src/ding/mod.rs::successful_transport_with_retained_or_unproven_pixels_is_not_delivered`; `src/ding/mod.rs::ambiguous_transport_receipt_and_retry_errors_retain_staged_ownership`; `src/ding/mod.rs::adapter_recognized_notice_with_an_empty_live_composer_is_a_positive_receipt`; `src/ding/mod.rs::staged_retry_submits_only_retained_safe_and_requires_a_receipt`; `src/ding/mod.rs::staged_retry_keeps_unproven_and_retained_blocked_owned`; `src/ding/mod.rs::staged_ownership_survives_archive_and_never_repastes`; `src/ding/mod.rs::archived_not_retained_releases_fifo_without_repasting_owned_notice`; `src/ding/mod.rs::unread_not_retained_keeps_fifo_ownership_without_repasting`; `src/ding/mod.rs::pty_commands_have_a_real_outer_timeout`; `src/ding/mod.rs::session_watch_has_startup_grace_debounce_and_live_reset`; `src/ding/mod.rs::new_arrivals_is_fifo_and_archive_receipts_prevent_reding`; `src/ding/mod.rs::pending_delivery_ignores_busy_but_respects_fresh_dnd_archive_and_retry`; `src/ding/mod.rs::startup_recovery_notice_retries_in_memory`; `src/ding/mod.rs::startup_backlog_gets_one_generic_recovery_then_new_arrivals_poke` | | **Mutation-only filesystem wakeups** | Supervisor and DING filesystem watchers ignore read/open access events and wake early only for create, modify, rename, or remove events. Their own catalog and inbox reads therefore cannot bypass the bounded timer cadence or form a Linux inotify CPU loop. | `src/watch.rs::only_mutations_wake_watch_loops`; `src/watch.rs::linux_reads_are_silent_but_real_mutations_wake`; `src/ding/mod.rs::idle_ding_does_not_spin_on_its_own_inbox_reads`; `src/run.rs::idle_supervisor_does_not_spin_on_its_own_catalog_reads` | | **Bounded DING PTY probe churn** | An unsafe or active composer retains its FIFO notice but deferred delivery retries use a bounded backoff, so each inbox poll cannot spawn another short-lived PTY probe. | `src/ding/mod.rs::deferred_delivery_backoff_bounds_short_lived_pty_attempts` | diff --git a/crates/agent-spec/src/declared.rs b/crates/agent-spec/src/declared.rs index 69210e3b..db1c28e2 100644 --- a/crates/agent-spec/src/declared.rs +++ b/crates/agent-spec/src/declared.rs @@ -39,6 +39,7 @@ pub enum DeclaredDiagnosticCode { TaskNameMissing, UnsupportedSchedule, DuplicateRoutingField, + UnsupportedStreamInterval, } impl DeclaredDiagnosticCode { @@ -49,6 +50,7 @@ impl DeclaredDiagnosticCode { Self::TaskNameMissing => "task-name-missing", Self::UnsupportedSchedule => "unsupported-schedule", Self::DuplicateRoutingField => "duplicate-routing-field", + Self::UnsupportedStreamInterval => "unsupported-stream-interval", } } } @@ -301,7 +303,9 @@ pub fn parse_declared_document(source_name: &Path, source: &str) -> DeclaredPars } for child in &node.children { match child.name.as_str() { - "pty" | "exec" if child.argument(0).and_then(DeclaredValue::as_str).is_none() => { + "pty" | "exec" | "stream" + if child.argument(0).and_then(DeclaredValue::as_str).is_none() => + { diagnostics.push(shape_diagnostic( source_name, child.span, @@ -309,6 +313,21 @@ pub fn parse_declared_document(source_name: &Path, source: &str) -> DeclaredPars format!("{} task must have one positional string name", child.name), )); } + // A command-bearing stream lowers to a derived exec companion. A command-less + // stream is an external ingress endpoint. `every` would make either one a schedule, + // which is the reserved `schedule` node's business. + "stream" => { + for field in child.children_named("every") { + diagnostics.push(shape_diagnostic( + source_name, + field.span, + DeclaredDiagnosticCode::UnsupportedStreamInterval, + "stream `every` is reserved for the future `schedule` contract; a stream \ + declares a long-running event source" + .to_owned(), + )); + } + } "schedule" => diagnostics.push(shape_diagnostic( source_name, child.span, diff --git a/crates/agent-spec/src/discovery.rs b/crates/agent-spec/src/discovery.rs index eca745c9..191cc026 100644 --- a/crates/agent-spec/src/discovery.rs +++ b/crates/agent-spec/src/discovery.rs @@ -80,10 +80,7 @@ pub fn discover_strict(root: &Path) -> Discovered { /// /// `root` supplies the same path defaults as [`discover`]. The returned warnings describe only /// this file. -pub fn discover_file( - root: &Path, - path: &Path, -) -> anyhow::Result<(Vec, Vec)> { +pub fn discover_file(root: &Path, path: &Path) -> anyhow::Result<(Vec, Vec)> { let raws = parse_raw_file(path)?; load_specs(root, path, raws) } diff --git a/crates/agent-spec/src/kdl_format.rs b/crates/agent-spec/src/kdl_format.rs index 79a5a685..472eed6e 100644 --- a/crates/agent-spec/src/kdl_format.rs +++ b/crates/agent-spec/src/kdl_format.rs @@ -79,7 +79,10 @@ fn agent_node_to_raw(node: &DeclaredNode) -> anyhow::Result { "workspace" => raw.workspace = arg_string(child), "supervisor" => raw.supervisor = arg_string(child), "retired" => { - anyhow::ensure!(raw.retired.is_none(), "agent declares `retired` more than once"); + anyhow::ensure!( + raw.retired.is_none(), + "agent declares `retired` more than once" + ); raw.retired = Some(Some(arg_bool(child))); } "desired-state" => { @@ -138,10 +141,9 @@ fn agent_node_to_raw(node: &DeclaredNode) -> anyhow::Result { && child.entries[0].name.is_none(), "agent `deliver` must contain exactly one positional string" ); - raw.deliver = Some(Some( - arg_string(child) - .ok_or_else(|| anyhow::anyhow!("agent `deliver` value must be a string"))?, - )); + raw.deliver = Some(Some(arg_string(child).ok_or_else(|| { + anyhow::anyhow!("agent `deliver` value must be a string") + })?)); } "claude" => { anyhow::ensure!( @@ -175,6 +177,15 @@ fn agent_node_to_raw(node: &DeclaredNode) -> anyhow::Result { raw.exec.insert(name, task_node_to_raw(child)?); } } + "stream" => { + if let Some(name) = arg_string(child) { + let stream = stream_node_to_raw(child, &name)?; + anyhow::ensure!( + raw.stream.insert(name.clone(), stream).is_none(), + "agent declares `stream \"{name}\"` more than once" + ); + } + } // meta, harness, model, persona, permissions, transport, strategy, … — ignored. _ => {} } @@ -295,8 +306,7 @@ fn common_driver_fields( } fn claude_driver_node_to_raw(node: &DeclaredNode) -> anyhow::Result { - let (model, effort, dev_channels, prompt, args) = - common_driver_fields(node, "claude", true)?; + let (model, effort, dev_channels, prompt, args) = common_driver_fields(node, "claude", true)?; Ok(ClaudeDriver { model, effort, @@ -453,6 +463,47 @@ fn task_node_to_raw(node: &DeclaredNode) -> anyhow::Result { Ok(t) } +/// `stream "" { command "…" }` or `stream "" { argv "prog" "arg" }`. +/// +/// The child set is deliberately minimal. A stream declares WHERE events come from; everything about +/// how they are supervised is inherited from the agent (restart policy, teardown, parking), and +/// everything about how they are delivered is the bus contract. `every` is rejected by the +/// declaration parser rather than accepted here: an interval makes this scheduled work, which is +/// the reserved `schedule` node's contract. +fn stream_node_to_raw(node: &DeclaredNode, name: &str) -> anyhow::Result { + let mut stream = crate::spec::RawStream::default(); + for child in &node.children { + match child.name.as_str() { + "command" => { + anyhow::ensure!( + stream.command.is_none(), + "stream '{name}' has duplicate `command`" + ); + stream.command = Some(arg_string(child).ok_or_else(|| { + anyhow::anyhow!("stream '{name}' `command` must be one positional string") + })?); + } + "argv" => { + anyhow::ensure!( + stream.argv.is_none(), + "stream '{name}' has duplicate `argv`" + ); + stream.argv = Some(argv(child)?); + } + "every" => anyhow::bail!( + "stream '{name}' declares `every`; scheduled work is the reserved `schedule` \ + contract, a stream is a long-running event source" + ), + other => anyhow::bail!("stream '{name}' has unsupported field `{other}`"), + } + } + anyhow::ensure!( + !(stream.command.is_some() && stream.argv.is_some()), + "stream '{name}' must declare at most one of `command` or `argv`" + ); + Ok(stream) +} + fn env_node_to_raw(node: &DeclaredNode) -> std::collections::BTreeMap { let mut env = std::collections::BTreeMap::new(); for child in &node.children { diff --git a/crates/agent-spec/src/lib.rs b/crates/agent-spec/src/lib.rs index 76e4f574..a236dab1 100644 --- a/crates/agent-spec/src/lib.rs +++ b/crates/agent-spec/src/lib.rs @@ -44,6 +44,6 @@ pub use discovery::{ }; pub use spec::{ AgentDesiredState, AgentSpec, ClaudeDriver, CodexDriver, DeliveryTransport, Driver, JobType, - PiDriver, Resource, Restart, RestartMode, Task, TaskKind, TaskLifecycle, parse_duration, - validate_desired_state_reason, + PiDriver, Resource, Restart, RestartMode, STREAM_TASK_PREFIX, Stream, StreamLaunch, Task, + TaskKind, TaskLifecycle, parse_duration, stream_name_of_task, validate_desired_state_reason, }; diff --git a/crates/agent-spec/src/spec.rs b/crates/agent-spec/src/spec.rs index 44dc5e5e..9deb8555 100644 --- a/crates/agent-spec/src/spec.rs +++ b/crates/agent-spec/src/spec.rs @@ -32,9 +32,13 @@ pub const AGENT_DESIRED_STATE_REASON_MAX_BYTES: usize = 160; #[derive(Debug, Clone, PartialEq, Eq)] pub enum AgentDesiredState { Running, - Suspended { reason: String }, + Suspended { + reason: String, + }, /// `None` exists only for legacy `retired #true` declarations. - Retired { reason: Option }, + Retired { + reason: Option, + }, } /// One provider-native message delivery transport declared by an agent. @@ -186,6 +190,9 @@ pub struct AgentSpec { /// Named typed references used by the agent. st2 preserves these for readers but does not /// resolve them or assign launch, readiness, access, or lifecycle semantics. pub resources: Vec, + /// Named event subscriptions. Command-less streams are external ingress endpoints; launched + /// streams additionally lower to one derived exec companion. + pub streams: Vec, /// The runnable tasks (`pty` + `exec`), sorted by name for determinism. pub tasks: Vec, /// Where this spec was loaded from — the anchor for its resources and for edits. @@ -272,12 +279,9 @@ impl<'de> Deserialize<'de> for Resource { let descriptor = ResourceDescriptor::deserialize(deserializer)?; let resource = match (descriptor.relation, descriptor.reason) { (None, None) => Self::new(descriptor.name, descriptor.uri), - (Some(relation), Some(reason)) => Self::new_with_relation_reason( - descriptor.name, - descriptor.uri, - relation, - reason, - ), + (Some(relation), Some(reason)) => { + Self::new_with_relation_reason(descriptor.name, descriptor.uri, relation, reason) + } (Some(_), None) => Err(format!( "resource binding '{}' with `relation` must also declare string `reason`", descriptor.name @@ -503,6 +507,59 @@ pub(crate) struct RawSpec { /// `exec "" {}` / `[exec.]` — terminal-free tasks. #[serde(default)] pub exec: BTreeMap, + /// `stream "" {}` / `[stream.]` — an external event source whose stdout lines st2 + /// delivers into this agent's inbox. Lowers to one derived exec companion, exactly like `ding`. + #[serde(default)] + pub stream: BTreeMap, +} + +/// A declared event source. Exactly one of `command` / `argv` launches the source process. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)] +pub(crate) struct RawStream { + pub command: Option, + pub argv: Option>, +} + +/// One agent-owned event subscription. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Stream { + pub name: String, + pub launch: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case", tag = "type", content = "value")] +pub enum StreamLaunch { + Command(String), + Argv(Vec), +} + +/// The longest stream name that still leaves a legible `..stream-` runtime id. +const STREAM_NAME_MAX_CHARS: usize = 40; + +/// Derived stream task names are exactly `stream-`. +pub const STREAM_TASK_PREFIX: &str = "stream-"; + +/// The declared stream name behind a derived task name, if this is a stream companion. +pub fn stream_name_of_task(task_name: &str) -> Option<&str> { + task_name.strip_prefix(STREAM_TASK_PREFIX) +} + +/// A stream name becomes part of a runner-owned task name and runtime id, so it is restricted to a +/// lowercase slug. This keeps `stream-` unambiguous against the `.` id grammar. +fn validate_stream_name(identity: &str, name: &str) -> anyhow::Result<()> { + anyhow::ensure!( + !name.is_empty() && name.chars().count() <= STREAM_NAME_MAX_CHARS, + "agent '{identity}' stream name '{name}' must be 1..={STREAM_NAME_MAX_CHARS} characters" + ); + anyhow::ensure!( + name.chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + && !name.starts_with('-') + && !name.ends_with('-'), + "agent '{identity}' stream name '{name}' must match [a-z0-9]([a-z0-9-]*[a-z0-9])?" + ); + Ok(()) } /// The permissive raw envelope keeps the provider name at the same level in KDL, TOML, and JSON. @@ -841,8 +898,7 @@ fn validate_uri_component(value: &str, extra: &[u8]) -> Result<(), &'static str> } else if byte.is_ascii_alphanumeric() || matches!( byte, - b'-' - | b'.' + b'-' | b'.' | b'_' | b'~' | b'!' @@ -886,6 +942,7 @@ impl RawSpec { || !self.resource.0.is_empty() || !self.pty.is_empty() || !self.exec.is_empty() + || !self.stream.is_empty() } /// Lower into an [`AgentSpec`], with `identity`/`host` resolved from the path when content omits them. @@ -902,8 +959,7 @@ impl RawSpec { AGENT_DESCRIPTION_MAX_CHARS, )?; let retired = reject_explicit_null("retired", self.retired)?; - let desired_state_value = - reject_explicit_null("desired_state", self.desired_state)?; + let desired_state_value = reject_explicit_null("desired_state", self.desired_state)?; let desired_state_reason = reject_explicit_null("desired_state_reason", self.desired_state_reason)?; let desired_state = lower_desired_state( @@ -939,6 +995,14 @@ impl RawSpec { .trim_start_matches('.') .to_string(); let mut tasks: Vec = Vec::new(); + // Authored task names, captured before the maps are consumed: a derived stream companion must + // not silently shadow an explicit sibling that already owns `stream-`. + let authored_task_names = self + .pty + .keys() + .chain(self.exec.keys()) + .cloned() + .collect::>(); for (name, t) in self.pty { tasks.push(t.lower(&identity, TaskKind::Pty, name, &self.env)?); } @@ -973,7 +1037,67 @@ impl RawSpec { argv: None, cwd: None, tags: BTreeMap::new(), - env: self.env, + env: self.env.clone(), + keep: false, + lifecycle: TaskLifecycle::Service, + }); + } + // One derived exec companion per stream, through the exact seam the DING sidecar uses. The + // marker argv carries the declared source launch; `reconcile` late-binds argv[0] to the + // running st2 binary and substitutes the effective `ST_ROOT`, exactly as it does for DING + // and for a driver expansion. + let mut streams = Vec::new(); + for (name, stream) in self.stream { + validate_stream_name(&identity, &name)?; + let task_name = format!("stream-{name}"); + anyhow::ensure!( + !authored_task_names.contains(&task_name), + "agent '{identity}' declares both `stream \"{name}\"` and a task named \ + `{task_name}`; choose one form" + ); + let (command, argv) = match (stream.command, stream.argv) { + (Some(command), None) => { + anyhow::ensure!( + !command.trim().is_empty(), + "agent '{identity}' stream '{name}' has an empty `command`" + ); + (Some(command), None) + } + (None, Some(argv)) => { + anyhow::ensure!( + !argv.is_empty() && !argv[0].trim().is_empty(), + "agent '{identity}' stream '{name}' has an empty `argv`" + ); + (None, Some(argv)) + } + (None, None) => (None, None), + (Some(_), Some(_)) => anyhow::bail!( + "agent '{identity}' stream '{name}' declares both `command` and `argv`; choose one" + ), + }; + let launch = match (&command, &argv) { + (Some(command), None) => Some(StreamLaunch::Command(command.clone())), + (None, Some(argv)) => Some(StreamLaunch::Argv(argv.clone())), + (None, None) => None, + (Some(_), Some(_)) => unreachable!("validated above"), + }; + streams.push(Stream { + name: name.clone(), + launch, + }); + if command.is_none() && argv.is_none() { + continue; + } + tasks.push(Task { + kind: TaskKind::Exec, + derived: true, + name: task_name.clone(), + id: Some(format!("{bus_id}.{task_name}")), + command, + argv, + cwd: None, + tags: BTreeMap::new(), + env: self.env.clone(), keep: false, lifecycle: TaskLifecycle::Service, }); @@ -999,6 +1123,7 @@ impl RawSpec { delivery, driver, resources, + streams, tasks, path, }) @@ -1007,9 +1132,7 @@ impl RawSpec { /// Preserve the distinction between an omitted TOML/JSON field and an explicit `null`. /// Serde's ordinary `Option` representation intentionally collapses those cases. -fn deserialize_explicit_optional<'de, D, T>( - deserializer: D, -) -> Result>, D::Error> +fn deserialize_explicit_optional<'de, D, T>(deserializer: D) -> Result>, D::Error> where D: serde::Deserializer<'de>, T: Deserialize<'de>, @@ -1072,8 +1195,10 @@ pub fn validate_desired_state_reason(reason: &str) -> anyhow::Result<()> { ); anyhow::ensure!( reason.trim() == reason - && !reason.chars().any(|character| character.is_control() - || matches!(character, '\u{2028}' | '\u{2029}')), + && !reason + .chars() + .any(|character| character.is_control() + || matches!(character, '\u{2028}' | '\u{2029}')), "agent desired-state `reason` must have no surrounding Unicode whitespace, controls, or line separators" ); Ok(()) @@ -1222,7 +1347,7 @@ mod tests { "thing://bad^caret", "thing://bad`tick", "thing://bad{brace", - "thing://bad|pipe", + "thing://bad|stream", "thing://bad}brace", "thing://bad space", "thing://bad%2", diff --git a/crates/agent-spec/tests/discovery.rs b/crates/agent-spec/tests/discovery.rs index 485c3a2e..ac9c3774 100644 --- a/crates/agent-spec/tests/discovery.rs +++ b/crates/agent-spec/tests/discovery.rs @@ -52,7 +52,10 @@ fn desired_state_rejects_illegal_state_reason_combinations() { "non-string-reason", "desired-state \"suspended\" reason=#true", ), - ("unknown-property", "desired-state \"running\" because=\"no\""), + ( + "unknown-property", + "desired-state \"running\" because=\"no\"", + ), ( "duplicate-reason", "desired-state \"suspended\" reason=\"one\" reason=\"two\"", @@ -155,9 +158,7 @@ fn explicit_json_null_fields_are_rejected_instead_of_granting_default_behavior() write( tmp.path(), &format!("agents/h/{name}/agent.json"), - &format!( - r#"{{"identity":"{name}","host":"h",{lifecycle},"argv":["true"]}}"# - ), + &format!(r#"{{"identity":"{name}","host":"h",{lifecycle},"argv":["true"]}}"#), ); let found = discover(tmp.path()); assert!(found.specs.is_empty(), "{name}: {:?}", found.specs); @@ -724,7 +725,10 @@ fn driver_blocks_reject_ambiguous_providers_and_untyped_fields() { ("pi-dev", r#"pi { dev-channels #true; prompt "go" }"#), ("pi-missing-prompt", r#"pi { model "anthropic/x" }"#), ("missing-prompt", r#"claude { model "opus" }"#), - ("wrong-bool", r#"claude { dev-channels "yes"; prompt "go" }"#), + ( + "wrong-bool", + r#"claude { dev-channels "yes"; prompt "go" }"#, + ), ("codex-dev", r#"codex { dev-channels #true; prompt "go" }"#), ("unknown", r#"claude { presence #true; prompt "go" }"#), ] { @@ -1097,10 +1101,7 @@ fn malformed_resource_envelopes_are_rejected_without_defining_downstream_types() r#"resource "work" _tag="issue" uri="issue://example/1""#, ), ("missing-uri", r#"resource "work""#), - ( - "relative-uri", - r#"resource "work" uri="./issue/1""#, - ), + ("relative-uri", r#"resource "work" uri="./issue/1""#), ( "policy", r#"resource "work" uri="issue://example/1" required=#true"#, @@ -1828,3 +1829,80 @@ fn only_contextually_reserved_namespaces_are_ignored() { "reserved control/state namespaces must not become declarations" ); } + +#[test] +fn streams_are_typed_and_only_launched_streams_lower_to_derived_exec_tasks() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/h/worker/agent.kdl", + r#"agent "worker" { + host "h" + command "agent" + stream "external" {} + stream "shell" { command "watch-ci" } + stream "direct" { argv "watch" "--json" } +}"#, + ); + let found = discover(tmp.path()); + assert!(found.errors.is_empty(), "{:?}", found.errors); + let spec = find(&found.specs, "worker"); + assert_eq!( + spec.streams + .iter() + .map(|stream| stream.name.as_str()) + .collect::>(), + ["direct", "external", "shell"] + ); + assert!(spec.tasks.iter().all(|task| task.name != "stream-external")); + let direct = spec + .tasks + .iter() + .find(|task| task.name == "stream-direct") + .unwrap(); + assert_eq!(direct.kind, TaskKind::Exec); + assert!(direct.derived); + assert_eq!(argv(direct), ["watch", "--json"]); + let shell = spec + .tasks + .iter() + .find(|task| task.name == "stream-shell") + .unwrap(); + assert_eq!(shell.command.as_deref(), Some("watch-ci")); +} + +#[test] +fn stream_names_launches_and_task_collisions_fail_closed() { + for (identity, body, expected) in [ + ( + "both", + "stream \"x\" { command \"a\"; argv \"b\" }", + "at most one", + ), + ( + "every", + "stream \"x\" { every \"1m\" }", + "reserved for the future", + ), + ("bad-name", "stream \"Bad\" {}", "must match"), + ( + "collision", + "stream \"x\" { command \"a\" }; exec \"stream-x\" { command \"b\" }", + "declares both", + ), + ] { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + &format!("agents/h/{identity}/agent.kdl"), + &format!("agent \"{identity}\" {{ host \"h\"; command \"agent\"; {body} }}"), + ); + let found = discover(tmp.path()); + assert_eq!(found.errors.len(), 1, "{identity}: {:?}", found.errors); + assert!( + found.errors[0].message.contains(expected), + "{identity}: {:?}", + found.errors + ); + } +} diff --git a/crates/st2-wire/src/message.rs b/crates/st2-wire/src/message.rs index 44f778e8..c09a2942 100644 --- a/crates/st2-wire/src/message.rs +++ b/crates/st2-wire/src/message.rs @@ -34,7 +34,11 @@ pub struct MessageRow { pub tags: Vec, pub priority: Option, /// The caller's optional operation identity for exact retry semantics. - #[serde(rename = "idempotencyKey", default, skip_serializing_if = "Option::is_none")] + #[serde( + rename = "idempotencyKey", + default, + skip_serializing_if = "Option::is_none" + )] pub idempotency_key: Option, /// The markdown body. /// @@ -71,7 +75,11 @@ pub struct SentMessageRow { #[serde(default)] pub tags: Vec, pub priority: Option, - #[serde(rename = "idempotencyKey", default, skip_serializing_if = "Option::is_none")] + #[serde( + rename = "idempotencyKey", + default, + skip_serializing_if = "Option::is_none" + )] pub idempotency_key: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub body: Option, @@ -205,7 +213,9 @@ mod tests { ); let indexed = SentMessages { - coverage: SentCoverage::Since { since: 1_785_000_000_000 }, + coverage: SentCoverage::Since { + since: 1_785_000_000_000, + }, messages: vec![SentMessageRow { filename: "1785000000000-abcdef.md".to_string(), ts: 1_785_000_000_000, diff --git a/docs/vrs/.delta/DELTA-003-stream-subsystem-specified-not-implemented.md b/docs/vrs/.delta/DELTA-003-stream-subsystem-specified-not-implemented.md deleted file mode 100644 index 5fee9e45..00000000 --- a/docs/vrs/.delta/DELTA-003-stream-subsystem-specified-not-implemented.md +++ /dev/null @@ -1,60 +0,0 @@ -# DELTA-003: the stream subsystem is ratified with no shipped implementation - -Status: open - -## Divergence - -[04-stream/requirements.md](../04-stream/requirements.md) (STREAM-R01..R09) is -ratified and [04-stream/spec.md](../04-stream/spec.md) is a settled Draft, but -`main` contains no stream code: no `stream` KDL node, no `st2 event emit`, no -`st2 stream add/rm`, no dedup ring, no `»` DING marker. The evidence behind the -design lives in the four committed -[04-stream experiments](../04-stream/.experiments/). The implementation is -under review in the stacked [PR #300](https://github.com/compoundingtech/st2/pull/300), -not yet shipped on `main`. - -## VRS - -Decisions [0004](../.decisions/0004-stream-events-are-a-distinct-record-kind.md) -and [0005](../.decisions/0005-streams-are-agent-nested-and-stream-named.md) are -accepted; design issue -[#286](https://github.com/compoundingtech/st2/issues/286) requests upstream -review per root spec DQ1's approval bar. Open sub-questions are -[DQ-S1..DQ-S7](../04-stream/open-questions.md); DQ-S1 (producer `from` -grammar) blocks the event-record wire shape and should be resolved first. - -## Implementation - -[PR #300](https://github.com/compoundingtech/st2/pull/300) is the durable -implementation record. Its reviewed commit series implements declared ingress, -fail-closed and no-follow boundaries, bounded publication state, stream -authoring, lifecycle lowering, and publish-before-compact crash safety. The -normative design inputs remain decision 0005 and the committed differentiation -and lifecycle experiments; local worktree names are deliberately not -provenance. - -The remaining review work is tracked on that PR and its exact head rather than -copied here as a mutable commit list. In particular, its publication state must -reconcile an abandoned pending reservation as specified in STREAM-R05 before -this delta can close. Wait-style adapters must keep their process alive after -a terminal emit (DQ-S8): the task model has no run-to-completion lifecycle, so -an exiting adapter flaps into a park. - -Out-of-repo obligation: the `stream` node is an Agent Spec capability, so the -canonical `compoundingtech/evals/AGENT-SPEC.md` and the -[02-agent-spec](../02-agent-spec/spec.md) field rules must gain it in the same -effort — root R01 forbids shipping an undeclared capability, and R02's -admission must move `stream` from unknown-node rejection to typed validation. - -## Direction - -update implementation - -## Resolution Signal - -The verification plan in [04-stream/spec.md](../04-stream/spec.md) is -realized: the ingress, bounded-state, supersession, and companion-lifecycle -proofs exist under their final test names, the new INVARIANTS rows land with -`qualified_proof_references_resolve` green, `AGENT-SPEC.md` and 02-agent-spec -carry the `stream` field rules, and 04-stream/spec.md flips its Status from -Draft to Active. Close this delta in the commit that flips the Status line. diff --git a/docs/vrs/02-agent-spec/spec.md b/docs/vrs/02-agent-spec/spec.md index 5d3982b2..c078ca36 100644 --- a/docs/vrs/02-agent-spec/spec.md +++ b/docs/vrs/02-agent-spec/spec.md @@ -490,6 +490,32 @@ change lands. st2 source: [`AgentSpec`](../../../crates/agent-spec/src/spec.rs), [roster](../../../src/agents.rs), and [reconciliation](../../../src/reconcile.rs). Evidence: parser, roster, exact-ID metadata, and no-restart presentation tests. +

F19 Agent stream

+ +A `stream "" {}` declares one agent-owned event ingress endpoint. Names +are 1..=40 characters matching +`[a-z0-9]([a-z0-9-]*[a-z0-9])?` and cannot collide with an authored task named +`stream-`. The declaration contains at most one launch: `command` is an +opaque shell command, `argv` is a non-empty structured argument vector, and an +empty body means external ingress. Unknown children, including the reserved +`every`, fail admission. + +A launched stream adds exactly one derived exec task named `stream-` and +with runtime ID `..stream-`. Its authored `command` or +`argv` lowers directly to that task; no stream runner or stdout line protocol +is inserted. An external-ingress stream adds no task. Adding or removing a +launched stream therefore adds or removes that exact derived companion under +the owning agent's lifecycle; changing its launch is spawn-input drift under +F11. It does not change the canonical agent task or select a delivery +transport. + +Authoring: canonical Agent Spec stream field after the matching evals change +lands. st2 source: [`Stream`](../../../crates/agent-spec/src/spec.rs), +[KDL lowering](../../../crates/agent-spec/src/kdl_format.rs), and +[reconciliation](../../../src/reconcile.rs). Evidence: +[`streams_are_typed_and_only_launched_streams_lower_to_derived_exec_tasks`](../../../crates/agent-spec/tests/discovery.rs) +and stream lifecycle tests in [`tests/run.rs`](../../../tests/run.rs). + Catalog and PTY roots are host runtime inputs, not Agent Spec fields. Their migration contract is outside this VRS. See [#85](https://github.com/compoundingtech/st2/issues/85). @@ -559,6 +585,11 @@ replacement of drifted work. structured diagnostic envelope. Publication verifies exact digests and a full-catalog overlay before its atomic transition, but must also re-admit the published view under the lock and bind the policy profile in its receipt. +- **G11, F19 canonical ownership:** st2 admits, lowers, authors, and runs the + `stream` field, but the canonical evals `AGENT-SPEC.md` and its maintained + acceptance cells do not yet define or prove that capability. Until the + matching evals change lands, st2's stream implementation is ahead of the + authoring authority rather than conformant to it. ## Acceptance cases diff --git a/src/agent_author.rs b/src/agent_author.rs index fd69190b..5747a2b5 100644 --- a/src/agent_author.rs +++ b/src/agent_author.rs @@ -14,7 +14,7 @@ use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; use std::path::{Path, PathBuf}; use agent_spec::spec::{ - AGENT_DESCRIPTION_MAX_CHARS, AGENT_NAME_MAX_CHARS, validate_desired_state_reason, + AGENT_DESCRIPTION_MAX_CHARS, AGENT_NAME_MAX_CHARS, StreamLaunch, validate_desired_state_reason, validate_presentation, }; use kdl::{KdlDocument, KdlNode}; @@ -117,6 +117,23 @@ pub struct DesiredStateReceipt { pub reason: Option, } +/// Stable machine-readable receipt from adding one agent-owned stream. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct StreamAddReceipt { + pub result: AuthorOutcome, + pub identity: String, + pub name: String, + pub launch: Option, +} + +/// Stable machine-readable receipt from removing one agent-owned stream. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct StreamRemoveReceipt { + pub result: AuthorOutcome, + pub identity: String, + pub name: String, +} + /// A classified authoring refusal. `code` is stable for machine consumers. #[derive(Debug)] pub struct AuthorError { @@ -154,6 +171,100 @@ struct AgentTarget { retired: bool, } +/// Add an agent-owned stream, or prove that the identical declaration already exists. +pub fn add_stream( + catalog_root: &Path, + selector: &str, + this_host: &str, + actor: Option<&str>, + name: &str, + launch: Option, +) -> Result { + author_stream( + catalog_root, + selector, + this_host, + actor, + name, + launch.as_ref(), + false, + ) + .map(|(result, identity)| StreamAddReceipt { + result, + identity, + name: name.to_owned(), + launch, + }) +} + +/// Remove one agent-owned stream. An already absent stream is an idempotent success. +pub fn remove_stream( + catalog_root: &Path, + selector: &str, + this_host: &str, + actor: Option<&str>, + name: &str, +) -> Result { + author_stream(catalog_root, selector, this_host, actor, name, None, true).map( + |(result, identity)| StreamRemoveReceipt { + result, + identity, + name: name.to_owned(), + }, + ) +} + +fn author_stream( + catalog_root: &Path, + selector: &str, + this_host: &str, + actor: Option<&str>, + name: &str, + launch: Option<&StreamLaunch>, + remove: bool, +) -> Result<(AuthorOutcome, String), AuthorError> { + let catalog_lock = CatalogLock::exclusive(catalog_root).map_err(|error| { + AuthorError::new( + "catalog-lock-failed", + format!("acquire catalog-authoring lock: {error:#}"), + ) + })?; + let found = crate::discover(catalog_root); + if let Some(error) = found.errors.first() { + return Err(AuthorError::new( + "catalog-malformed", + format!( + "cannot prove an exact stream target while {} is malformed: {}", + error.path.display(), + error.message + ), + )); + } + let target = resolve_target(&found.specs, selector, this_host)?; + authorize_actor( + &found.specs, + &target.identity, + this_host, + actor, + "stream-not-authorized", + )?; + let result = edit_stream_declaration( + &catalog_lock, + catalog_root, + &crate::catalog_transaction::retained_dir_path(catalog_lock.control()) + .map_err(|error| AuthorError::new("declaration-write-failed", error.to_string()))?, + &target.declaration, + &target.identity, + &target.source_host, + &target.source_identity, + name, + launch, + remove, + || {}, + )?; + Ok((result, target.identity)) +} + /// Author one whole-agent desired state without claiming runtime convergence. pub fn set_desired_state( catalog_root: &Path, @@ -179,9 +290,8 @@ pub fn set_desired_state( _ => {} } if let Some(reason) = reason { - validate_desired_state_reason(reason).map_err(|error| { - AuthorError::new("invalid-desired-state", error.to_string()) - })?; + validate_desired_state_reason(reason) + .map_err(|error| AuthorError::new("invalid-desired-state", error.to_string()))?; } let catalog_lock = CatalogLock::exclusive(catalog_root).map_err(|error| { AuthorError::new( @@ -195,7 +305,8 @@ pub fn set_desired_state( "catalog-malformed", format!( "cannot prove an exact desired-state target while {} is malformed: {}", - error.path.display(), error.message + error.path.display(), + error.message ), )); } @@ -448,6 +559,245 @@ fn edit_desired_state_for_test( ) } +#[allow(clippy::too_many_arguments)] +fn edit_stream_declaration( + catalog_lock: &CatalogLock, + catalog: &Path, + control: &Path, + path: &Path, + expected_identity: &str, + expected_host: &str, + expected_agent: &str, + name: &str, + launch: Option<&StreamLaunch>, + remove: bool, + before_commit: impl FnOnce(), +) -> Result { + if path.extension().and_then(|value| value.to_str()) != Some("kdl") { + return Err(AuthorError::new( + "unsupported-declaration-format", + format!( + "stream authoring requires canonical KDL, found {}", + path.display() + ), + )); + } + let metadata = fs::symlink_metadata(path).map_err(|error| { + AuthorError::new( + "declaration-read-failed", + format!("reading declaration {}: {error}", path.display()), + ) + })?; + if !metadata.file_type().is_file() { + return Err(AuthorError::new( + "unsafe-declaration-path", + format!("refusing non-regular declaration path {}", path.display()), + )); + } + let original = fs::read(path).map_err(|error| { + AuthorError::new( + "declaration-read-failed", + format!("reading declaration {}: {error}", path.display()), + ) + })?; + let original_version = SourceVersion::from_metadata(&metadata); + let text = std::str::from_utf8(&original).map_err(|error| { + AuthorError::new( + "malformed-declaration", + format!("declaration {} is not UTF-8: {error}", path.display()), + ) + })?; + let document = KdlDocument::parse(text).map_err(|error| { + AuthorError::new( + "malformed-declaration", + format!("parsing declaration {}: {error}", path.display()), + ) + })?; + let target = exact_agent_node(&document, expected_identity, expected_host, expected_agent)?; + if is_nix_managed(target) { + return Err(AuthorError::new( + "nix-managed-declaration", + format!( + "agent {expected_identity:?} is Nix-owned; edit its Nix source instead of {}", + path.display() + ), + )); + } + let replacement = stream_edit(text, target, name, launch, remove)?; + let Some(replacement) = replacement else { + return Ok(AuthorOutcome::Unchanged); + }; + verify_stream_candidate( + catalog, + path, + &replacement, + expected_agent, + name, + launch, + remove, + )?; + atomic_replace_checked( + catalog_lock, + catalog, + control, + path, + &original, + original_version, + replacement.as_bytes(), + metadata.permissions().mode() & 0o7777, + before_commit, + )?; + Ok(AuthorOutcome::Changed) +} + +fn stream_edit( + text: &str, + target: &KdlNode, + name: &str, + launch: Option<&StreamLaunch>, + remove: bool, +) -> Result, AuthorError> { + let streams = target + .children() + .into_iter() + .flat_map(|children| children.nodes()) + .filter(|child| { + child.name().value() == "stream" + && child.get(0).and_then(|entry| entry.as_string()) == Some(name) + }) + .collect::>(); + if streams.len() > 1 { + return Err(AuthorError::new( + "duplicate-stream", + format!("target declares stream {name:?} more than once"), + )); + } + if remove { + return streams + .first() + .map(|node| remove_field(text, node).map(Some)) + .unwrap_or(Ok(None)); + } + if let Some(existing) = streams.first() { + if parsed_stream_launch(existing)? == launch.cloned() { + return Ok(None); + } + return Err(AuthorError::new( + "stream-already-exists", + format!( + "stream {name:?} already exists with a different launch; remove it before adding a replacement" + ), + )); + } + let authored = match launch { + None => format!("stream {} {{}}", quoted(name)?), + Some(StreamLaunch::Command(command)) => format!( + "stream {} {{ command {} }}", + quoted(name)?, + quoted(command)? + ), + Some(StreamLaunch::Argv(argv)) => { + let values = argv + .iter() + .map(|value| quoted(value)) + .collect::, _>>()?; + format!("stream {} {{ argv {} }}", quoted(name)?, values.join(" ")) + } + }; + insert_node(text, target, &authored).map(Some) +} + +fn parsed_stream_launch(node: &KdlNode) -> Result, AuthorError> { + let children = node + .children() + .into_iter() + .flat_map(|children| children.nodes()) + .collect::>(); + match children.as_slice() { + [] => Ok(None), + [child] if child.name().value() == "command" => child + .get(0) + .and_then(|entry| entry.as_string()) + .map(|value| Some(StreamLaunch::Command(value.to_owned()))) + .ok_or_else(|| { + AuthorError::new("malformed-stream", "stream command must contain one string") + }), + [child] if child.name().value() == "argv" => { + let argv = child + .entries() + .iter() + .map(|entry| entry.value().as_string().map(str::to_owned)) + .collect::>>() + .ok_or_else(|| { + AuthorError::new("malformed-stream", "stream argv values must be strings") + })?; + Ok(Some(StreamLaunch::Argv(argv))) + } + _ => Err(AuthorError::new( + "malformed-stream", + "stream must contain exactly one command or argv node, or be empty", + )), + } +} + +fn verify_stream_candidate( + catalog: &Path, + path: &Path, + candidate: &str, + expected_agent: &str, + name: &str, + launch: Option<&StreamLaunch>, + removed: bool, +) -> Result<(), AuthorError> { + let temporary = tempfile::tempdir() + .map_err(|error| AuthorError::new("unsafe-source-edit", error.to_string()))?; + let relative = path.strip_prefix(catalog).map_err(|_| { + AuthorError::new( + "unsafe-declaration-path", + format!( + "declaration {} is outside catalog {}", + path.display(), + catalog.display() + ), + ) + })?; + let candidate_path = temporary.path().join(relative); + fs::create_dir_all( + candidate_path + .parent() + .expect("candidate declaration has a parent"), + ) + .and_then(|()| fs::write(&candidate_path, candidate)) + .map_err(|error| { + AuthorError::new( + "unsafe-source-edit", + format!("stage stream validation: {error}"), + ) + })?; + let (specs, _) = agent_spec::discover_file(temporary.path(), &candidate_path) + .map_err(|error| AuthorError::new("invalid-stream", error.to_string()))?; + let spec = specs + .iter() + .find(|spec| spec.identity == expected_agent) + .ok_or_else(|| { + AuthorError::new( + "unsafe-source-edit", + "stream candidate lost the authored agent", + ) + })?; + let observed = spec.streams.iter().find(|stream| stream.name == name); + if removed && observed.is_none() + || !removed && observed.is_some_and(|stream| stream.launch.as_ref() == launch) + { + Ok(()) + } else { + Err(AuthorError::new( + "unsafe-source-edit", + "stream candidate did not read back as the authored intent", + )) + } +} + fn edit_declaration( catalog_lock: &CatalogLock, catalog: &Path, @@ -551,11 +901,17 @@ fn edit_desired_state_declaration( if path.extension().and_then(|value| value.to_str()) != Some("kdl") { return Err(AuthorError::new( "unsupported-declaration-format", - format!("desired-state authoring requires canonical KDL, found {}", path.display()), + format!( + "desired-state authoring requires canonical KDL, found {}", + path.display() + ), )); } let metadata = fs::symlink_metadata(path).map_err(|error| { - AuthorError::new("declaration-read-failed", format!("reading declaration {}: {error}", path.display())) + AuthorError::new( + "declaration-read-failed", + format!("reading declaration {}: {error}", path.display()), + ) })?; if !metadata.file_type().is_file() { return Err(AuthorError::new( @@ -564,20 +920,32 @@ fn edit_desired_state_declaration( )); } let original = fs::read(path).map_err(|error| { - AuthorError::new("declaration-read-failed", format!("reading declaration {}: {error}", path.display())) + AuthorError::new( + "declaration-read-failed", + format!("reading declaration {}: {error}", path.display()), + ) })?; let original_version = SourceVersion::from_metadata(&metadata); let text = std::str::from_utf8(&original).map_err(|error| { - AuthorError::new("malformed-declaration", format!("declaration {} is not UTF-8: {error}", path.display())) + AuthorError::new( + "malformed-declaration", + format!("declaration {} is not UTF-8: {error}", path.display()), + ) })?; let document = KdlDocument::parse(text).map_err(|error| { - AuthorError::new("malformed-declaration", format!("parsing declaration {}: {error}", path.display())) + AuthorError::new( + "malformed-declaration", + format!("parsing declaration {}: {error}", path.display()), + ) })?; let target = exact_agent_node(&document, expected_identity, expected_host, expected_agent)?; if is_nix_managed(target) { return Err(AuthorError::new( "nix-managed-declaration", - format!("agent {expected_identity:?} is Nix-owned; edit its Nix source instead of {}", path.display()), + format!( + "agent {expected_identity:?} is Nix-owned; edit its Nix source instead of {}", + path.display() + ), )); } let Some(replacement) = desired_state_edit(text, target, state, reason)? else { @@ -659,7 +1027,10 @@ fn verify_desired_state_candidate( reason: Option<&str>, ) -> Result<(), AuthorError> { let document = KdlDocument::parse(candidate).map_err(|error| { - AuthorError::new("unsafe-source-edit", format!("desired-state edit did not produce valid KDL: {error}")) + AuthorError::new( + "unsafe-source-edit", + format!("desired-state edit did not produce valid KDL: {error}"), + ) })?; let target = exact_agent_node(&document, expected_identity, expected_host, expected_agent)?; let lifecycle = target @@ -1445,4 +1816,137 @@ mod tests { "agent \"worker\" { host \"h\"; command \"sleep 60\"; desired-state \"suspended\" reason=\"Waiting for capacity\" }\nagent { host \"h\"; command \"sleep 60\" }\n" ); } + + #[test] + fn stream_add_supports_external_command_and_argv_and_remove_is_idempotent() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path(); + let path = write( + root, + "h/worker/agent.kdl", + &declaration("worker", "h", None, "catalog"), + ); + let original = fs::read_to_string(&path).unwrap(); + + assert_eq!( + add_stream(root, "h.worker", "h", Some("h.worker"), "webhook", None) + .unwrap() + .result, + AuthorOutcome::Changed + ); + assert_eq!( + add_stream( + root, + "h.worker", + "h", + Some("h.worker"), + "github-ci", + Some(StreamLaunch::Command("gh watch --repo st2".to_owned())), + ) + .unwrap() + .result, + AuthorOutcome::Changed + ); + assert_eq!( + add_stream( + root, + "h.worker", + "h", + Some("h.worker"), + "tick", + Some(StreamLaunch::Argv(vec![ + "tick-source".to_owned(), + "--daily".to_owned() + ])), + ) + .unwrap() + .result, + AuthorOutcome::Changed + ); + assert_eq!( + add_stream(root, "h.worker", "h", Some("h.worker"), "webhook", None) + .unwrap() + .result, + AuthorOutcome::Unchanged + ); + let authored = fs::read_to_string(&path).unwrap(); + assert!(authored.contains("stream \"webhook\" {}")); + assert!(authored.contains("stream \"github-ci\" { command \"gh watch --repo st2\" }")); + assert!(authored.contains("stream \"tick\" { argv \"tick-source\" \"--daily\" }")); + + for name in ["webhook", "github-ci", "tick"] { + assert_eq!( + remove_stream(root, "h.worker", "h", None, name) + .unwrap() + .result, + AuthorOutcome::Changed + ); + assert_eq!( + remove_stream(root, "h.worker", "h", None, name) + .unwrap() + .result, + AuthorOutcome::Unchanged + ); + } + assert_eq!(fs::read_to_string(path).unwrap(), original); + } + + #[test] + fn stream_authoring_enforces_authority_nix_ownership_and_canonical_validation() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path(); + write( + root, + "h/root/agent.kdl", + &declaration("root", "h", None, "catalog"), + ); + write( + root, + "h/child/agent.kdl", + &declaration("child", "h", Some("root"), "catalog"), + ); + write( + root, + "h/sibling/agent.kdl", + &declaration("sibling", "h", Some("root"), "catalog"), + ); + write( + root, + "h/nix/agent.kdl", + &declaration("nix", "h", Some("root"), "nix"), + ); + + add_stream(root, "h.child", "h", Some("h.root"), "events", None).unwrap(); + assert_eq!( + add_stream(root, "h.sibling", "h", Some("h.child"), "events", None) + .unwrap_err() + .code(), + "stream-not-authorized" + ); + assert_eq!( + add_stream(root, "h.nix", "h", Some("h.root"), "events", None) + .unwrap_err() + .code(), + "nix-managed-declaration" + ); + assert_eq!( + add_stream(root, "h.child", "h", None, "Bad Name", None) + .unwrap_err() + .code(), + "invalid-stream" + ); + assert_eq!( + add_stream( + root, + "h.child", + "h", + None, + "empty-argv", + Some(StreamLaunch::Argv(Vec::new())), + ) + .unwrap_err() + .code(), + "invalid-stream" + ); + } } diff --git a/src/catalog.rs b/src/catalog.rs index 2d8f7b1c..03bea5e9 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -63,7 +63,9 @@ pub fn parse(text: &str) -> anyhow::Result { .and_then(|v| v.as_string()) .filter(|v| !v.is_empty()) .ok_or_else(|| { - anyhow::anyhow!("pty-root needs a non-empty path, e.g. pty-root \"/run/agents/pty\"") + anyhow::anyhow!( + "pty-root needs a non-empty path, e.g. pty-root \"/run/agents/pty\"" + ) })?; config.pty_root = Some(value.to_string()); } @@ -109,7 +111,11 @@ mod tests { assert_eq!(pty_root(tmp.path()), tmp.path().join("pty")); // A file that declares other things, but no pty root. - std::fs::write(config_path(tmp.path()), "agent \"a\" { command \"true\" }\n").unwrap(); + std::fs::write( + config_path(tmp.path()), + "agent \"a\" { command \"true\" }\n", + ) + .unwrap(); assert_eq!(pty_root(tmp.path()), tmp.path().join("pty")); } @@ -123,12 +129,19 @@ mod tests { .unwrap(); assert_eq!(pty_root(tmp.path()), PathBuf::from("/run/agents/pty")); - std::fs::write(config_path(tmp.path()), "catalog { pty-root \"$CATALOG/../shared\" }\n") - .unwrap(); + std::fs::write( + config_path(tmp.path()), + "catalog { pty-root \"$CATALOG/../shared\" }\n", + ) + .unwrap(); assert_eq!(pty_root(tmp.path()), tmp.path().join("../shared")); // A relative value belongs to the catalog, never to the caller's cwd. - std::fs::write(config_path(tmp.path()), "catalog { pty-root \"registry\" }\n").unwrap(); + std::fs::write( + config_path(tmp.path()), + "catalog { pty-root \"registry\" }\n", + ) + .unwrap(); assert_eq!(pty_root(tmp.path()), tmp.path().join("registry")); } @@ -142,8 +155,11 @@ mod tests { // Reported by `st2 validate`; the runtime path stays on the native root. let tmp = tempfile::tempdir().unwrap(); - std::fs::write(config_path(tmp.path()), "catalog { pty_root \"/run/agents/pty\" }\n") - .unwrap(); + std::fs::write( + config_path(tmp.path()), + "catalog { pty_root \"/run/agents/pty\" }\n", + ) + .unwrap(); assert_eq!(pty_root(tmp.path()), tmp.path().join("pty")); } } diff --git a/src/catalog_transaction.rs b/src/catalog_transaction.rs index 4bb7cf34..e6e4f656 100644 --- a/src/catalog_transaction.rs +++ b/src/catalog_transaction.rs @@ -782,10 +782,7 @@ fn normalize_agent(spec: &agent_spec::AgentSpec) -> Result { - fields.insert( - format!("{root}/cwd"), - default_atom(SemanticType::String), - ); + fields.insert(format!("{root}/cwd"), default_atom(SemanticType::String)); } } for (key, value) in &task.tags { @@ -1177,13 +1174,7 @@ pub fn bootstrap(request: BootstrapRequest) -> Result { match fs::symlink_metadata(&catalog) { Ok(_) => { - return inspect_existing_bootstrap( - &parent_file, - name, - &catalog, - &prepared, - &desired, - ); + return inspect_existing_bootstrap(&parent_file, name, &catalog, &prepared, &desired); } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => { @@ -1231,13 +1222,7 @@ pub fn bootstrap(request: BootstrapRequest) -> Result { Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { drop(staged_lock); let _ = remove_tree_at(&parent_file, &source_name); - return inspect_existing_bootstrap( - &parent_file, - name, - &catalog, - &prepared, - &desired, - ); + return inspect_existing_bootstrap(&parent_file, name, &catalog, &prepared, &desired); } Err(error) => { drop(staged_lock); @@ -1382,7 +1367,10 @@ fn unlinkat(parent: &File, name: &std::ffi::OsStr, flags: libc::c_int) -> std::i use std::os::unix::ffi::OsStrExt as _; let name = CString::new(name.as_bytes()).map_err(|_| { - std::io::Error::new(std::io::ErrorKind::InvalidInput, "path component contains NUL") + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "path component contains NUL", + ) })?; let result = unsafe { libc::unlinkat(parent.as_raw_fd(), name.as_ptr(), flags) }; if result == 0 { diff --git a/src/codex_app_server.rs b/src/codex_app_server.rs index e7574512..98a81a18 100644 --- a/src/codex_app_server.rs +++ b/src/codex_app_server.rs @@ -1964,14 +1964,15 @@ fn pump_control( } } } - let message = match poll_json_message(&mut websocket).context("polling Codex control socket")? { - ControlRead::Message(message) => Some(message), - ControlRead::Timeout => None, - ControlRead::Closed => { - let _ = events.send(ControlEvent::Closed); - return Ok(()); - } - }; + let message = + match poll_json_message(&mut websocket).context("polling Codex control socket")? { + ControlRead::Message(message) => Some(message), + ControlRead::Timeout => None, + ControlRead::Closed => { + let _ = events.send(ControlEvent::Closed); + return Ok(()); + } + }; let Some(message) = message else { if let (Some(state), Some(delivery)) = (control_state.as_ref(), delivery.as_mut()) && let Some(request) = delivery.maybe_request(state)? @@ -1994,10 +1995,15 @@ fn pump_control( ); subscription_pending = false; let mut bound = CodexControlState::new(runtime, thread_id.to_string()); - match bound.accept_subscription(&message).context("accepting Codex resume subscription")? { + match bound + .accept_subscription(&message) + .context("accepting Codex resume subscription")? + { SubscriptionAcceptance::Accepted { .. } => { if let Some(delivery) = delivery.as_mut() { - delivery.reconcile_resume(&message, &bound).context("reconciling Codex resume delivery")?; + delivery + .reconcile_resume(&message, &bound) + .context("reconciling Codex resume delivery")?; } } SubscriptionAcceptance::Deferred => anyhow::bail!( @@ -2009,13 +2015,16 @@ fn pump_control( &CodexThreadBinding::new(runtime, thread_id.to_string()), ) .context("persisting Codex resume binding")?; - atomic_json(control_state_path, &bound).context("persisting Codex control state")?; + atomic_json(control_state_path, &bound) + .context("persisting Codex control state")?; control_state = Some(bound); let _ = events.send(ControlEvent::Bound); continue; } - let Some(thread_id) = binding_candidate(&message).context("reading Codex thread binding candidate")? else { + let Some(thread_id) = binding_candidate(&message) + .context("reading Codex thread binding candidate")? + else { continue; }; atomic_json( @@ -2028,7 +2037,8 @@ fn pump_control( // notification is already subscribed to that thread's broadcasts. Before its // first turn there is no persisted rollout for a redundant `thread/resume`. bound.subscribed = true; - atomic_json(control_state_path, &bound).context("persisting Codex fresh control state")?; + atomic_json(control_state_path, &bound) + .context("persisting Codex fresh control state")?; control_state = Some(bound); let _ = events.send(ControlEvent::Bound); } @@ -2038,8 +2048,12 @@ fn pump_control( .context("Codex control state is unbound")?; let delivery_response = match delivery.as_mut() { Some(delivery) => { - delivery.accept_response(&message, &state.observed).context("accepting Codex delivery response")? - || delivery.accept_typed_receipt(&message, state).context("accepting Codex typed receipt")? + delivery + .accept_response(&message, &state.observed) + .context("accepting Codex delivery response")? + || delivery + .accept_typed_receipt(&message, state) + .context("accepting Codex typed receipt")? } None => false, }; @@ -2053,20 +2067,28 @@ fn pump_control( "Codex control received an unexpected thread/resume response" ); subscription_pending = false; - match state.accept_subscription(&message).context("accepting Codex subscription")? { + match state + .accept_subscription(&message) + .context("accepting Codex subscription")? + { SubscriptionAcceptance::Accepted { changed } => { if let Some(delivery) = delivery.as_mut() { - delivery.reconcile_resume(&message, state).context("reconciling Codex subscription delivery")?; + delivery + .reconcile_resume(&message, state) + .context("reconciling Codex subscription delivery")?; } changed } SubscriptionAcceptance::Deferred => false, } } else { - state.observe(&message).context("observing Codex control event")? + state + .observe(&message) + .context("observing Codex control event")? }; if changed { - atomic_json(control_state_path, state).context("persisting Codex observed control state")?; + atomic_json(control_state_path, state) + .context("persisting Codex observed control state")?; let _ = events.send(ControlEvent::Observed); } if !state.subscribed @@ -3388,7 +3410,10 @@ mod tests { ) }); let first_event = rx.recv_timeout(Duration::from_secs(2)).unwrap(); - assert!(matches!(first_event, ControlEvent::Bound), "first control event: {first_event:?}"); + assert!( + matches!(first_event, ControlEvent::Bound), + "first control event: {first_event:?}" + ); server.join().unwrap(); let _ = shutdown.shutdown(Shutdown::Both); pump.join().unwrap(); @@ -4550,9 +4575,7 @@ mod tests { let mut command = Command::new("sh"); command .arg("-c") - .arg( - r#"sh -c 'printf "%s" "$$" > "$DESCENDANT_PIDFILE"; exec sleep 60' & sleep 60"#, - ) + .arg(r#"sh -c 'printf "%s" "$$" > "$DESCENDANT_PIDFILE"; exec sleep 60' & sleep 60"#) .env("DESCENDANT_PIDFILE", &descendant_pidfile) .stdin(Stdio::null()) .stdout(Stdio::null()) diff --git a/src/ding/mod.rs b/src/ding/mod.rs index 05d60ef3..1222572f 100644 --- a/src/ding/mod.rs +++ b/src/ding/mod.rs @@ -36,6 +36,9 @@ const BRACKETED_PASTE_END: &str = "\x1b[201~"; const SUBJECT_MAX_CHARS: usize = 160; const SENDER_MAX_CHARS: usize = 80; const SUPERVISOR_CHAIN_LIMIT: usize = 64; +/// The marker for a declared non-agent event source. A fixed st2-chosen literal — never +/// producer-supplied text — so the bounded-notice proofs are unaffected. +const SOURCE_MARKER: &str = "»"; const RECOVERY_POKE: &str = "[DING] unread st2 messages remain; check your inbox"; // Must exceed face607's bounded 0.5s delivery delay plus PTY/Node startup overhead; otherwise a // successful pane write is misreported as a timeout and retried, duplicating the owned payload. @@ -222,7 +225,11 @@ fn poke_text_with_resolver( ) -> String { let subject = normalize_field(msg.subject.as_deref(), "(no subject)", SUBJECT_MAX_CHARS); let from = normalize_field(msg.from.as_deref(), "unknown", SENDER_MAX_CHARS); - let marker = relationship_marker(resolver, this_host, recipient, msg.from.as_deref()); + let marker = if msg.stream.is_some() && msg.event_id.is_some() { + SOURCE_MARKER.to_string() + } else { + relationship_marker(resolver, this_host, recipient, msg.from.as_deref()) + }; format!( "[DING] {marker} {from}: {subject} [id:{}]", poke_id(&msg.filename) @@ -1177,6 +1184,9 @@ mod tests { tags: vec![], priority: None, idempotency_key: None, + stream: None, + event_id: None, + event_key: None, body: String::new(), } } @@ -3083,6 +3093,204 @@ Enter to select · ↑/↓ to navigate · Esc to cancel"; ); } + // ----------------------------------------------------------------------------------------- + // SPIKE — producer-side event supersede against staged payload ownership. + // + // Supersede is the one genuinely new semantic in + // `docs/vrs/.experiments/2026-08-20-pipes-event-model-differentiation.md`, and it archives an + // inbox item *from outside DING*, possibly while DING already owns that item's payload in a + // composer. These two tests are the seam: if either fails, the differentiated event model's + // main earned semantic is unsafe and the design collapses back to unification. + // ----------------------------------------------------------------------------------------- + + /// A real catalog with one agent-owned stream. + fn event_catalog() -> (tempfile::TempDir, PathBuf) { + let catalog = tempfile::tempdir().unwrap(); + declare_agent(catalog.path(), "hetz", "worker", None); + let declaration = catalog.path().join("hetz/worker/agent.kdl"); + let source = std::fs::read_to_string(&declaration).unwrap(); + std::fs::write( + declaration, + source.replacen("\n}\n", "\n stream \"gh-ci\" {}\n}\n", 1), + ) + .unwrap(); + let inbox = inbox_dir(&catalog.path().join("hetz").join("worker")); + (catalog, inbox) + } + + fn emit_ci(root: &Path, event_id: &str, supersede: bool) -> String { + crate::event::emit( + root, + "hetz", + "hetz.worker", + "gh-ci", + event_id, + Some("pr-42"), + Some(&format!("CI {event_id} on PR #42")), + event_id, + supersede, + ) + .unwrap() + .filename + } + + fn flush_in(root: &Path, pending: &mut VecDeque, poker: &dyn Poker) { + flush_pending( + DingContext { + catalog_root: root, + this_host: "hetz", + recipient: "hetz.worker", + }, + None, + pending, + poker, + ); + } + + /// The race: the producer supersedes event N *while DING owns N's staged payload*. The + /// existing ownership rules must carry it — N is pasted exactly once and never again, the + /// archived-and-not-retained head releases FIFO, and N+1 still delivers. Nothing about + /// `flush_pending` or `prune_archived_pending` changes to make this true. + #[test] + fn a_producer_supersede_of_a_staged_event_never_repastes_and_the_successor_delivers() { + let (catalog, inbox) = event_catalog(); + let root = catalog.path(); + + let failure = emit_ci(root, "failure", true); + let mut seen = HashSet::new(); + let mut pending: VecDeque = new_arrivals(&inbox, &mut seen) + .into_iter() + .map(PendingNotice::message) + .collect(); + let failure_text = pending[0].text( + DingContext { + catalog_root: root, + this_host: "hetz", + recipient: "hetz.worker", + }, + &mut None, + ); + assert!( + failure_text.starts_with("[DING] » hetz.worker/gh-ci:"), + "an event announces itself as a world-event: {failure_text}" + ); + + let poker = OwnershipPoker { + pokes: Mutex::new(Vec::new()), + retries: Mutex::new(Vec::new()), + poke_outcomes: Mutex::new(VecDeque::from([ + // the successor, once ownership of the superseded head is released + PokeOutcome::Delivered, + ])), + retry_outcomes: Mutex::new(VecDeque::from([PokeOutcome::NotRetained])), + }; + // DING stages the failure notice and owns it. + let stage_only = OwnershipPoker { + pokes: Mutex::new(Vec::new()), + retries: Mutex::new(Vec::new()), + poke_outcomes: Mutex::new(VecDeque::from([PokeOutcome::Staged])), + retry_outcomes: Mutex::new(VecDeque::new()), + }; + flush_in(root, &mut pending, &stage_only); + assert_eq!(pending[0].staged_text(), Some(failure_text.as_str())); + + // The producer now supersedes: `success` is materialized and `failure` is archived under + // DING's feet. + let success = emit_ci(root, "success", true); + assert!(!inbox.join(&failure).exists(), "the head was retired"); + assert!(inbox.join(&success).exists(), "the successor is unread"); + + pending.extend( + new_arrivals(&inbox, &mut seen) + .into_iter() + .map(PendingNotice::message), + ); + prune_archived_pending(&inbox, &mut pending); + assert_eq!( + pending.len(), + 2, + "the staged-but-archived head keeps ownership; the successor queues behind it" + ); + + flush_in(root, &mut pending, &poker); + + assert!(pending.is_empty(), "FIFO drained, nothing stuck"); + let success_text = poker.pokes.lock().unwrap()[0].clone(); + assert!( + success_text.contains("CI success on PR #42"), + "{success_text}" + ); + assert_eq!( + stage_only.pokes.lock().unwrap().as_slice(), + [failure_text.as_str()], + "the superseded notice was pasted exactly once, ever" + ); + assert_eq!( + poker.pokes.lock().unwrap().len(), + 1, + "and the only fresh paste after supersede is the successor" + ); + assert_eq!( + poker.retries.lock().unwrap().as_slice(), + [failure_text.as_str()], + "the superseded head was released by inspection only, never re-pasted" + ); + } + + /// The pessimistic half of the same race: the adapter still sees the superseded notice in the + /// composer. Ownership is retained, FIFO stays blocked behind it, and the successor is *not* + /// pasted on top of a live payload. Supersede therefore cannot leak a second paste into a + /// composer that is still holding the first. + #[test] + fn a_superseded_but_still_retained_staged_event_keeps_ownership_without_repasting() { + let (catalog, inbox) = event_catalog(); + let root = catalog.path(); + + emit_ci(root, "failure", true); + let mut seen = HashSet::new(); + let mut pending: VecDeque = new_arrivals(&inbox, &mut seen) + .into_iter() + .map(PendingNotice::message) + .collect(); + let failure_text = pending[0].text( + DingContext { + catalog_root: root, + this_host: "hetz", + recipient: "hetz.worker", + }, + &mut None, + ); + + let poker = OwnershipPoker { + pokes: Mutex::new(Vec::new()), + retries: Mutex::new(Vec::new()), + poke_outcomes: Mutex::new(VecDeque::from([PokeOutcome::Staged])), + retry_outcomes: Mutex::new(VecDeque::from([PokeOutcome::Staged])), + }; + flush_in(root, &mut pending, &poker); + + emit_ci(root, "success", true); + pending.extend( + new_arrivals(&inbox, &mut seen) + .into_iter() + .map(PendingNotice::message), + ); + prune_archived_pending(&inbox, &mut pending); + flush_in(root, &mut pending, &poker); + + assert_eq!(pending.len(), 2, "later FIFO work remains blocked"); + assert_eq!( + poker.pokes.lock().unwrap().as_slice(), + [failure_text.as_str()], + "the successor is never pasted on top of a retained payload" + ); + assert_eq!( + poker.retries.lock().unwrap().as_slice(), + [failure_text.as_str()], + "the retained superseded notice is retried by inspection only" + ); + } + #[test] fn archived_not_retained_releases_fifo_without_repasting_owned_notice() { let agent = tempfile::tempdir().unwrap(); @@ -3211,11 +3419,7 @@ Enter to select · ↑/↓ to navigate · Esc to cancel"; .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_millis(); - std::fs::write( - &status_path, - format!("dnd\nv1 {stale_ms}\n"), - ) - .unwrap(); + std::fs::write(&status_path, format!("dnd\nv1 {stale_ms}\n")).unwrap(); flush_without_catalog(Some(&status_path), &mut pending, &poker); assert!( pending.is_empty(), diff --git a/src/driver.rs b/src/driver.rs index d6501a97..fe09648b 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -49,10 +49,7 @@ fn expand_codex(driver: &CodexDriver, bus_id: &str) -> KdlDocument { provider.extend(["--model".to_string(), model.clone()]); } if let Some(effort) = &driver.effort { - provider.extend([ - "-c".to_string(), - format!("model_reasoning_effort={effort}"), - ]); + provider.extend(["-c".to_string(), format!("model_reasoning_effort={effort}")]); } provider.extend(driver.args.iter().cloned()); provider.push(driver.prompt.clone()); @@ -195,6 +192,7 @@ mod tests { delivery: None, driver: Some(driver), resources: Vec::new(), + streams: Vec::new(), tasks: Vec::new(), path: PathBuf::from("/catalog/agents/host/worker/agent.kdl"), } diff --git a/src/eval_run.rs b/src/eval_run.rs index 431e0990..53907a4d 100644 --- a/src/eval_run.rs +++ b/src/eval_run.rs @@ -6,8 +6,8 @@ use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; @@ -16,9 +16,9 @@ use crate::eval_spec::{ }; use crate::expand::expand_catalog; use crate::flapping::FlappingCap; -use crate::reconcile::{TaskCompileContext, compile_generated_tasks, reconcile}; #[cfg(test)] use crate::reconcile::compile_generated_ding_tasks; +use crate::reconcile::{TaskCompileContext, compile_generated_tasks, reconcile}; use crate::run::{Runner, SystemRunner, UpReport, detect_host, execute}; use agent_spec::spec::{AgentDesiredState, AgentSpec, JobType, Task, TaskKind, TaskLifecycle}; @@ -39,17 +39,25 @@ fn install_eval_signal_handlers() -> (libc::sighandler_t, libc::sighandler_t) { // libc exposes sighandler_t as a numeric ABI token on some targets. #[allow(clippy::fn_to_numeric_cast, function_casts_as_integer)] unsafe { - (libc::signal(libc::SIGINT, on_eval_signal as libc::sighandler_t), libc::signal(libc::SIGTERM, on_eval_signal as libc::sighandler_t)) + ( + libc::signal(libc::SIGINT, on_eval_signal as libc::sighandler_t), + libc::signal(libc::SIGTERM, on_eval_signal as libc::sighandler_t), + ) } } fn restore_eval_signal_handlers(previous: (libc::sighandler_t, libc::sighandler_t)) { - unsafe { libc::signal(libc::SIGINT, previous.0); libc::signal(libc::SIGTERM, previous.1); } + unsafe { + libc::signal(libc::SIGINT, previous.0); + libc::signal(libc::SIGTERM, previous.1); + } } struct EvalSignalGuard((libc::sighandler_t, libc::sighandler_t)); impl Drop for EvalSignalGuard { - fn drop(&mut self) { restore_eval_signal_handlers(self.0); } + fn drop(&mut self) { + restore_eval_signal_handlers(self.0); + } } /// Map a parsed spec's agents into in-memory [`AgentSpec`]s rooted at `root` (which becomes `$CATALOG` @@ -122,6 +130,7 @@ pub fn spec_to_agent_specs(agents: &[SpecAgent], host: &str, root: &Path) -> Vec delivery: None, driver: None, resources: Vec::new(), + streams: Vec::new(), tasks, path: path.clone(), } @@ -302,8 +311,7 @@ fn load_canonical_eval_team(catalog: &Path, host: &str) -> Result Option { /// prefix wholesale costs nothing. fn sanitize_agent_env() { let should_strip = |k: &str| { - matches!(k, "CLAUDECODE" | "CLAUDE_PID" | "CLAUDE_EFFORT" | "AI_AGENT") - || k.starts_with("CLAUDE_CODE_") + matches!( + k, + "CLAUDECODE" | "CLAUDE_PID" | "CLAUDE_EFFORT" | "AI_AGENT" + ) || k.starts_with("CLAUDE_CODE_") || k.starts_with("CODEX_") || k.starts_with("PI_") || k.starts_with("ST2_PI_CHANNEL_") @@ -456,7 +466,11 @@ pub fn resolve_spec_path(path: &Path) -> Option { .collect(); if let [one] = kdls.as_slice() { // Confirm it parses as a spec (vs a stray .kdl in a catalog dir). - if std::fs::read_to_string(one).ok().and_then(|t| parse_spec(&t).ok()).is_some() { + if std::fs::read_to_string(one) + .ok() + .and_then(|t| parse_spec(&t).ok()) + .is_some() + { return Some(one.clone()); } } @@ -535,12 +549,17 @@ pub fn copy_tree(src: &Path, dst: &Path) -> Result<()> { let entry = entry?; let from = entry.path(); let name = entry.file_name(); - let dst_name = if name == "_git" { std::ffi::OsString::from(".git") } else { name }; + let dst_name = if name == "_git" { + std::ffi::OsString::from(".git") + } else { + name + }; let to = dst.join(&dst_name); if entry.file_type()?.is_dir() { copy_tree(&from, &to)?; } else { - std::fs::copy(&from, &to).with_context(|| format!("copy {} → {}", from.display(), to.display()))?; + std::fs::copy(&from, &to) + .with_context(|| format!("copy {} → {}", from.display(), to.display()))?; } } Ok(()) @@ -559,7 +578,8 @@ fn bus_root(spec: &Spec, catalog: &Path) -> PathBuf { fn resolve_content(content: &str, spec_dir: &Path) -> Result { let candidate = spec_dir.join(content); if candidate.is_file() { - std::fs::read_to_string(&candidate).with_context(|| format!("reading kickoff content {}", candidate.display())) + std::fs::read_to_string(&candidate) + .with_context(|| format!("reading kickoff content {}", candidate.display())) } else { Ok(content.to_string()) } @@ -592,10 +612,7 @@ fn wait_done( let route = admitted_route(routes, sup); (route.inbox.clone(), route.archive.clone()) } - None => ( - bus.join(sup).join("inbox"), - bus.join(sup).join("archive"), - ), + None => (bus.join(sup).join("inbox"), bus.join(sup).join("archive")), }; // The requester is eval-owned, not an admitted Agent Spec, and deliberately keeps one explicit // flat mailbox. Every canonical agent route above comes from the frozen admitted vector. @@ -665,8 +682,16 @@ fn boot_gate(task_ids: &[String], specs: &[AgentSpec], host: &str, catalog: &Pat let deadline = Instant::now() + Duration::from_secs(5); loop { let sessions = runner.list_sessions().unwrap_or_default(); - let alive: HashSet<&str> = sessions.iter().filter(|s| s.alive).map(|s| s.pty_id.as_str()).collect(); - let dead: Vec<&str> = want.iter().copied().filter(|id| !alive.contains(id)).collect(); + let alive: HashSet<&str> = sessions + .iter() + .filter(|s| s.alive) + .map(|s| s.pty_id.as_str()) + .collect(); + let dead: Vec<&str> = want + .iter() + .copied() + .filter(|id| !alive.contains(id)) + .collect(); if dead.is_empty() { return Ok(()); } @@ -723,12 +748,7 @@ fn message_timestamp(filename: &str) -> Result { /// team-standup specialist the CoS spun up mid-run). Declared teardown only reaps declared tasks, so /// a runtime task would leak as an orphan; since the PTY_ROOT is hermetic to this eval, anything still /// alive is ours to clean. Killing an already-dead declared session is a harmless no-op. -fn teardown_team_with_runner( - specs: &[AgentSpec], - host: &str, - runner: &dyn Runner, - reap_all: bool, -) { +fn teardown_team_with_runner(specs: &[AgentSpec], host: &str, runner: &dyn Runner, reap_all: bool) { let retired: Vec = specs .iter() .cloned() @@ -744,9 +764,7 @@ fn teardown_team_with_runner( let mut cap = FlappingCap::default(); execute(&plan, runner, &mut cap, &mut report); } - if reap_all - && let Ok(remaining) = runner.list_sessions() - { + if reap_all && let Ok(remaining) = runner.list_sessions() { for s in &remaining { let _ = runner.kill(&s.pty_id); let _ = runner.remove(&s.pty_id); @@ -767,17 +785,27 @@ pub fn run_eval(spec_file: &Path, host: Option, keep: bool) -> Result(runner: &R, host: &str) -> Resu let _host = host; let mut last_error = None; for _ in 0..5 { - let sessions = runner.list_sessions().with_context(|| format!("listing eval sessions for host {host}"))?; - if sessions.is_empty() { return Ok(()); } + let sessions = runner + .list_sessions() + .with_context(|| format!("listing eval sessions for host {host}"))?; + if sessions.is_empty() { + return Ok(()); + } for session in sessions { - if session.alive && let Err(error) = runner.kill(&session.pty_id) { last_error = Some(format!("kill {}: {error:#}", session.pty_id)); } - if let Err(error) = runner.remove(&session.pty_id) { last_error = Some(format!("remove {}: {error:#}", session.pty_id)); } + if session.alive + && let Err(error) = runner.kill(&session.pty_id) + { + last_error = Some(format!("kill {}: {error:#}", session.pty_id)); + } + if let Err(error) = runner.remove(&session.pty_id) { + last_error = Some(format!("remove {}: {error:#}", session.pty_id)); + } } std::thread::sleep(Duration::from_millis(20)); } - anyhow::bail!("eval session reap did not reach empty state on host {host}; last error: {}", last_error.unwrap_or_else(|| "none".into())) + anyhow::bail!( + "eval session reap did not reach empty state on host {host}; last error: {}", + last_error.unwrap_or_else(|| "none".into()) + ) } fn reap_all_eval_sessions(catalog: &Path, host: &str) -> Result<()> { @@ -835,15 +876,25 @@ fn reap_all_eval_sessions(catalog: &Path, host: &str) -> Result<()> { /// Idempotent safety net for eval catalog lifetime. Normal teardown remains responsible for /// sessions; this guard ensures an unwind cannot strand the hermetic catalog on disk. -struct EvalCleanupGuard { runner: R, catalog: PathBuf, host: String, keep: bool } +struct EvalCleanupGuard { + runner: R, + catalog: PathBuf, + host: String, + keep: bool, +} impl Drop for EvalCleanupGuard { fn drop(&mut self) { let reap = reap_all_eval_sessions_with_runner(&self.runner, &self.host); if let Err(error) = reap { - eprintln!("st2 eval cleanup: {error:#}; preserving catalog {}", self.catalog.display()); + eprintln!( + "st2 eval cleanup: {error:#}; preserving catalog {}", + self.catalog.display() + ); return; } - if !self.keep { let _ = std::fs::remove_dir_all(&self.catalog); } + if !self.keep { + let _ = std::fs::remove_dir_all(&self.catalog); + } } } @@ -889,8 +940,11 @@ fn run_steps( Some(w) => catalog.join(expand_catalog(w, catalog)), None => catalog.to_path_buf(), }; - let (attempts, backoff) = - step.retry.as_ref().map(|r| (r.attempts.max(1), r.delay)).unwrap_or((1, Duration::ZERO)); + let (attempts, backoff) = step + .retry + .as_ref() + .map(|r| (r.attempts.max(1), r.delay)) + .unwrap_or((1, Duration::ZERO)); let mut exit = -1; let (mut out, mut err) = (Vec::new(), Vec::new()); @@ -934,7 +988,16 @@ fn run_steps( combined.extend_from_slice(&err); let _ = std::fs::write(logs_dir.join(format!("{}.log", step.id)), &combined); runtime.insert(format!("RUN_{}_EXIT", env_key(&step.id)), exit.to_string()); - eval_log!("== run step {} → exit {}{} ==", step.id, exit, if step.allow_nonzero { " (allow-nonzero)" } else { "" }); + eval_log!( + "== run step {} → exit {}{} ==", + step.id, + exit, + if step.allow_nonzero { + " (allow-nonzero)" + } else { + "" + } + ); if !step.allow_nonzero { // Default: a run step must succeed. A non-zero final exit hard-fails the verdict as a @@ -979,7 +1042,9 @@ fn dump_agent_logs(pty_task_ids: &[String], catalog: &Path) { /// An env-var-safe form of a step id (non-alphanumerics → `_`), for `RUN__EXIT`. fn env_key(id: &str) -> String { - id.chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }).collect() + id.chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect() } /// The supervisor chain of `agent_id`, walked transitively via each agent's `supervisor` field to the @@ -1048,7 +1113,11 @@ fn run_eval_inner( } let bus = bus_root(spec, catalog); - let requester = eval.message.as_ref().map(|m| m.from.clone()).unwrap_or_else(|| "eval-runner".to_string()); + let requester = eval + .message + .as_ref() + .map(|m| m.from.clone()) + .unwrap_or_else(|| "eval-runner".to_string()); // The run{} stage runs to completion BEFORE judging — the WHOLE work of a team-less eval, or setup // before a team. must-exit-0 failures come back as failing synthetic judge results (folded into @@ -1063,7 +1132,10 @@ fn run_eval_inner( let (done, specs, pty_task_ids) = if compact_agents.is_empty() && !eval.canonical_agents { // TEAM-LESS: nothing to boot, kick off, or wait on — the run steps did the work → straight to judging. if !eval.run_steps.is_empty() { - eval_log!("== team-less eval: {} run step(s) ran → judging ==", eval.run_steps.len()); + eval_log!( + "== team-less eval: {} run step(s) ran → judging ==", + eval.run_steps.len() + ); } (true, Vec::new(), Vec::new()) } else { @@ -1081,11 +1153,7 @@ fn run_eval_inner( .iter() .map(|spec| spec.bus_id(host)) .collect::>(); - ( - team.specs, - participants, - Some(team.routes), - ) + (team.specs, participants, Some(team.routes)) } else { let specs = spec_to_agent_specs(&compact_agents, host, catalog); let participants = specs @@ -1179,7 +1247,7 @@ fn run_eval_inner( }); let kickoff_receipt = crate::message::send_to_inbox(&to_inbox, &msg.from, None, None, &[], &body) - .with_context(|| format!("seeding kickoff into {}", to_inbox.display()))?; + .with_context(|| format!("seeding kickoff into {}", to_inbox.display()))?; let kickoff_ts = eval .canonical_agents .then(|| message_timestamp(&kickoff_receipt)) @@ -1192,7 +1260,8 @@ fn run_eval_inner( .collect(); eval_log!( "== waiting for {sup}→{} confirmation post-dating a worker report (≤{:?}) ==", - msg.from, eval.max_timeout + msg.from, + eval.max_timeout ); // `supervise`: each wait tick, respawn any dead task FROM SPEC (full env → rejoins cold). Carry a @@ -1220,10 +1289,8 @@ fn run_eval_inner( // finish is as bad as a missed crash). let report = match supervised_eval_sessions(&specs, host, &supervise_runner) { Ok(sessions) => { - let by_id: std::collections::HashMap< - &str, - &crate::reconcile::Session, - > = sessions.iter().map(|s| (s.pty_id.as_str(), s)).collect(); + let by_id: std::collections::HashMap<&str, &crate::reconcile::Session> = + sessions.iter().map(|s| (s.pty_id.as_str(), s)).collect(); for task in &runtime_tasks { let id = task.runtime_id.as_str(); match by_id.get(id) { @@ -1232,12 +1299,9 @@ fn run_eval_inner( dinged.remove(id); // healthy again → re-arm for a future crash } found => { - let clean = - matches!(found, Some(s) if s.exit_code == Some(0)) - || eval_exit_code(catalog, id) == Some(0); - if ever_alive.contains(id) - && !clean - && !dinged.contains(id) + let clean = matches!(found, Some(s) if s.exit_code == Some(0)) + || eval_exit_code(catalog, id) == Some(0); + if ever_alive.contains(id) && !clean && !dinged.contains(id) { crash_ding( &task.agent_id, @@ -1277,7 +1341,10 @@ fn run_eval_inner( eval_log!("== supervise: launched {:?} from spec ==", report.launched); } if !report.restarted.is_empty() { - eval_log!("== supervise: restarted {:?} from spec ==", report.restarted); + eval_log!( + "== supervise: restarted {:?} from spec ==", + report.restarted + ); } } }; @@ -1301,7 +1368,10 @@ fn run_eval_inner( if done { eval_log!("== team signalled done — judging =="); } else { - eval_log!("== max-timeout: no confirmation within {:?} — judging the final state ==", eval.max_timeout); + eval_log!( + "== max-timeout: no confirmation within {:?} — judging the final state ==", + eval.max_timeout + ); } (done, specs, pty_task_ids) }; @@ -1326,10 +1396,21 @@ fn run_eval_inner( // Judges: the run-step gate results first, then the declared judges (all must pass). Judge BEFORE // teardown — an ask-agent judge needs its judge agent still alive to answer. - judges.extend(run_judges(&eval.judges, spec_dir, catalog, &bus, &requester, &run_env)); + judges.extend(run_judges( + &eval.judges, + spec_dir, + catalog, + &bus, + &requester, + &run_env, + )); // Under `supervise`, reap runtime-spawned tasks too (team-standup), not just the declared team. teardown_team(&specs, host, catalog, eval.supervise); - Ok(EvalReport { done, judges, timeout: eval.max_timeout }) + Ok(EvalReport { + done, + judges, + timeout: eval.max_timeout, + }) } // ── P4: the judge engine (all-must-pass; declarative / bash / ask-agent; per-judge timeout) ───────── @@ -1352,10 +1433,19 @@ pub fn run_judges( let timeout = j.timeout.unwrap_or(default_timeout); let (passed, detail) = match &j.kind { JudgeKind::Declarative(checks) => run_declarative(checks, catalog), - JudgeKind::Bash(cmd) => run_bash_judge(cmd, spec_dir, catalog, bus, timeout, run_env), - JudgeKind::Ask { agent, prompt } => run_ask_judge(agent, prompt, bus, requester, timeout), + JudgeKind::Bash(cmd) => { + run_bash_judge(cmd, spec_dir, catalog, bus, timeout, run_env) + } + JudgeKind::Ask { agent, prompt } => { + run_ask_judge(agent, prompt, bus, requester, timeout) + } }; - JudgeResult { name: j.name.clone(), passed, detail, signal: j.signal } + JudgeResult { + name: j.name.clone(), + passed, + detail, + signal: j.signal, + } }) .collect() } @@ -1370,12 +1460,27 @@ fn run_declarative(checks: &[Check], catalog: &Path) -> (bool, String) { } Check::FileLacks { path, text } => { let body = std::fs::read_to_string(catalog.join(path)).unwrap_or_default(); - (!body.contains(text.as_str()), format!("{path} lacks {text:?}")) + ( + !body.contains(text.as_str()), + format!("{path} lacks {text:?}"), + ) } Check::JsonField { path, field, value } => { - let expected = match value { crate::eval_spec::JsonScalar::String(s) => serde_json::Value::String(s.clone()), crate::eval_spec::JsonScalar::Bool(b) => serde_json::Value::Bool(*b), crate::eval_spec::JsonScalar::Integer(i) => serde_json::Value::Number((*i).into()) }; - let got = std::fs::read_to_string(catalog.join(path)).ok().and_then(|t| serde_json::from_str::(&t).ok()).and_then(|v| v.get(field).cloned()); - (got.as_ref() == Some(&expected), format!("{path} field {field} is {value:?} (got {got:?})")) + let expected = match value { + crate::eval_spec::JsonScalar::String(s) => serde_json::Value::String(s.clone()), + crate::eval_spec::JsonScalar::Bool(b) => serde_json::Value::Bool(*b), + crate::eval_spec::JsonScalar::Integer(i) => { + serde_json::Value::Number((*i).into()) + } + }; + let got = std::fs::read_to_string(catalog.join(path)) + .ok() + .and_then(|t| serde_json::from_str::(&t).ok()) + .and_then(|v| v.get(field).cloned()); + ( + got.as_ref() == Some(&expected), + format!("{path} field {field} is {value:?} (got {got:?})"), + ) } Check::Committed { path } => { let ok = is_committed_clean(catalog, path); @@ -1398,11 +1503,21 @@ fn is_committed_clean(catalog: &Path, path: &str) -> bool { if dir.join(".git").exists() { let rel = target.strip_prefix(dir).unwrap_or(&target); let tracked = std::process::Command::new("git") - .arg("-C").arg(dir).args(["ls-files", "--error-unmatch"]).arg(rel) - .output().map(|o| o.status.success()).unwrap_or(false); + .arg("-C") + .arg(dir) + .args(["ls-files", "--error-unmatch"]) + .arg(rel) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); let clean = std::process::Command::new("git") - .arg("-C").arg(dir).args(["status", "--porcelain", "--"]).arg(rel) - .output().map(|o| o.stdout.is_empty()).unwrap_or(false); + .arg("-C") + .arg(dir) + .args(["status", "--porcelain", "--"]) + .arg(rel) + .output() + .map(|o| o.stdout.is_empty()) + .unwrap_or(false); return tracked && clean; } repo = dir.parent(); @@ -1426,8 +1541,9 @@ fn run_bash_judge( // `sh` reports the physical cwd on macOS (for example `/private/var/...`) even when tempfile // handed us its symlinked spelling (`/var/...`). Export the same physical path so `$SPEC_DIR` // remains a truthful explicit name for the judge's cwd on every platform. - let physical_spec_dir = - spec_dir.canonicalize().unwrap_or_else(|_| spec_dir.to_path_buf()); + let physical_spec_dir = spec_dir + .canonicalize() + .unwrap_or_else(|_| spec_dir.to_path_buf()); let mut command = Command::new("sh"); command .arg("-c") @@ -1450,7 +1566,12 @@ fn run_bash_judge( let deadline = Instant::now() + timeout; loop { match child.try_wait() { - Ok(Some(status)) => return (status.success(), format!("exit {}", status.code().unwrap_or(-1))), + Ok(Some(status)) => { + return ( + status.success(), + format!("exit {}", status.code().unwrap_or(-1)), + ); + } Ok(None) => { if Instant::now() > deadline { let _ = child.kill(); @@ -1465,10 +1586,18 @@ fn run_bash_judge( /// An ask-agent judge: message the judge agent the prompt, wait for its reply (post the ask) in the /// requester's inbox within the timeout, and read PASS/FAIL out of it. -fn run_ask_judge(agent: &str, prompt: &str, bus: &Path, requester: &str, timeout: Duration) -> (bool, String) { +fn run_ask_judge( + agent: &str, + prompt: &str, + bus: &Path, + requester: &str, + timeout: Duration, +) -> (bool, String) { let ask_ts = now_ms(); let to_inbox = bus.join(agent).join("inbox"); - if let Err(e) = crate::message::send_to_inbox(&to_inbox, requester, Some("judge"), None, &[], prompt) { + if let Err(e) = + crate::message::send_to_inbox(&to_inbox, requester, Some("judge"), None, &[], prompt) + { return (false, format!("could not ask judge '{agent}': {e}")); } let req_inbox = bus.join(requester).join("inbox"); @@ -1488,7 +1617,10 @@ fn run_ask_judge(agent: &str, prompt: &str, bus: &Path, requester: &str, timeout }; } if Instant::now() > deadline { - return (false, format!("judge '{agent}' did not reply within {timeout:?}")); + return ( + false, + format!("judge '{agent}' did not reply within {timeout:?}"), + ); } std::thread::sleep(Duration::from_millis(500)); } @@ -1566,7 +1698,11 @@ mod tests { .collect::>(), ["evalhost.sup", "evalhost.worker"] ); - assert!(team.specs.iter().all(|spec| spec.path.ends_with("agent.kdl"))); + assert!( + team.specs + .iter() + .all(|spec| spec.path.ends_with("agent.kdl")) + ); } #[test] @@ -1843,12 +1979,25 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } ); } - struct RaceRunner { lists: RefCell>>, ops: RefCell> } + struct RaceRunner { + lists: RefCell>>, + ops: RefCell>, + } impl Runner for RaceRunner { - fn list_sessions(&self) -> anyhow::Result> { Ok(self.lists.borrow_mut().remove(0)) } - fn spawn(&self, _: &TaskTarget, _: &Path) -> anyhow::Result<()> { Ok(()) } - fn kill(&self, id: &str) -> anyhow::Result<()> { self.ops.borrow_mut().push(format!("kill:{id}")); anyhow::bail!("already gone") } - fn remove(&self, id: &str) -> anyhow::Result<()> { self.ops.borrow_mut().push(format!("remove:{id}")); anyhow::bail!("already gone") } + fn list_sessions(&self) -> anyhow::Result> { + Ok(self.lists.borrow_mut().remove(0)) + } + fn spawn(&self, _: &TaskTarget, _: &Path) -> anyhow::Result<()> { + Ok(()) + } + fn kill(&self, id: &str) -> anyhow::Result<()> { + self.ops.borrow_mut().push(format!("kill:{id}")); + anyhow::bail!("already gone") + } + fn remove(&self, id: &str) -> anyhow::Result<()> { + self.ops.borrow_mut().push(format!("remove:{id}")); + anyhow::bail!("already gone") + } } struct InventoryRunner { @@ -1910,23 +2059,58 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } #[test] fn reap_race_errors_converge_only_after_empty_list() { - let runner = RaceRunner { lists: RefCell::new(vec![vec![Session { pty_id: "x".into(), alive: true, exit_code: None, presentation: None }], vec![]]), ops: RefCell::new(Vec::new()) }; + let runner = RaceRunner { + lists: RefCell::new(vec![ + vec![Session { + pty_id: "x".into(), + alive: true, + exit_code: None, + presentation: None, + }], + vec![], + ]), + ops: RefCell::new(Vec::new()), + }; assert!(reap_all_eval_sessions_with_runner(&runner, "test").is_ok()); assert_eq!(runner.ops.borrow().len(), 2); } - struct PersistentRunner { lists: RefCell, ops: RefCell } + struct PersistentRunner { + lists: RefCell, + ops: RefCell, + } impl Runner for PersistentRunner { - fn list_sessions(&self) -> anyhow::Result> { *self.lists.borrow_mut() += 1; Ok(vec![Session { pty_id: "stuck".into(), alive: true, exit_code: None, presentation: None }]) } - fn spawn(&self, _: &TaskTarget, _: &Path) -> anyhow::Result<()> { Ok(()) } - fn kill(&self, _: &str) -> anyhow::Result<()> { *self.ops.borrow_mut() += 1; Ok(()) } - fn remove(&self, _: &str) -> anyhow::Result<()> { *self.ops.borrow_mut() += 1; Ok(()) } + fn list_sessions(&self) -> anyhow::Result> { + *self.lists.borrow_mut() += 1; + Ok(vec![Session { + pty_id: "stuck".into(), + alive: true, + exit_code: None, + presentation: None, + }]) + } + fn spawn(&self, _: &TaskTarget, _: &Path) -> anyhow::Result<()> { + Ok(()) + } + fn kill(&self, _: &str) -> anyhow::Result<()> { + *self.ops.borrow_mut() += 1; + Ok(()) + } + fn remove(&self, _: &str) -> anyhow::Result<()> { + *self.ops.borrow_mut() += 1; + Ok(()) + } } #[test] fn reap_persistent_residual_fails_after_bounded_attempts() { - let runner = PersistentRunner { lists: RefCell::new(0), ops: RefCell::new(0) }; - let error = reap_all_eval_sessions_with_runner(&runner, "host-x").unwrap_err().to_string(); + let runner = PersistentRunner { + lists: RefCell::new(0), + ops: RefCell::new(0), + }; + let error = reap_all_eval_sessions_with_runner(&runner, "host-x") + .unwrap_err() + .to_string(); assert!(error.contains("host-x") && error.contains("empty state")); assert_eq!(*runner.lists.borrow(), 5); assert_eq!(*runner.ops.borrow(), 10); @@ -1935,18 +2119,47 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } #[test] fn cleanup_guard_reaps_on_unwind_without_double_panic() { use std::rc::Rc; - let lists = Rc::new(RefCell::new(vec![vec![Session { pty_id: "panic".into(), alive: true, exit_code: None, presentation: None }], vec![]])); + let lists = Rc::new(RefCell::new(vec![ + vec![Session { + pty_id: "panic".into(), + alive: true, + exit_code: None, + presentation: None, + }], + vec![], + ])); let ops = Rc::new(RefCell::new(Vec::new())); - struct Shared { lists: Rc>>>, ops: Rc>> } + struct Shared { + lists: Rc>>>, + ops: Rc>>, + } impl Runner for Shared { - fn list_sessions(&self) -> anyhow::Result> { Ok(self.lists.borrow_mut().remove(0)) } - fn spawn(&self, _: &TaskTarget, _: &Path) -> anyhow::Result<()> { Ok(()) } - fn kill(&self, id: &str) -> anyhow::Result<()> { self.ops.borrow_mut().push(format!("kill:{id}")); Ok(()) } - fn remove(&self, id: &str) -> anyhow::Result<()> { self.ops.borrow_mut().push(format!("remove:{id}")); Ok(()) } + fn list_sessions(&self) -> anyhow::Result> { + Ok(self.lists.borrow_mut().remove(0)) + } + fn spawn(&self, _: &TaskTarget, _: &Path) -> anyhow::Result<()> { + Ok(()) + } + fn kill(&self, id: &str) -> anyhow::Result<()> { + self.ops.borrow_mut().push(format!("kill:{id}")); + Ok(()) + } + fn remove(&self, id: &str) -> anyhow::Result<()> { + self.ops.borrow_mut().push(format!("remove:{id}")); + Ok(()) + } } - let runner = Shared { lists: lists.clone(), ops: ops.clone() }; + let runner = Shared { + lists: lists.clone(), + ops: ops.clone(), + }; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _guard = EvalCleanupGuard { runner, catalog: std::env::temp_dir().join("st2-test-panic"), host: "test".into(), keep: true }; + let _guard = EvalCleanupGuard { + runner, + catalog: std::env::temp_dir().join("st2-test-panic"), + host: "test".into(), + keep: true, + }; panic!("panic after guard acquisition"); })); assert!(result.is_err()); @@ -1957,14 +2170,46 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } #[test] fn cleanup_guard_catalog_lifetime_matrix() { for keep in [false, true] { - let dir = tempfile::tempdir().unwrap(); let catalog = dir.path().join("catalog"); std::fs::create_dir_all(&catalog).unwrap(); - let runner = RaceRunner { lists: RefCell::new(vec![vec![Session { pty_id: "x".into(), alive: true, exit_code: None, presentation: None }], vec![]]), ops: RefCell::new(Vec::new()) }; - { let _guard = EvalCleanupGuard { runner, catalog: catalog.clone(), host: "test".into(), keep }; } + let dir = tempfile::tempdir().unwrap(); + let catalog = dir.path().join("catalog"); + std::fs::create_dir_all(&catalog).unwrap(); + let runner = RaceRunner { + lists: RefCell::new(vec![ + vec![Session { + pty_id: "x".into(), + alive: true, + exit_code: None, + presentation: None, + }], + vec![], + ]), + ops: RefCell::new(Vec::new()), + }; + { + let _guard = EvalCleanupGuard { + runner, + catalog: catalog.clone(), + host: "test".into(), + keep, + }; + } assert_eq!(catalog.exists(), keep); } - let dir = tempfile::tempdir().unwrap(); let catalog = dir.path().join("catalog"); std::fs::create_dir_all(&catalog).unwrap(); - let runner = PersistentRunner { lists: RefCell::new(0), ops: RefCell::new(0) }; - { let _guard = EvalCleanupGuard { runner, catalog: catalog.clone(), host: "test".into(), keep: false }; } + let dir = tempfile::tempdir().unwrap(); + let catalog = dir.path().join("catalog"); + std::fs::create_dir_all(&catalog).unwrap(); + let runner = PersistentRunner { + lists: RefCell::new(0), + ops: RefCell::new(0), + }; + { + let _guard = EvalCleanupGuard { + runner, + catalog: catalog.clone(), + host: "test".into(), + keep: false, + }; + } assert!(catalog.exists()); } @@ -1976,7 +2221,11 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } detail: String::new(), signal, }; - let report = |judges: Vec| EvalReport { done: true, judges, timeout: Duration::ZERO }; + let report = |judges: Vec| EvalReport { + done: true, + judges, + timeout: Duration::ZERO, + }; // A FAILING signal judge does NOT gate a passing gating judge. assert!(report(vec![j("gate", true, false), j("sig", false, true)]).passed()); // A failing GATING judge does gate. @@ -2067,9 +2316,7 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } add_eval_exit_markers(&mut specs, catalog.path(), "evalhost"); let command = specs[0].tasks[0].command.as_deref().unwrap(); - let canonical = catalog - .path() - .join(".eval-exits/evalhost.worker.status"); + let canonical = catalog.path().join(".eval-exits/evalhost.worker.status"); let flat = catalog.path().join(".eval-exits/worker.status"); assert!(command.contains(&canonical.display().to_string())); assert!(!command.contains(&flat.display().to_string())); @@ -2286,7 +2533,8 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } .iter() .find(|task| task.derived) .unwrap(); - let compact_runtime = task_runtime_id(&compact_specs[0], &compact_specs[0].tasks[0], "host"); + let compact_runtime = + task_runtime_id(&compact_specs[0], &compact_specs[0].tasks[0], "host"); let eval_runtime = task_runtime_id(&eval_specs[0], &eval_specs[0].tasks[0], "host"); assert_eq!(compact_ding.command, eval_ding.command); @@ -2299,7 +2547,11 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } fn seed_msg(inbox: &Path, ts: u64, rand: &str, from: &str) { std::fs::create_dir_all(inbox).unwrap(); - std::fs::write(inbox.join(format!("{ts:013}-{rand}.md")), format!("---\nfrom: {from}\n---\nbody\n")).unwrap(); + std::fs::write( + inbox.join(format!("{ts:013}-{rand}.md")), + format!("---\nfrom: {from}\n---\nbody\n"), + ) + .unwrap(); } #[test] @@ -2307,20 +2559,39 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } let src = tempfile::tempdir().unwrap(); let dst = tempfile::tempdir().unwrap(); std::fs::create_dir_all(src.path().join("worker/_git")).unwrap(); - std::fs::write(src.path().join("worker/_git/HEAD"), "ref: refs/heads/main\n").unwrap(); + std::fs::write( + src.path().join("worker/_git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); std::fs::write(src.path().join("worker/LICENSE"), "proprietary\n").unwrap(); copy_tree(src.path(), dst.path()).unwrap(); - assert!(dst.path().join("worker/.git/HEAD").is_file(), ".git materialized from _git"); - assert!(!dst.path().join("worker/_git").exists(), "_git renamed away"); - assert_eq!(std::fs::read_to_string(dst.path().join("worker/LICENSE")).unwrap(), "proprietary\n"); + assert!( + dst.path().join("worker/.git/HEAD").is_file(), + ".git materialized from _git" + ); + assert!( + !dst.path().join("worker/_git").exists(), + "_git renamed away" + ); + assert_eq!( + std::fs::read_to_string(dst.path().join("worker/LICENSE")).unwrap(), + "proprietary\n" + ); } #[test] fn resolve_content_reads_a_file_else_inline() { let dir = tempfile::tempdir().unwrap(); std::fs::write(dir.path().join("task.md"), "the task\n").unwrap(); - assert_eq!(resolve_content("./task.md", dir.path()).unwrap(), "the task\n"); - assert_eq!(resolve_content("just do it inline", dir.path()).unwrap(), "just do it inline"); + assert_eq!( + resolve_content("./task.md", dir.path()).unwrap(), + "the task\n" + ); + assert_eq!( + resolve_content("just do it inline", dir.path()).unwrap(), + "just do it inline" + ); } #[test] @@ -2344,8 +2615,18 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } )); // A worker→sup report at t=2000 + a sup→requester confirm that PRE-dates it (t=1000) → false // (the early "on it" ack the discriminator exists to reject). - seed_msg(&root.join(sup).join("inbox"), 1_700_000_002_000, "aaaaaa", "mix.worker"); - seed_msg(&root.join(req).join("inbox"), 1_700_000_001_000, "bbbbbb", "mix.sup"); + seed_msg( + &root.join(sup).join("inbox"), + 1_700_000_002_000, + "aaaaaa", + "mix.worker", + ); + seed_msg( + &root.join(req).join("inbox"), + 1_700_000_001_000, + "bbbbbb", + "mix.sup", + ); assert!(!wait_done( root, None, @@ -2358,7 +2639,12 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } noop, )); // A confirm that POST-dates the report (t=3000) → done. - seed_msg(&root.join(req).join("inbox"), 1_700_000_003_000, "cccccc", "mix.sup"); + seed_msg( + &root.join(req).join("inbox"), + 1_700_000_003_000, + "cccccc", + "mix.sup", + ); assert!(wait_done( root, None, @@ -2384,8 +2670,18 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } let noop = &mut (|| {}) as &mut dyn FnMut(); // The report is ONLY in the sup's archive (inbox is empty — the sup archived on-act); the confirm // post-dates it in the requester's inbox → the loop closed → done must fire. - seed_msg(&root.join(sup).join("archive"), 1_700_000_002_000, "aaaaaa", "mix.worker"); - seed_msg(&root.join(req).join("inbox"), 1_700_000_003_000, "cccccc", "mix.sup"); + seed_msg( + &root.join(sup).join("archive"), + 1_700_000_002_000, + "aaaaaa", + "mix.worker", + ); + seed_msg( + &root.join(req).join("inbox"), + 1_700_000_003_000, + "cccccc", + "mix.sup", + ); assert!( wait_done( root, @@ -2422,7 +2718,10 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } &mut tick, ); assert!(!fired, "no confirmation was seeded → must time out"); - assert!(ticks.get() >= 1, "the supervise tick must run during the wait"); + assert!( + ticks.get() >= 1, + "the supervise tick must run during the wait" + ); } #[test] @@ -2514,8 +2813,12 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } #[test] fn bus_root_expands_st_root_else_defaults() { - let s = parse_spec("env { ST_ROOT \"$CATALOG/bus\" }\nagent \"a\" { command \"run\" }").unwrap(); - assert_eq!(bus_root(&s, Path::new("/tmp/cat")), PathBuf::from("/tmp/cat/bus")); + let s = parse_spec("env { ST_ROOT \"$CATALOG/bus\" }\nagent \"a\" { command \"run\" }") + .unwrap(); + assert_eq!( + bus_root(&s, Path::new("/tmp/cat")), + PathBuf::from("/tmp/cat/bus") + ); let s2 = parse_spec(r#"agent "a" { command "run" }"#).unwrap(); assert_eq!( bus_root(&s2, Path::new("/tmp/cat")), @@ -2528,7 +2831,10 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } fn parse_pass_fail_takes_the_first_token() { assert_eq!(parse_pass_fail("PASS — cites the real commit"), Some(true)); assert_eq!(parse_pass_fail("FAIL — vague 'done!'"), Some(false)); - assert_eq!(parse_pass_fail("PASS, though it could FAIL on edge cases"), Some(true)); // PASS first + assert_eq!( + parse_pass_fail("PASS, though it could FAIL on edge cases"), + Some(true) + ); // PASS first assert_eq!(parse_pass_fail("verdict: fail, because…"), Some(false)); assert_eq!(parse_pass_fail("no verdict here"), None); } @@ -2537,35 +2843,84 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } fn declarative_checks_file_json() { let cat = tempfile::tempdir().unwrap(); std::fs::create_dir_all(cat.path().join("worker")).unwrap(); - std::fs::write(cat.path().join("worker/LICENSE"), "Permission is hereby granted, free of charge\n").unwrap(); - std::fs::write(cat.path().join("worker/package.json"), r#"{"name":"w","license":"MIT","ok":true,"count":3}"#).unwrap(); + std::fs::write( + cat.path().join("worker/LICENSE"), + "Permission is hereby granted, free of charge\n", + ) + .unwrap(); + std::fs::write( + cat.path().join("worker/package.json"), + r#"{"name":"w","license":"MIT","ok":true,"count":3}"#, + ) + .unwrap(); let ok = [ - Check::FileHas { path: "worker/LICENSE".into(), text: "Permission is hereby granted".into() }, - Check::FileLacks { path: "worker/LICENSE".into(), text: "proprietary".into() }, - Check::JsonField { path: "worker/package.json".into(), field: "license".into(), value: crate::eval_spec::JsonScalar::String("MIT".into()) }, + Check::FileHas { + path: "worker/LICENSE".into(), + text: "Permission is hereby granted".into(), + }, + Check::FileLacks { + path: "worker/LICENSE".into(), + text: "proprietary".into(), + }, + Check::JsonField { + path: "worker/package.json".into(), + field: "license".into(), + value: crate::eval_spec::JsonScalar::String("MIT".into()), + }, ]; assert!(run_declarative(&ok, cat.path()).0); // A wrong json value fails. - let bad = [Check::JsonField { path: "worker/package.json".into(), field: "license".into(), value: crate::eval_spec::JsonScalar::String("GPL".into()) }]; + let bad = [Check::JsonField { + path: "worker/package.json".into(), + field: "license".into(), + value: crate::eval_spec::JsonScalar::String("GPL".into()), + }]; assert!(!run_declarative(&bad, cat.path()).0); let typed = [ - Check::JsonField { path: "worker/package.json".into(), field: "ok".into(), value: crate::eval_spec::JsonScalar::Bool(true) }, - Check::JsonField { path: "worker/package.json".into(), field: "count".into(), value: crate::eval_spec::JsonScalar::Integer(3) }, + Check::JsonField { + path: "worker/package.json".into(), + field: "ok".into(), + value: crate::eval_spec::JsonScalar::Bool(true), + }, + Check::JsonField { + path: "worker/package.json".into(), + field: "count".into(), + value: crate::eval_spec::JsonScalar::Integer(3), + }, ]; assert!(run_declarative(&typed, cat.path()).0); let mismatches = [ - Check::JsonField { path: "worker/package.json".into(), field: "count".into(), value: crate::eval_spec::JsonScalar::String("3".into()) }, - Check::JsonField { path: "worker/package.json".into(), field: "ok".into(), value: crate::eval_spec::JsonScalar::String("true".into()) }, + Check::JsonField { + path: "worker/package.json".into(), + field: "count".into(), + value: crate::eval_spec::JsonScalar::String("3".into()), + }, + Check::JsonField { + path: "worker/package.json".into(), + field: "ok".into(), + value: crate::eval_spec::JsonScalar::String("true".into()), + }, ]; assert!(!run_declarative(&mismatches[..1], cat.path()).0); assert!(!run_declarative(&mismatches[1..], cat.path()).0); std::fs::write(cat.path().join("worker/malformed.json"), "{").unwrap(); - let malformed = [Check::JsonField { path: "worker/malformed.json".into(), field: "n".into(), value: crate::eval_spec::JsonScalar::Integer(1) }]; - let missing_json = [Check::JsonField { path: "worker/missing.json".into(), field: "n".into(), value: crate::eval_spec::JsonScalar::Integer(1) }]; + let malformed = [Check::JsonField { + path: "worker/malformed.json".into(), + field: "n".into(), + value: crate::eval_spec::JsonScalar::Integer(1), + }]; + let missing_json = [Check::JsonField { + path: "worker/missing.json".into(), + field: "n".into(), + value: crate::eval_spec::JsonScalar::Integer(1), + }]; assert!(!run_declarative(&malformed, cat.path()).0); assert!(!run_declarative(&missing_json, cat.path()).0); // FileHas on a missing file fails. - let missing = [Check::FileHas { path: "worker/NOPE".into(), text: "x".into() }]; + let missing = [Check::FileHas { + path: "worker/NOPE".into(), + text: "x".into(), + }]; assert!(!run_declarative(&missing, cat.path()).0); } @@ -2582,9 +2937,26 @@ agent "worker" { identity "worker"; host "evalhost"; argv "true" } assert!(!run_bash_judge("sleep 30", s, c, b, Duration::from_millis(300), je).0); // CWD is the SPEC folder (so ./judges/x.sh resolves); $CATALOG/$ST_ROOT reach the sandbox+bus. std::fs::create_dir_all(s.join("judges")).unwrap(); - std::fs::write(s.join("judges/ok.sh"), "#!/bin/sh\ntest -n \"$CATALOG\" && test -n \"$ST_ROOT\"\n").unwrap(); - assert!(run_bash_judge("test \"$(pwd)\" = \"$SPEC_DIR\"", s, c, b, Duration::from_secs(5), je).0); - assert!(run_bash_judge("sh ./judges/ok.sh", s, c, b, Duration::from_secs(5), je).0, "./judges resolves from CWD=spec"); + std::fs::write( + s.join("judges/ok.sh"), + "#!/bin/sh\ntest -n \"$CATALOG\" && test -n \"$ST_ROOT\"\n", + ) + .unwrap(); + assert!( + run_bash_judge( + "test \"$(pwd)\" = \"$SPEC_DIR\"", + s, + c, + b, + Duration::from_secs(5), + je + ) + .0 + ); + assert!( + run_bash_judge("sh ./judges/ok.sh", s, c, b, Duration::from_secs(5), je).0, + "./judges resolves from CWD=spec" + ); } #[test] diff --git a/src/eval_spec.rs b/src/eval_spec.rs index 7db3cfc5..1c589d92 100644 --- a/src/eval_spec.rs +++ b/src/eval_spec.rs @@ -159,13 +159,21 @@ pub enum Check { /// `file "p" lacks "text"` — path does NOT contain the substring. FileLacks { path: String, text: String }, /// `json "p" field "x" is "y"` — the JSON field equals the value. - JsonField { path: String, field: String, value: JsonScalar }, + JsonField { + path: String, + field: String, + value: JsonScalar, + }, /// `committed "p"` — the path is tracked + committed (checked by the eval flow via git). Committed { path: String }, } #[derive(Debug, Clone, PartialEq, Eq)] -pub enum JsonScalar { String(String), Bool(bool), Integer(i64) } +pub enum JsonScalar { + String(String), + Bool(bool), + Integer(i64), +} // ── KDL helpers (match the idiom in render.rs / kdl_format.rs) ─────────────────────────────────── @@ -181,7 +189,9 @@ fn arg_n(node: &KdlNode, n: usize) -> Option { /// The first positional argument as a `u32` (KDL bare integer, e.g. `attempts 3`). fn arg_u32(node: &KdlNode) -> Option { - node.get(0).and_then(|v| v.as_integer()).and_then(|i| u32::try_from(i).ok()) + node.get(0) + .and_then(|v| v.as_integer()) + .and_then(|i| u32::try_from(i).ok()) } /// A child node's first argument, by child name. @@ -220,8 +230,9 @@ fn parse_agent_presentation( /// the eval block run in file order. (The old nested `run { step … }` wrapper was retired once every /// cell moved to the flat form — a `run` with no label is now an error.) fn parse_run_stage(node: &KdlNode, out: &mut Vec) -> anyhow::Result<()> { - let label = arg(node) - .ok_or_else(|| anyhow::anyhow!("run needs a label: write `run \"label\" {{ command … }}`"))?; + let label = arg(node).ok_or_else(|| { + anyhow::anyhow!("run needs a label: write `run \"label\" {{ command … }}`") + })?; out.push(parse_step_body(node, label)?); Ok(()) } @@ -230,7 +241,9 @@ fn parse_run_stage(node: &KdlNode, out: &mut Vec) -> anyhow::Result<()> /// allow-nonzero }`. By default the step must exit 0; `allow-nonzero` opts out. `require-exit 0` is /// still accepted (it now just affirms the default). fn parse_step_body(node: &KdlNode, id: String) -> anyhow::Result { - let ch = node.children().ok_or_else(|| anyhow::anyhow!("run step '{id}' is empty"))?; + let ch = node + .children() + .ok_or_else(|| anyhow::anyhow!("run step '{id}' is empty"))?; let mut workspace = None; let mut command = None; let mut env = BTreeMap::new(); @@ -257,8 +270,12 @@ fn parse_step_body(node: &KdlNode, id: String) -> anyhow::Result { // (accepted, no-op). Any other value is still an error. "require-exit" => match c.get(0).and_then(|v| v.as_integer()) { Some(0) => {} - Some(n) => anyhow::bail!("step '{id}': `require-exit {n}` — exit 0 is the default; use `allow-nonzero` to accept a non-zero exit"), - None => anyhow::bail!("step '{id}': require-exit needs the integer `0` (or drop it — exit 0 is the default)"), + Some(n) => anyhow::bail!( + "step '{id}': `require-exit {n}` — exit 0 is the default; use `allow-nonzero` to accept a non-zero exit" + ), + None => anyhow::bail!( + "step '{id}': require-exit needs the integer `0` (or drop it — exit 0 is the default)" + ), }, other => anyhow::bail!( "step '{id}': unexpected node '{other}' (expected workspace|command|env|unset|retry|allow-nonzero)" @@ -278,7 +295,9 @@ fn parse_step_body(node: &KdlNode, id: String) -> anyhow::Result { /// `retry { attempts N; interval T }` → a [`Restart`] (mode=fail; `delay` = the inter-retry backoff). fn parse_retry(node: &KdlNode, step_id: &str) -> anyhow::Result { - let ch = node.children().ok_or_else(|| anyhow::anyhow!("step '{step_id}': retry{{}} is empty"))?; + let ch = node + .children() + .ok_or_else(|| anyhow::anyhow!("step '{step_id}': retry{{}} is empty"))?; let mut attempts = None; let mut interval = None; for c in ch.nodes() { @@ -290,17 +309,30 @@ fn parse_retry(node: &KdlNode, step_id: &str) -> anyhow::Result { } } "interval" => { - let s = arg(c).ok_or_else(|| anyhow::anyhow!("step '{step_id}': retry interval needs a duration"))?; - interval = - Some(parse_duration(&s).map_err(|e| anyhow::anyhow!("step '{step_id}': retry interval: {e}"))?); + let s = arg(c).ok_or_else(|| { + anyhow::anyhow!("step '{step_id}': retry interval needs a duration") + })?; + interval = Some( + parse_duration(&s) + .map_err(|e| anyhow::anyhow!("step '{step_id}': retry interval: {e}"))?, + ); } - other => anyhow::bail!("step '{step_id}': retry has unexpected node '{other}' (expected attempts|interval)"), + other => anyhow::bail!( + "step '{step_id}': retry has unexpected node '{other}' (expected attempts|interval)" + ), } } - let attempts = attempts.ok_or_else(|| anyhow::anyhow!("step '{step_id}': retry needs `attempts N`"))?; - let interval = interval.ok_or_else(|| anyhow::anyhow!("step '{step_id}': retry needs `interval T`"))?; + let attempts = + attempts.ok_or_else(|| anyhow::anyhow!("step '{step_id}': retry needs `attempts N`"))?; + let interval = + interval.ok_or_else(|| anyhow::anyhow!("step '{step_id}': retry needs `interval T`"))?; // Retry maps to a fail-mode Restart: stop after `attempts`, wait `interval` (the backoff) between. - Ok(Restart { attempts, interval, delay: interval, mode: RestartMode::Fail }) + Ok(Restart { + attempts, + interval, + delay: interval, + mode: RestartMode::Fail, + }) } /// Parse an `env { }` block into a map of var → value (raw; `$CATALOG` expands at spawn). @@ -317,7 +349,10 @@ fn parse_env(node: &KdlNode) -> BTreeMap { } /// Merge `parent` and `child` env, child winning. (The env cascade: top → team → agent → process.) -fn cascade(parent: &BTreeMap, child: &BTreeMap) -> BTreeMap { +fn cascade( + parent: &BTreeMap, + child: &BTreeMap, +) -> BTreeMap { let mut out = parent.clone(); out.extend(child.iter().map(|(k, v)| (k.clone(), v.clone()))); out @@ -358,7 +393,9 @@ pub fn parse_spec(text: &str) -> anyhow::Result { "agent" => agents.push(parse_agent(node, "", &top_env)?), "eval" => eval = Some(parse_eval(node, &top_env)?), other => { - anyhow::bail!("st2 spec: unexpected top-level node '{other}' (expected host|env|team|agent|eval)") + anyhow::bail!( + "st2 spec: unexpected top-level node '{other}' (expected host|env|team|agent|eval)" + ) } } } @@ -372,7 +409,12 @@ pub fn parse_spec(text: &str) -> anyhow::Result { "eval `canonical-agents` is mutually exclusive with compact `team` / `agent` declarations" ); } - Ok(Spec { host, env: top_env, agents, eval }) + Ok(Spec { + host, + env: top_env, + agents, + eval, + }) } /// Recurse a `team "name" { }`: prefix its agents' ids and cascade its env into them. @@ -383,7 +425,11 @@ fn collect_team( out: &mut Vec, ) -> anyhow::Result<()> { let name = arg(node).ok_or_else(|| anyhow::anyhow!("team needs a name"))?; - let new_prefix = if prefix.is_empty() { name } else { format!("{prefix}.{name}") }; + let new_prefix = if prefix.is_empty() { + name + } else { + format!("{prefix}.{name}") + }; // A team may carry its own env, cascading to the agents it holds. let mut team_env = parent_env.clone(); if let Some(ch) = node.children() { @@ -397,7 +443,9 @@ fn collect_team( "env" => {} "agent" => out.push(parse_agent(c, &new_prefix, &team_env)?), "team" => collect_team(c, &new_prefix, &team_env, out)?, - other => anyhow::bail!("team '{new_prefix}': unexpected node '{other}' (expected env|agent|team)"), + other => anyhow::bail!( + "team '{new_prefix}': unexpected node '{other}' (expected env|agent|team)" + ), } } } @@ -405,9 +453,17 @@ fn collect_team( } /// Parse one `agent "id" { }` with the accumulated (top+team) env as its parent scope. -fn parse_agent(node: &KdlNode, prefix: &str, parent_env: &BTreeMap) -> anyhow::Result { +fn parse_agent( + node: &KdlNode, + prefix: &str, + parent_env: &BTreeMap, +) -> anyhow::Result { let name = arg(node).ok_or_else(|| anyhow::anyhow!("agent needs an id"))?; - let id = if prefix.is_empty() { name } else { format!("{prefix}.{name}") }; + let id = if prefix.is_empty() { + name + } else { + format!("{prefix}.{name}") + }; let mut display_name = None; let mut description = None; @@ -429,9 +485,7 @@ fn parse_agent(node: &KdlNode, prefix: &str, parent_env: &BTreeMap {} "name" => parse_agent_presentation(c, &id, "name", &mut display_name)?, - "description" => { - parse_agent_presentation(c, &id, "description", &mut description)? - } + "description" => parse_agent_presentation(c, &id, "description", &mut description)?, "workspace" => workspace = arg(c), "supervisor" => supervisor = arg(c), "command" => command = arg(c), @@ -458,14 +512,16 @@ fn parse_agent(node: &KdlNode, prefix: &str, parent_env: &BTreeMap { - let ex_leaf = arg(c).ok_or_else(|| anyhow::anyhow!("agent '{id}': exec needs an id"))?; + let ex_leaf = + arg(c).ok_or_else(|| anyhow::anyhow!("agent '{id}': exec needs an id"))?; let ex_id = if ex_leaf == id || ex_leaf.starts_with(&format!("{id}.")) { ex_leaf.clone() } else { format!("{id}.{ex_leaf}") }; - let ex_command = child_arg(c, "command") - .ok_or_else(|| anyhow::anyhow!("agent '{id}': exec '{ex_id}' needs a command"))?; + let ex_command = child_arg(c, "command").ok_or_else(|| { + anyhow::anyhow!("agent '{id}': exec '{ex_id}' needs a command") + })?; let mut ex_env = BTreeMap::new(); if let Some(exc) = c.children() { for e in exc.nodes() { @@ -509,7 +565,9 @@ fn parse_agent(node: &KdlNode, prefix: &str, parent_env: &BTreeMap) -> anyhow::Result { - let ch = node.children().ok_or_else(|| anyhow::anyhow!("eval {{}} is empty"))?; + let ch = node + .children() + .ok_or_else(|| anyhow::anyhow!("eval {{}} is empty"))?; let mut copy = None; let mut message = None; let mut max_timeout = None; @@ -524,8 +582,10 @@ fn parse_eval(node: &KdlNode, top_env: &BTreeMap) -> anyhow::Res "copy" => copy = arg(c), "message" => message = Some(parse_message(c)?), "max-timeout" => { - let s = arg(c).ok_or_else(|| anyhow::anyhow!("eval: max-timeout needs a duration"))?; - max_timeout = Some(parse_duration(&s).map_err(|e| anyhow::anyhow!("eval max-timeout: {e}"))?); + let s = + arg(c).ok_or_else(|| anyhow::anyhow!("eval: max-timeout needs a duration"))?; + max_timeout = + Some(parse_duration(&s).map_err(|e| anyhow::anyhow!("eval max-timeout: {e}"))?); } "agent" => agents.push(parse_agent(c, "", top_env)?), "team" => collect_team(c, "", top_env, &mut agents)?, @@ -571,10 +631,11 @@ fn parse_message(node: &KdlNode) -> anyhow::Result { // Catch the reference-example glitch first: `message { from "r" to "a" content "c" }` on one line // (no newline/`;` separators) mis-parses as a SINGLE `from` node with the rest as args. Reject it // with a fix-it rather than silently taking one field. - if node - .children() - .is_some_and(|d| d.nodes().iter().any(|n| n.name().value() == "from" && n.entries().len() > 1)) - { + if node.children().is_some_and(|d| { + d.nodes() + .iter() + .any(|n| n.name().value() == "from" && n.entries().len() > 1) + }) { anyhow::bail!( "message: `from`/`to`/`content` must be SEPARATE nodes (newline- or `;`-separated), not one \ unseparated line — `message {{ from \"r\"; to \"a\"; content \"./task.md\" }}`" @@ -582,17 +643,23 @@ fn parse_message(node: &KdlNode) -> anyhow::Result { } let from = child_arg(node, "from").ok_or_else(|| anyhow::anyhow!("message needs `from`"))?; let to = child_arg(node, "to").ok_or_else(|| anyhow::anyhow!("message needs `to`"))?; - let content = child_arg(node, "content").ok_or_else(|| anyhow::anyhow!("message needs `content`"))?; + let content = + child_arg(node, "content").ok_or_else(|| anyhow::anyhow!("message needs `content`"))?; Ok(Kick { from, to, content }) } /// Parse the `judges { }` block into all-must-pass judges. fn parse_judges(node: &KdlNode) -> anyhow::Result> { let mut judges = Vec::new(); - let Some(ch) = node.children() else { return Ok(judges) }; + let Some(ch) = node.children() else { + return Ok(judges); + }; for c in ch.nodes() { if c.name().value() != "judge" { - anyhow::bail!("judges: unexpected node '{}' (expected judge)", c.name().value()); + anyhow::bail!( + "judges: unexpected node '{}' (expected judge)", + c.name().value() + ); } judges.push(parse_judge(c)?); } @@ -618,34 +685,59 @@ fn parse_judge(node: &KdlNode) -> anyhow::Result { "signal" => signal = true, "timeout" => { if let Some(s) = arg(c) { - timeout = Some(parse_duration(&s).map_err(|e| anyhow::anyhow!("judge '{name}' timeout: {e}"))?); + timeout = Some( + parse_duration(&s) + .map_err(|e| anyhow::anyhow!("judge '{name}' timeout: {e}"))?, + ); } } "exec" => exec_cmd = arg(c), "ask" => { - let agent = arg(c).ok_or_else(|| anyhow::anyhow!("judge '{name}': ask needs an agent"))?; - let prompt = arg_n(c, 1).ok_or_else(|| anyhow::anyhow!("judge '{name}': ask needs a prompt"))?; + let agent = + arg(c).ok_or_else(|| anyhow::anyhow!("judge '{name}': ask needs an agent"))?; + let prompt = arg_n(c, 1) + .ok_or_else(|| anyhow::anyhow!("judge '{name}': ask needs a prompt"))?; ask = Some((agent, prompt)); } "file" => { - let path = arg(c).ok_or_else(|| anyhow::anyhow!("judge '{name}': file needs a path"))?; - let op = arg_n(c, 1).ok_or_else(|| anyhow::anyhow!("judge '{name}': file needs has/lacks"))?; - let text = arg_n(c, 2).ok_or_else(|| anyhow::anyhow!("judge '{name}': file needs a value"))?; + let path = + arg(c).ok_or_else(|| anyhow::anyhow!("judge '{name}': file needs a path"))?; + let op = arg_n(c, 1) + .ok_or_else(|| anyhow::anyhow!("judge '{name}': file needs has/lacks"))?; + let text = arg_n(c, 2) + .ok_or_else(|| anyhow::anyhow!("judge '{name}': file needs a value"))?; checks.push(match op.as_str() { "has" => Check::FileHas { path, text }, "lacks" => Check::FileLacks { path, text }, - other => anyhow::bail!("judge '{name}': file op '{other}' (expected has|lacks)"), + other => { + anyhow::bail!("judge '{name}': file op '{other}' (expected has|lacks)") + } }); } "json" => { // json "p" field "x" is "y" - let path = arg(c).ok_or_else(|| anyhow::anyhow!("judge '{name}': json needs a path"))?; - let field = arg_n(c, 2).ok_or_else(|| anyhow::anyhow!("judge '{name}': json needs `field `"))?; - let value = c.get(4).and_then(|v| match v { KdlValue::String(s) => Some(JsonScalar::String(s.clone())), KdlValue::Bool(b) => Some(JsonScalar::Bool(*b)), KdlValue::Integer(i) => (*i).try_into().ok().map(JsonScalar::Integer), _ => None }).ok_or_else(|| anyhow::anyhow!("judge '{name}': json value must be string, boolean, or integer"))?; + let path = + arg(c).ok_or_else(|| anyhow::anyhow!("judge '{name}': json needs a path"))?; + let field = arg_n(c, 2) + .ok_or_else(|| anyhow::anyhow!("judge '{name}': json needs `field `"))?; + let value = c + .get(4) + .and_then(|v| match v { + KdlValue::String(s) => Some(JsonScalar::String(s.clone())), + KdlValue::Bool(b) => Some(JsonScalar::Bool(*b)), + KdlValue::Integer(i) => (*i).try_into().ok().map(JsonScalar::Integer), + _ => None, + }) + .ok_or_else(|| { + anyhow::anyhow!( + "judge '{name}': json value must be string, boolean, or integer" + ) + })?; checks.push(Check::JsonField { path, field, value }); } "committed" => { - let path = arg(c).ok_or_else(|| anyhow::anyhow!("judge '{name}': committed needs a path"))?; + let path = arg(c) + .ok_or_else(|| anyhow::anyhow!("judge '{name}': committed needs a path"))?; checks.push(Check::Committed { path }); } other => anyhow::bail!("judge '{name}': unexpected node '{other}'"), @@ -653,12 +745,19 @@ fn parse_judge(node: &KdlNode) -> anyhow::Result { } // Exactly one flavor. - let flavors = [exec_cmd.is_some(), ask.is_some(), !checks.is_empty()].iter().filter(|b| **b).count(); + let flavors = [exec_cmd.is_some(), ask.is_some(), !checks.is_empty()] + .iter() + .filter(|b| **b) + .count(); if flavors == 0 { - anyhow::bail!("judge '{name}' has no check (need exec | ask | declarative file/json/committed)"); + anyhow::bail!( + "judge '{name}' has no check (need exec | ask | declarative file/json/committed)" + ); } if flavors > 1 { - anyhow::bail!("judge '{name}' mixes flavors — a judge is exactly one of exec | ask | declarative"); + anyhow::bail!( + "judge '{name}' mixes flavors — a judge is exactly one of exec | ask | declarative" + ); } let kind = if let Some(cmd) = exec_cmd { JudgeKind::Bash(cmd) @@ -667,7 +766,12 @@ fn parse_judge(node: &KdlNode) -> anyhow::Result { } else { JudgeKind::Declarative(checks) }; - Ok(Judge { name, timeout, kind, signal }) + Ok(Judge { + name, + timeout, + kind, + signal, + }) } #[cfg(test)] @@ -736,33 +840,50 @@ eval { // Agent IS its pty (command on the agent) + cascaded authored env. The runner adds identity. let sup = &s.agents[0]; assert_eq!(sup.workspace.as_deref(), Some("./sup")); - assert!(sup.command.contains("exec claude --permission-mode bypassPermissions")); + assert!( + sup.command + .contains("exec claude --permission-mode bypassPermissions") + ); assert_eq!(sup.env.get("ST_ROOT").unwrap(), "$CATALOG/custom-bus"); // cascaded from top assert!(!sup.env.contains_key("ST_AGENT")); // The bare `ding` node auto-derives `.ding` and st2 generates the standard sidecar // against the cascaded $ST_ROOT; it inherits the agent env. assert_eq!(sup.execs.len(), 1); assert_eq!(sup.execs[0].id, "mix.sup.ding"); - assert_eq!(sup.execs[0].command, "st2 ding --identity mix.sup --root $ST_ROOT"); + assert_eq!( + sup.execs[0].command, + "st2 ding --identity mix.sup --root $ST_ROOT" + ); assert!(!sup.execs[0].env.contains_key("ST_AGENT")); // Eval block. let ev = s.eval.as_ref().unwrap(); assert_eq!(ev.copy.as_deref(), Some("./fixture")); - assert_eq!(ev.message, Some(Kick { from: "requester".into(), to: "mix.sup".into(), content: "./task.md".into() })); + assert_eq!( + ev.message, + Some(Kick { + from: "requester".into(), + to: "mix.sup".into(), + content: "./task.md".into() + }) + ); assert_eq!(ev.max_timeout, Duration::from_secs(1200)); assert_eq!(ev.agents.len(), 1); // the eval-only judge agent assert_eq!(ev.agents[0].id, "judge"); // Judges — all three flavors parsed. assert_eq!(ev.judges.len(), 5); - assert!(matches!(&ev.judges[0].kind, JudgeKind::Bash(c) if c == "sh ./judges/isolation.sh")); + assert!( + matches!(&ev.judges[0].kind, JudgeKind::Bash(c) if c == "sh ./judges/isolation.sh") + ); match &ev.judges[1].kind { JudgeKind::Declarative(checks) => { assert_eq!(checks.len(), 2); assert!(matches!(&checks[0], Check::FileHas { path, text } if path == "worker/LICENSE" && text.starts_with("Permission is hereby"))); - assert!(matches!(&checks[1], Check::FileLacks { path, .. } if path == "worker/LICENSE")); + assert!( + matches!(&checks[1], Check::FileLacks { path, .. } if path == "worker/LICENSE") + ); } other => panic!("expected declarative, got {other:?}"), } @@ -781,12 +902,15 @@ eval { // The dedicated `ding` node must parse to EXACTLY what the `ding_exec` helper generates. let sup = &parse_spec(REFERENCE).unwrap().agents[0]; assert!(sup.execs[0].derived); - assert_eq!(ding_exec("mix.sup"), SpecExec { - id: sup.execs[0].id.clone(), - command: sup.execs[0].command.clone(), - env: BTreeMap::new(), - derived: true, - }); + assert_eq!( + ding_exec("mix.sup"), + SpecExec { + id: sup.execs[0].id.clone(), + command: sup.execs[0].command.clone(), + env: BTreeMap::new(), + derived: true, + } + ); } #[test] @@ -809,16 +933,25 @@ eval { let source = format!( "agent \"worker\" {{ {field} \"left{separator}right\"; command \"true\" }}" ); - assert!(parse_spec(&source).is_err(), "accepted {field} U+{:04X}", separator as u32); + assert!( + parse_spec(&source).is_err(), + "accepted {field} U+{:04X}", + separator as u32 + ); } let duplicate = format!( "agent \"worker\" {{ {field} \"one\"; {field} \"two\"; command \"true\" }}" ); - assert!(parse_spec(&duplicate).is_err(), "accepted duplicate {field}"); + assert!( + parse_spec(&duplicate).is_err(), + "accepted duplicate {field}" + ); - let malformed = - format!("agent \"worker\" {{ {field} 1; command \"true\" }}"); - assert!(parse_spec(&malformed).is_err(), "accepted malformed {field}"); + let malformed = format!("agent \"worker\" {{ {field} 1; command \"true\" }}"); + assert!( + parse_spec(&malformed).is_err(), + "accepted malformed {field}" + ); } } @@ -875,18 +1008,21 @@ team "mix" { assert_eq!(ex.command, "run it"); // ...and a command-less non-ding exec is an error (only "ding" has a built-in). - let err = parse_spec( - r#"team "mix" { agent "sup" { command "boot"; exec "sidecar" } }"#, - ) - .unwrap_err(); + let err = parse_spec(r#"team "mix" { agent "sup" { command "boot"; exec "sidecar" } }"#) + .unwrap_err(); assert!(err.to_string().contains("needs a command"), "{err}"); } #[test] fn json_judge_rejects_non_scalar_expected_values() { for value in ["null", "[1]", "{\"x\":1}", "1.5"] { - let text = format!("eval {{ judges {{ judge \"j\" {{ json \"x.json\" field \"n\" is {value} }} }} }}"); - assert!(parse_spec(&text).is_err(), "accepted invalid JSON scalar {value}"); + let text = format!( + "eval {{ judges {{ judge \"j\" {{ json \"x.json\" field \"n\" is {value} }} }} }}" + ); + assert!( + parse_spec(&text).is_err(), + "accepted invalid JSON scalar {value}" + ); } } @@ -909,7 +1045,13 @@ team "mix" { eval {{\n message {{ from \"r\"; to \"sup\"; content \"go\" }}\n max-timeout \"5s\"\n{extra}}}\n" ) }; - assert!(parse_spec(&ev(" supervise\n")).unwrap().eval.unwrap().supervise); + assert!( + parse_spec(&ev(" supervise\n")) + .unwrap() + .eval + .unwrap() + .supervise + ); assert!(!parse_spec(&ev("")).unwrap().eval.unwrap().supervise); } @@ -962,8 +1104,15 @@ team "mix" { .unwrap(); assert!(spec.agents.is_empty(), "team-less: no base-team agents"); let ev = spec.eval.unwrap(); - assert!(ev.message.is_none(), "a team-less eval has no kickoff message"); - assert_eq!(ev.run_steps.len(), 2, "two flat run nodes → two ordered steps"); + assert!( + ev.message.is_none(), + "a team-less eval has no kickoff message" + ); + assert_eq!( + ev.run_steps.len(), + 2, + "two flat run nodes → two ordered steps" + ); let build = &ev.run_steps[0]; assert_eq!(build.id, "build"); assert_eq!(build.workspace.as_deref(), Some("repo")); @@ -975,7 +1124,10 @@ team "mix" { let probe = &ev.run_steps[1]; assert_eq!(probe.id, "probe"); assert_eq!(probe.env.get("FOO").map(String::as_str), Some("bar")); - assert_eq!(probe.unset, vec!["ST_ROOT".to_string(), "PTY_ROOT".to_string()]); + assert_eq!( + probe.unset, + vec!["ST_ROOT".to_string(), "PTY_ROOT".to_string()] + ); assert!(probe.allow_nonzero, "opted out of must-exit-0"); } diff --git a/src/event.rs b/src/event.rs new file mode 100644 index 00000000..b1e3244d --- /dev/null +++ b/src/event.rs @@ -0,0 +1,367 @@ +//! Declared event-stream ingress. +//! +//! Events are ordinary inbox files with additional provenance frontmatter. Publication has its +//! own bounded, agent-local receipt ring rather than writing the agent's immutable Sent ledger. + +use std::fs::{self, File, OpenOptions}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::Context as _; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; + +use crate::message; + +const EVENT_VERSION: u32 = 1; +pub const RING_CAPACITY: usize = 128; +static TMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StreamEntry { + event_id: String, + filename: String, + key: Option, + rendered_sha256: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StreamPending { + event_id: String, + filename: String, + key: Option, + rendered_sha256: String, + supersede: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StreamRecord { + version: u32, + stream: String, + recipient: String, + pending: Option, + recent: Vec, +} + +impl StreamRecord { + fn fresh(stream: &str, recipient: &str) -> Self { + Self { + version: EVENT_VERSION, + stream: stream.to_owned(), + recipient: recipient.to_owned(), + pending: None, + recent: Vec::new(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct EventReceipt { + pub recipient: String, + pub stream: String, + pub event_id: String, + pub filename: String, + pub status: EventReceiptStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub superseded: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum EventReceiptStatus { + Created, + Deduplicated, +} + +struct ResolvedStream { + recipient: String, + agent_dir: PathBuf, +} + +fn resolve_stream( + root: &Path, + this_host: &str, + recipient: &str, + stream: &str, +) -> anyhow::Result { + let discovered = crate::discover(root); + anyhow::ensure!( + discovered.errors.is_empty(), + "catalog has errors; refusing event publication: {}", + discovered + .errors + .iter() + .map(|error| format!("{}: {}", error.path.display(), error.message)) + .collect::>() + .join("; ") + ); + let spec = discovered + .specs + .into_iter() + .find(|spec| { + spec.bus_id(this_host) == recipient + || (spec.resolved_host(this_host) == this_host && spec.identity == recipient) + }) + .with_context(|| format!("no agent '{recipient}' found in catalog {}", root.display()))?; + anyhow::ensure!( + spec.streams.iter().any(|declared| declared.name == stream), + "agent '{}' does not declare stream '{stream}'", + spec.bus_id(this_host) + ); + anyhow::ensure!( + spec.desired_state.is_running(), + "agent '{}' is {}; refusing event while its eyes are closed", + spec.bus_id(this_host), + spec.desired_state.as_str() + ); + let agent_dir = spec + .path + .parent() + .context("agent declaration has no parent")? + .to_path_buf(); + Ok(ResolvedStream { + recipient: spec.bus_id(this_host), + agent_dir, + }) +} + +pub fn render_event( + from: &str, + subject: Option<&str>, + stream: &str, + event_id: &str, + key: Option<&str>, + body: &str, +) -> String { + let mut rendered = String::from("---\n"); + rendered.push_str(&format!("from: {from}\n")); + if let Some(subject) = subject { + rendered.push_str(&format!("subject: {subject}\n")); + } + rendered.push_str(&format!("stream: {stream}\n")); + rendered.push_str(&format!("event-id: {event_id}\n")); + if let Some(key) = key { + rendered.push_str(&format!("key: {key}\n")); + } + rendered.push_str("---\n"); + rendered.push_str(body); + if !body.ends_with('\n') { + rendered.push('\n'); + } + rendered +} + +#[allow(clippy::too_many_arguments)] +pub fn emit( + root: &Path, + this_host: &str, + recipient: &str, + stream: &str, + event_id: &str, + key: Option<&str>, + subject: Option<&str>, + body: &str, + supersede: bool, +) -> anyhow::Result { + validate_component("stream", stream)?; + validate_component("event id", event_id)?; + if let Some(key) = key { + validate_component("event key", key)?; + } + if let Some(subject) = subject { + validate_header("event subject", subject, 1_000)?; + } + // Serialize the eligibility observation with self-authoring and desired-state changes. Once a + // suspension edit owns this lock, no later emit can publish from a stale running observation. + let _catalog_lock = crate::catalog_lock::CatalogLock::exclusive(root)?; + let resolved = resolve_stream(root, this_host, recipient, stream)?; + let from = format!("{}/{}", resolved.recipient, stream); + let rendered = render_event(&from, subject, stream, event_id, key, body); + let state_dir = resolved.agent_dir.join("resources/streams").join(stream); + fs::create_dir_all(&state_dir)?; + let _lock = StreamLock::exclusive(&state_dir)?; + let record_path = state_dir.join("state.json"); + let mut record = read_record(&record_path)? + .unwrap_or_else(|| StreamRecord::fresh(stream, &resolved.recipient)); + anyhow::ensure!( + record.version == EVENT_VERSION + && record.stream == stream + && record.recipient == resolved.recipient, + "stream state for '{}#{stream}' is not readable at version {EVENT_VERSION}", + resolved.recipient + ); + + let rendered_sha256 = hex_digest(rendered.as_bytes()); + if let Some(entry) = record + .recent + .iter() + .find(|entry| entry.event_id == event_id) + { + anyhow::ensure!( + entry.rendered_sha256 == rendered_sha256, + "event identity `{stream}#{event_id}` reused with different content" + ); + return Ok(EventReceipt { + recipient: resolved.recipient, + stream: stream.to_owned(), + event_id: event_id.to_owned(), + filename: entry.filename.clone(), + status: EventReceiptStatus::Deduplicated, + superseded: None, + }); + } + + let (filename, resumed) = match record.pending.as_ref() { + Some(pending) if pending.event_id == event_id => { + anyhow::ensure!( + pending.rendered_sha256 == rendered_sha256 + && pending.key.as_deref() == key + && pending.supersede == supersede, + "event identity `{stream}#{event_id}` reused with different content" + ); + (pending.filename.clone(), true) + } + Some(pending) => anyhow::bail!( + "stream '{stream}' has an interrupted event '{}'; replay it before publishing another", + pending.event_id + ), + None => (message::new_filename(), false), + }; + if !resumed { + record.pending = Some(StreamPending { + event_id: event_id.to_owned(), + filename: filename.clone(), + key: key.map(str::to_owned), + rendered_sha256: rendered_sha256.clone(), + supersede, + }); + write_record(&record_path, &record)?; + } + + let inbox = message::inbox_dir(&resolved.agent_dir); + let archive = message::archive_dir(&resolved.agent_dir); + let predecessor = if supersede { + record + .recent + .iter() + .find(|entry| entry.key.as_deref() == key && entry.filename != filename) + .map(|entry| entry.filename.clone()) + } else { + None + }; + if let Some(predecessor) = &predecessor { + message::archive_msg(&inbox, &archive, predecessor)?; + } + // An archive filename is the bus's authoritative durable receipt. A crash after materializing + // and external archive, but before advancing this state, must not restore the inbox replica. + let created = if archive.join(&filename).is_file() { + false + } else { + message::materialize_message_once(&inbox, &filename, &rendered)? + }; + + record.pending = None; + record.recent.insert( + 0, + StreamEntry { + event_id: event_id.to_owned(), + filename: filename.clone(), + key: key.map(str::to_owned), + rendered_sha256, + }, + ); + record.recent.truncate(RING_CAPACITY); + write_record(&record_path, &record)?; + + Ok(EventReceipt { + recipient: resolved.recipient, + stream: stream.to_owned(), + event_id: event_id.to_owned(), + filename, + status: if created { + EventReceiptStatus::Created + } else { + EventReceiptStatus::Deduplicated + }, + superseded: predecessor, + }) +} + +fn validate_component(label: &str, value: &str) -> anyhow::Result<()> { + anyhow::ensure!( + !value.is_empty() + && value.len() <= 200 + && value.trim() == value + && !value.chars().any(char::is_control), + "{label} must be 1..=200 bytes without surrounding whitespace or controls" + ); + Ok(()) +} + +fn validate_header(label: &str, value: &str, max_bytes: usize) -> anyhow::Result<()> { + anyhow::ensure!( + !value.is_empty() + && value.len() <= max_bytes + && value.trim() == value + && !value.chars().any(char::is_control), + "{label} must be 1..={max_bytes} bytes without surrounding whitespace or controls" + ); + Ok(()) +} + +fn hex_digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +fn read_record(path: &Path) -> anyhow::Result> { + match fs::read(path) { + Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes).with_context(|| { + format!("stream state {} is malformed", path.display()) + })?)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } +} + +fn write_record(path: &Path, record: &StreamRecord) -> anyhow::Result<()> { + let parent = path.parent().context("stream state has no parent")?; + fs::create_dir_all(parent)?; + let temporary = parent.join(format!( + ".state.tmp-{}-{}", + std::process::id(), + TMP_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + fs::write(&temporary, serde_json::to_vec(record)?)?; + fs::rename(&temporary, path)?; + Ok(()) +} + +struct StreamLock(File); + +impl StreamLock { + fn exclusive(state_dir: &Path) -> anyhow::Result { + use std::os::fd::AsRawFd as _; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(state_dir.join(".lock"))?; + anyhow::ensure!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } == 0, + "locking stream state failed" + ); + Ok(Self(file)) + } +} + +impl Drop for StreamLock { + fn drop(&mut self) { + use std::os::fd::AsRawFd as _; + unsafe { libc::flock(self.0.as_raw_fd(), libc::LOCK_UN) }; + } +} diff --git a/src/flapping.rs b/src/flapping.rs index 8f3c7521..b5c2e5da 100644 --- a/src/flapping.rs +++ b/src/flapping.rs @@ -192,10 +192,16 @@ mod tests { assert_eq!(cap.decide("p", now, &p), RestartDecision::Allow); cap.record("p", now); } - assert_eq!(cap.decide("p", t0 + Duration::from_secs(4), &p), RestartDecision::GaveUp); + assert_eq!( + cap.decide("p", t0 + Duration::from_secs(4), &p), + RestartDecision::GaveUp + ); assert!(cap.is_parked("p")); // stays parked even after the window empties - assert_eq!(cap.decide("p", t0 + Duration::from_secs(600), &p), RestartDecision::GaveUp); + assert_eq!( + cap.decide("p", t0 + Duration::from_secs(600), &p), + RestartDecision::GaveUp + ); } #[test] @@ -209,10 +215,16 @@ mod tests { cap.record("p", now); } // 4th within the window → rate-limited, NOT parked. - assert_eq!(cap.decide("p", t0 + Duration::from_secs(4), &p), RestartDecision::RateLimited); + assert_eq!( + cap.decide("p", t0 + Duration::from_secs(4), &p), + RestartDecision::RateLimited + ); assert!(!cap.is_parked("p")); // Once the window clears, it's allowed again. - assert_eq!(cap.decide("p", t0 + Duration::from_secs(120), &p), RestartDecision::Allow); + assert_eq!( + cap.decide("p", t0 + Duration::from_secs(120), &p), + RestartDecision::Allow + ); } #[test] @@ -222,9 +234,15 @@ mod tests { let t0 = Instant::now(); cap.record("p", t0); // 5s later — still within the 10s delay. - assert_eq!(cap.decide("p", t0 + Duration::from_secs(5), &p), RestartDecision::Delaying); + assert_eq!( + cap.decide("p", t0 + Duration::from_secs(5), &p), + RestartDecision::Delaying + ); // 11s later — delay satisfied. - assert_eq!(cap.decide("p", t0 + Duration::from_secs(11), &p), RestartDecision::Allow); + assert_eq!( + cap.decide("p", t0 + Duration::from_secs(11), &p), + RestartDecision::Allow + ); } #[test] @@ -233,8 +251,14 @@ mod tests { let p = policy(1, 60, 0, RestartMode::Fail); let t0 = Instant::now(); cap.record("a", t0); - assert_eq!(cap.decide("a", t0 + Duration::from_secs(1), &p), RestartDecision::GaveUp); - assert_eq!(cap.decide("b", t0 + Duration::from_secs(1), &p), RestartDecision::Allow); + assert_eq!( + cap.decide("a", t0 + Duration::from_secs(1), &p), + RestartDecision::GaveUp + ); + assert_eq!( + cap.decide("b", t0 + Duration::from_secs(1), &p), + RestartDecision::Allow + ); } /// Reproduction for the crash-loop budget being unreachable at the supervisor's own cadence. @@ -268,7 +292,10 @@ mod tests { launches += 1; } RestartDecision::GaveUp => { - assert_eq!(launches, 3, "parked, but not after exactly `attempts` launches"); + assert_eq!( + launches, 3, + "parked, but not after exactly `attempts` launches" + ); return; } other => panic!("unexpected {other:?} at pass {pass} (delay is 0)"), @@ -342,7 +369,10 @@ mod tests { assert!(cap.is_parked("p")); // The operator fixed the cause and cleared this one task's park. - assert!(cap.unpark("p"), "unpark reports that it cleared a parked task"); + assert!( + cap.unpark("p"), + "unpark reports that it cleared a parked task" + ); assert!(!cap.is_parked("p")); let mut launches = 0; @@ -378,12 +408,21 @@ mod tests { for id in ["a", "b"] { cap.record(id, t0); - assert_eq!(cap.decide(id, t0 + Duration::from_secs(1), &p), RestartDecision::GaveUp); + assert_eq!( + cap.decide(id, t0 + Duration::from_secs(1), &p), + RestartDecision::GaveUp + ); } assert!(cap.unpark("a")); - assert!(cap.is_parked("b"), "unparking 'a' also released the untouched 'b'"); - assert_eq!(cap.decide("b", t0 + Duration::from_secs(2), &p), RestartDecision::GaveUp); + assert!( + cap.is_parked("b"), + "unparking 'a' also released the untouched 'b'" + ); + assert_eq!( + cap.decide("b", t0 + Duration::from_secs(2), &p), + RestartDecision::GaveUp + ); // Unparking something that was never parked is a no-op the caller can distinguish, so // `st2 unpark` can tell an operator it acted on nothing rather than claim a recovery. @@ -407,7 +446,10 @@ mod tests { cap.end_pass(now, &[]); } - assert!(!cap.unpark("p"), "a non-parked task was reported as recovered"); + assert!( + !cap.unpark("p"), + "a non-parked task was reported as recovered" + ); let last_launch = t0 + Duration::from_secs(2); assert_eq!(cap.decide("p", last_launch, &p), RestartDecision::Allow); diff --git a/src/host_lock.rs b/src/host_lock.rs index 1a27e3ad..83cc8860 100644 --- a/src/host_lock.rs +++ b/src/host_lock.rs @@ -16,7 +16,9 @@ pub struct HostLock { impl HostLock { pub fn new(root: &Path, host: &str) -> Self { - Self { path: root.join(format!(".st2.{host}.lock")) } + Self { + path: root.join(format!(".st2.{host}.lock")), + } } pub fn pid_path(&self) -> &Path { @@ -93,7 +95,10 @@ mod tests { lock.acquire().unwrap(); assert!(lock.pid_path().exists()); - assert!(lock.live_owner().is_none(), "our own lock is not a foreign owner"); + assert!( + lock.live_owner().is_none(), + "our own lock is not a foreign owner" + ); lock.release(); assert!(!lock.pid_path().exists()); @@ -105,7 +110,14 @@ mod tests { let a = HostLock::new(tmp.path(), "hetz"); let b = HostLock::new(tmp.path(), "silber"); assert_ne!(a.pid_path(), b.pid_path(), "per-host lock files"); - assert!(a.pid_path().file_name().unwrap().to_str().unwrap().starts_with('.')); + assert!( + a.pid_path() + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with('.') + ); } #[test] diff --git a/src/isolate.rs b/src/isolate.rs index 5ba00aa3..5b8fe1d9 100644 --- a/src/isolate.rs +++ b/src/isolate.rs @@ -95,7 +95,13 @@ static SCOPE_SEQ: AtomicU64 = AtomicU64::new(0); pub fn scope_unit(task_id: &str) -> String { let safe: String = task_id .chars() - .map(|c| if c.is_ascii_alphanumeric() || matches!(c, ':' | '_' | '.' | '-') { c } else { '_' }) + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, ':' | '_' | '.' | '-') { + c + } else { + '_' + } + }) .collect(); let seq = SCOPE_SEQ.fetch_add(1, Ordering::Relaxed); format!("st2-{safe}-{}-{seq}.scope", std::process::id()) @@ -135,7 +141,10 @@ mod tests { fn scope_unit_is_unique_and_systemd_safe() { // Keeps the (dotted) id for greppability, gains the st2- prefix, a nonce, and the .scope suffix. let u = scope_unit("hetz.demo.agent"); - assert!(u.starts_with("st2-hetz.demo.agent-"), "unexpected unit name {u}"); + assert!( + u.starts_with("st2-hetz.demo.agent-"), + "unexpected unit name {u}" + ); assert!(u.ends_with(".scope"), "unexpected unit name {u}"); // Unsafe bytes (space, slash) are replaced so systemd never rejects the unit name. assert!(scope_unit("a b/c").starts_with("st2-a_b_c-")); @@ -147,8 +156,15 @@ mod tests { fn wrap_scope_prefixes_systemd_run_but_passthrough_does_not() { // We can't force `mode()` per-test (it's process-cached), so assert the shape that matches the // detected mode: on a systemd Linux CI box it's Scope; otherwise pass-through. - let cmd = wrap("st2-x.scope", OsStr::new("sh"), &[OsStr::new("-c"), OsStr::new("true")]); - let args: Vec = cmd.get_args().map(|a| a.to_string_lossy().into_owned()).collect(); + let cmd = wrap( + "st2-x.scope", + OsStr::new("sh"), + &[OsStr::new("-c"), OsStr::new("true")], + ); + let args: Vec = cmd + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); match mode() { Isolation::Scope => { assert_eq!(cmd.get_program(), OsStr::new("systemd-run")); diff --git a/src/lib.rs b/src/lib.rs index d8a9b14e..87fbb618 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub mod ding; pub mod driver; pub mod eval_run; pub mod eval_spec; +pub mod event; pub mod exec_backend; pub mod expand; pub mod flapping; @@ -47,9 +48,7 @@ mod watch; // `st2::spec::…` / `st2::discovery::…` keep working for the binary and the test suite. pub use agent_spec::{discovery, spec}; -pub use agent_spec::discovery::{ - Discovered, SpecError, discover, discover_file, discover_strict, -}; +pub use agent_spec::discovery::{Discovered, SpecError, discover, discover_file, discover_strict}; pub use agent_spec::spec::{ AgentDesiredState, AgentSpec, ClaudeDriver, CodexDriver, DeliveryTransport, Driver, JobType, PiDriver, Resource, Restart, RestartMode, Task, TaskKind, TaskLifecycle, parse_duration, diff --git a/src/main.rs b/src/main.rs index 40bb067e..8b754de5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,6 +69,12 @@ enum Command { /// The stable wire format is a `-.md` Markdown file. #[command(subcommand)] Message(MessageCmd), + /// Declared event streams: durable, bounded, idempotent ingress into an agent inbox. + #[command(subcommand)] + Event(EventCmd), + /// Self-author declared event streams through the serialized catalog path. + #[command(subcommand)] + Stream(StreamCmd), /// Idempotent JSON request/reply transport for declared non-agent service principals. #[command(subcommand)] Request(RequestCmd), @@ -825,6 +831,70 @@ enum MessageCmd { }, } +#[derive(Subcommand)] +enum EventCmd { + /// Emit one producer-identified event into a declared agent stream. + Emit { + /// Owning agent: `.` or a bare local identity. + recipient: String, + /// Declared stream name. + #[arg(long)] + stream: String, + /// Stable producer-supplied event identity. + #[arg(long = "event-id")] + event_id: String, + /// Producer grouping key used by --supersede. + #[arg(long)] + key: Option, + /// Archive the unread predecessor for the same key, or the stream-wide head without --key. + #[arg(long)] + supersede: bool, + /// One-line wake-time summary. + #[arg(long)] + subject: Option, + /// Event body. Read from stdin when omitted. + #[arg(short = 'm', long = "message")] + body: Option, + /// Emit the stable machine receipt. + #[arg(long)] + json: bool, + #[command(flatten)] + ctx: MsgCtx, + }, +} + +#[derive(Subcommand)] +enum StreamCmd { + /// Add a stream to your declaration, optionally with a supervised adapter launch. + Add { + name: String, + /// Exact target agent; defaults to --as / $ST_AGENT. + #[arg(long)] + agent: Option, + /// Adapter command run under `sh -c`; omit both launch forms for external ingress. + #[arg(long, conflicts_with = "adapter_argv")] + command: Option, + /// Direct adapter argv. Element 0 is the program. + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + adapter_argv: Vec, + #[arg(long)] + json: bool, + #[command(flatten)] + ctx: MsgCtx, + }, + /// Remove a stream from your declaration. + Rm { + name: String, + /// Exact target agent; defaults to --as / $ST_AGENT. + #[arg(long)] + agent: Option, + #[arg(long)] + json: bool, + #[command(flatten)] + ctx: MsgCtx, + }, +} + #[derive(Subcommand)] enum RequestCmd { /// Publish one idempotent JSON request from a declared service principal to an agent. @@ -912,6 +982,8 @@ fn main() -> Result<()> { up(&root, host, once, materialize_only, interval, agent, task) } Command::Message(cmd) => message_cmd(cmd), + Command::Event(cmd) => event_cmd(cmd), + Command::Stream(cmd) => stream_cmd(cmd), Command::Request(cmd) => request_cmd(cmd), Command::Context(cmd) => context_cmd(cmd), Command::Resource(cmd) => resource_cmd(cmd), @@ -931,19 +1003,18 @@ fn main() -> Result<()> { } => { let catalog = catalog_arg(None)?; let catalog = catalog.canonicalize().unwrap_or(catalog); - st2::codex_app_server::run_controlled( - &catalog, - identity, - runtime_id, - codex_argv, - ) + st2::codex_app_server::run_controlled(&catalog, identity, runtime_id, codex_argv) } Command::ClaudeMcp { identity } => { let catalog = catalog_arg(None)?; let catalog = catalog.canonicalize().unwrap_or(catalog); st2::claude_mcp::run(&catalog, &identity) } - Command::Driver(DriverCmd::Codex { identity, runtime_id, argv }) => { + Command::Driver(DriverCmd::Codex { + identity, + runtime_id, + argv, + }) => { let catalog = catalog_arg(None)?; let catalog = catalog.canonicalize().unwrap_or(catalog); st2::codex_app_server::run_controlled(&catalog, identity, runtime_id, argv) @@ -1256,9 +1327,7 @@ fn driver_expand_cmd( eprintln!("warning: {warning}"); } if let Some(agent) = agent { - specs.retain(|spec| { - spec.identity == agent || spec.bus_id(host.unwrap_or("")) == agent - }); + specs.retain(|spec| spec.identity == agent || spec.bus_id(host.unwrap_or("")) == agent); } anyhow::ensure!( specs.len() == 1, @@ -1820,9 +1889,9 @@ fn tasks_cmd(root: &Path, host: Option) -> Result<()> { /// why the confirmation says "requested" rather than claiming the task is back. fn unpark_cmd(catalog: &Path, task: &str, host: Option) -> Result<()> { let host = host.unwrap_or_else(detect_host); - let catalog = catalog.canonicalize().with_context(|| { - format!("canonicalize catalog {}", catalog.display()) - })?; + let catalog = catalog + .canonicalize() + .with_context(|| format!("canonicalize catalog {}", catalog.display()))?; let dir = st2::park::SupervisorScope::current(&catalog, &host)?.unpark_request_dir(); st2::park::request_unpark(&dir, task)?; println!( @@ -1940,7 +2009,9 @@ fn desired_state_cmd( }; let root = catalog_arg(None)?; let host = host.unwrap_or_else(detect_host); - let actor = std::env::var("ST_AGENT").ok().filter(|value| !value.is_empty()); + let actor = std::env::var("ST_AGENT") + .ok() + .filter(|value| !value.is_empty()); match st2::agent_author::set_desired_state( &root, &identity, @@ -2479,6 +2550,102 @@ fn send_resolved_message( ) } +fn event_cmd(cmd: EventCmd) -> Result<()> { + match cmd { + EventCmd::Emit { + recipient, + stream, + event_id, + key, + supersede, + subject, + body, + json, + ctx, + } => { + let (root, host) = resolve_ctx(&ctx)?; + let body = body_or_stdin(body)?; + let receipt = st2::event::emit( + &root, + &host, + &recipient, + &stream, + &event_id, + key.as_deref(), + subject.as_deref(), + &body, + supersede, + )?; + if json { + println!("{}", serde_json::to_string(&receipt)?); + } else { + println!("{}", receipt.filename); + } + Ok(()) + } + } +} + +fn stream_cmd(cmd: StreamCmd) -> Result<()> { + let (name, agent, json, ctx, launch, remove) = match cmd { + StreamCmd::Add { + name, + agent, + command, + adapter_argv, + json, + ctx, + } => { + let launch = match (command, adapter_argv.is_empty()) { + (Some(command), true) => Some(agent_spec::StreamLaunch::Command(command)), + (None, false) => Some(agent_spec::StreamLaunch::Argv(adapter_argv)), + (None, true) => None, + (Some(_), false) => anyhow::bail!("stream add got both --command and adapter argv"), + }; + (name, agent, json, ctx, launch, false) + } + StreamCmd::Rm { + name, + agent, + json, + ctx, + } => (name, agent, json, ctx, None, true), + }; + let (root, host) = resolve_ctx(&ctx)?; + let actor = ctx + .as_id + .clone() + .or_else(|| std::env::var("ST_AGENT").ok()) + .filter(|value| !value.is_empty()); + let target = agent + .or_else(|| actor.clone()) + .context("no stream target: pass --agent, --as, or set $ST_AGENT")?; + if remove { + let receipt = + st2::agent_author::remove_stream(&root, &target, &host, actor.as_deref(), &name)?; + if json { + println!("{}", serde_json::to_string(&receipt)?); + } else { + println!( + "{:?} stream {} on {}", + receipt.result, receipt.name, receipt.identity + ); + } + } else { + let receipt = + st2::agent_author::add_stream(&root, &target, &host, actor.as_deref(), &name, launch)?; + if json { + println!("{}", serde_json::to_string(&receipt)?); + } else { + println!( + "{:?} stream {} on {}", + receipt.result, receipt.name, receipt.identity + ); + } + } + Ok(()) +} + fn request_cmd(cmd: RequestCmd) -> Result<()> { match cmd { RequestCmd::Send { @@ -2548,12 +2715,7 @@ fn request_cmd(cmd: RequestCmd) -> Result<()> { } => { let (root, host) = resolve_ctx(&ctx)?; let principal = acting_id(&ctx)?; - let status = st2::request::status( - &root, - &host, - &principal, - &idempotency_key, - )?; + let status = st2::request::status(&root, &host, &principal, &idempotency_key)?; if json { println!("{}", serde_json::to_string(&status)?); } else { diff --git a/src/materialize.rs b/src/materialize.rs index 0aadbe84..b0cd6a97 100644 --- a/src/materialize.rs +++ b/src/materialize.rs @@ -261,8 +261,8 @@ fn resolve_driver_render_executable(plan: &mut RenderPlan, agent: &str) -> Resul if plan.ops.is_empty() { return Ok(()); } - let executable = std::env::current_exe() - .context("resolving st2 executable for driver materialization")?; + let executable = + std::env::current_exe().context("resolving st2 executable for driver materialization")?; for operation in &mut plan.ops { let RenderOp::JsonUpsert { destination, @@ -278,9 +278,7 @@ fn resolve_driver_render_executable(plan: &mut RenderPlan, agent: &str) -> Resul let mut patch: serde_json::Value = serde_json::from_str(content)?; let command = patch .pointer_mut("/mcpServers/st2/command") - .with_context(|| { - format!("agent '{agent}' driver expansion has no st2 MCP command") - })?; + .with_context(|| format!("agent '{agent}' driver expansion has no st2 MCP command"))?; anyhow::ensure!( command.as_str() == Some("st2"), "agent '{agent}' driver expansion has an unexpected st2 MCP command" @@ -295,9 +293,7 @@ fn resolve_driver_render_executable(plan: &mut RenderPlan, agent: &str) -> Resul fn effective_plan(root: &Path, spec: &AgentSpec, this_host: &str) -> Result { crate::driver::ensure_single_source(spec)?; let mut plan = parse_plan_with_driver(spec, this_host)?; - if spec.driver.is_none() - && spec.delivery == Some(agent_spec::spec::DeliveryTransport::Mcp) - { + if spec.driver.is_none() && spec.delivery == Some(agent_spec::spec::DeliveryTransport::Mcp) { let executable = std::env::current_exe() .context("resolving st2 executable for Claude MCP declaration")?; let content = serde_json::json!({ diff --git a/src/message.rs b/src/message.rs index 091b53d8..7fe8e051 100644 --- a/src/message.rs +++ b/src/message.rs @@ -140,6 +140,10 @@ pub struct Message { pub priority: Option, /// `idempotency-key:` — the caller's optional operation identity for exact retries. pub idempotency_key: Option, + /// `stream:` + `event-id:` classify an ordinary inbox record as an event. + pub stream: Option>, + pub event_id: Option>, + pub event_key: Option>, /// The markdown body. pub body: String, } @@ -254,6 +258,9 @@ fn parse_message(filename: &str, contents: &str) -> Message { tags: Vec::new(), priority: None, idempotency_key: None, + stream: None, + event_id: None, + event_key: None, body: String::new(), }; @@ -286,6 +293,9 @@ fn parse_message(filename: &str, contents: &str) -> Message { } "priority" => msg.priority = Some(v.to_string()), "idempotency-key" => msg.idempotency_key = Some(v.to_string()), + "stream" => msg.stream = Some(v.into()), + "event-id" => msg.event_id = Some(v.into()), + "key" => msg.event_key = Some(v.into()), _ => {} } } @@ -509,8 +519,7 @@ fn list_sent_unlocked(root: &Path, include_body: bool) -> anyhow::Result {} (Some(active), [record]) => anyhow::ensure!( - active.filename == record.filename - && active.record_digest == digest_json(record)?, + active.filename == record.filename && active.record_digest == digest_json(record)?, "active sent intent differs from pending record" ), (Some(_), []) => {} @@ -539,9 +548,17 @@ fn list_sent_unlocked(root: &Path, include_body: bool) -> anyhow::Result anyhow::Result anyhow::Result anyhow::Result> { anyhow::ensure!(name.ends_with(".json"), "unexpected sent record entry"); let record: SentRecord = serde_json::from_slice(&fs::read(entry.path())?) .with_context(|| format!("reading sent record {}", entry.path().display()))?; - anyhow::ensure!(record.version == SENT_VERSION, "unsupported sent record version"); + anyhow::ensure!( + record.version == SENT_VERSION, + "unsupported sent record version" + ); anyhow::ensure!( sent_record_name(&record.filename) == name, "sent record filename does not match its payload" @@ -694,9 +717,18 @@ fn read_pending_records(directory: &Path) -> anyhow::Result> { anyhow::ensure!(is_sha256(digest), "invalid pending sent record digest"); let record: SentRecord = serde_json::from_slice(&fs::read(entry.path())?) .with_context(|| format!("reading pending sent record {}", entry.path().display()))?; - anyhow::ensure!(record.version == SENT_VERSION, "unsupported sent record version"); - anyhow::ensure!(is_message_filename(&record.filename), "invalid sent record filename"); - anyhow::ensure!(digest_json(&record)? == digest, "pending sent record digest mismatch"); + anyhow::ensure!( + record.version == SENT_VERSION, + "unsupported sent record version" + ); + anyhow::ensure!( + is_message_filename(&record.filename), + "invalid sent record filename" + ); + anyhow::ensure!( + digest_json(&record)? == digest, + "pending sent record digest mismatch" + ); records.push(record); } records.sort_by(|left, right| left.filename.cmp(&right.filename)); @@ -712,8 +744,14 @@ fn read_sent_active(root: &Path) -> anyhow::Result> { }; let active: SentActive = serde_json::from_slice(&bytes) .with_context(|| format!("reading active sent intent {}", path.display()))?; - anyhow::ensure!(active.version == SENT_VERSION, "unsupported active sent version"); - anyhow::ensure!(is_message_filename(&active.filename), "invalid active sent filename"); + anyhow::ensure!( + active.version == SENT_VERSION, + "unsupported active sent version" + ); + anyhow::ensure!( + is_message_filename(&active.filename), + "invalid active sent filename" + ); Ok(Some(active)) } @@ -738,10 +776,22 @@ fn read_sent_commits(directory: &Path) -> anyhow::Result anyhow::Result> let key: SentKey = serde_json::from_slice(&fs::read(entry.path())?) .with_context(|| format!("reading sent key {}", entry.path().display()))?; anyhow::ensure!(key.version == SENT_VERSION, "unsupported sent key version"); - anyhow::ensure!(digest_json(&(&key.to, &key.key))? == digest, "sent key filename mismatch"); - anyhow::ensure!(keys.insert(digest.to_string(), key).is_none(), "duplicate sent key"); + anyhow::ensure!( + digest_json(&(&key.to, &key.key))? == digest, + "sent key filename mismatch" + ); + anyhow::ensure!( + keys.insert(digest.to_string(), key).is_none(), + "duplicate sent key" + ); } Ok(keys) } @@ -1296,8 +1352,16 @@ pub fn collect_thread(catalog_root: &Path, filename: &str) -> anyhow::Result (from.to_string(), catalog_root.join(from)), - None => anyhow::bail!("no agent '{from}' found in catalog {}", catalog_root.display()), + None => anyhow::bail!( + "no agent '{from}' found in catalog {}", + catalog_root.display() + ), }; test_capability_checkpoint(); send_with_ledger( @@ -1451,14 +1517,8 @@ fn send_with_ledger( let recovered = recover_active(catalog_root, this_host, external, &root, &mut head)?; let filename = new_filename(); - let rendered_message = render_message_with_idempotency( - from, - subject, - in_reply_to, - tags, - body, - idempotency_key, - ); + let rendered_message = + render_message_with_idempotency(from, subject, in_reply_to, tags, body, idempotency_key); let parsed = parse_message(&filename, &rendered_message); let candidate = SentRecord { version: SENT_VERSION, @@ -1481,7 +1541,10 @@ fn send_with_ledger( .iter() .filter(|record| record.same_operation(&candidate)); if let Some(existing) = matching.next() { - anyhow::ensure!(matching.next().is_none(), "multiple recovered sends match one retry"); + anyhow::ensure!( + matching.next().is_none(), + "multiple recovered sends match one retry" + ); return Ok(existing.filename.clone()); } @@ -1530,8 +1593,14 @@ fn ensure_sent_head(root: &Path) -> anyhow::Result { } fn validate_sent_head(head: &SentHead) -> anyhow::Result<()> { - anyhow::ensure!(head.version == SENT_VERSION, "unsupported sent head version"); - anyhow::ensure!((head.count == 0) == head.tip.is_none(), "sent head count/tip mismatch"); + anyhow::ensure!( + head.version == SENT_VERSION, + "unsupported sent head version" + ); + anyhow::ensure!( + (head.count == 0) == head.tip.is_none(), + "sent head count/tip mismatch" + ); if let Some(tip) = &head.tip { anyhow::ensure!(is_sha256(tip), "invalid sent head tip"); } @@ -1544,17 +1613,35 @@ fn validate_sent_tip(root: &Path, head: &SentHead) -> anyhow::Result<()> { }; let node_path = root.join(SENT_COMMITS).join(format!("{digest}.json")); let node: SentCommit = serde_json::from_slice(&fs::read(node_path)?)?; - anyhow::ensure!(node.version == SENT_VERSION, "unsupported sent commit version"); - anyhow::ensure!(is_message_filename(&node.filename), "invalid sent commit filename"); - anyhow::ensure!(digest_json(&node)? == *digest, "sent commit digest mismatch"); + anyhow::ensure!( + node.version == SENT_VERSION, + "unsupported sent commit version" + ); + anyhow::ensure!( + is_message_filename(&node.filename), + "invalid sent commit filename" + ); + anyhow::ensure!( + digest_json(&node)? == *digest, + "sent commit digest mismatch" + ); anyhow::ensure!(node.ordinal == head.count, "sent commit ordinal mismatch"); let row_path = root .join(SENT_MESSAGES) .join(sent_record_name(&node.filename)); let row: SentRecord = serde_json::from_slice(&fs::read(row_path)?)?; - anyhow::ensure!(row.version == SENT_VERSION, "unsupported sent record version"); - anyhow::ensure!(row.filename == node.filename, "sent record filename does not match payload"); - anyhow::ensure!(digest_json(&row)? == node.row_digest, "sent row digest mismatch"); + anyhow::ensure!( + row.version == SENT_VERSION, + "unsupported sent record version" + ); + anyhow::ensure!( + row.filename == node.filename, + "sent record filename does not match payload" + ); + anyhow::ensure!( + digest_json(&row)? == node.row_digest, + "sent row digest mismatch" + ); Ok(()) } @@ -1578,11 +1665,7 @@ fn recover_active( "committed active intent differs from sender row" ); publish_key(root, &record)?; - remove_if_exists( - &root - .join(SENT_PENDING) - .join(pending_record_name(&record)?), - )?; + remove_if_exists(&root.join(SENT_PENDING).join(pending_record_name(&record)?))?; remove_if_exists(&root.join(SENT_ACTIVE))?; return Ok(Vec::new()); } @@ -1607,7 +1690,10 @@ fn recover_active( _ => unreachable!(), }; let recipient = resolve_delivery_endpoint(catalog_root, &record.to, this_host, external)?; - anyhow::ensure!(recipient.bus_id() == record.to, "pending recipient identity changed"); + anyhow::ensure!( + recipient.bus_id() == record.to, + "pending recipient identity changed" + ); deliver_record(&recipient, &record)?; publish_sent_record(root, &record)?; let node = publish_sent_commit(root, head, &record)?; @@ -1615,11 +1701,7 @@ fn recover_active( head.tip = Some(digest_json(&node)?); write_sent_head(root, head)?; publish_key(root, &record)?; - remove_if_exists( - &root - .join(SENT_PENDING) - .join(pending_record_name(&record)?), - )?; + remove_if_exists(&root.join(SENT_PENDING).join(pending_record_name(&record)?))?; remove_if_exists(&root.join(SENT_ACTIVE))?; Ok(vec![record]) } @@ -1645,7 +1727,10 @@ fn pending_record_name(record: &SentRecord) -> anyhow::Result { fn sent_commit(head: &SentHead, record: &SentRecord) -> anyhow::Result { Ok(SentCommit { version: SENT_VERSION, - ordinal: head.count.checked_add(1).context("sent commit count overflow")?, + ordinal: head + .count + .checked_add(1) + .context("sent commit count overflow")?, previous: head.tip.clone(), filename: record.filename.clone(), row_digest: digest_json(record)?, @@ -1679,22 +1764,30 @@ fn publish_sent_record(root: &Path, record: &SentRecord) -> anyhow::Result<()> { } fn read_sent_record(root: &Path, filename: &str) -> anyhow::Result { - anyhow::ensure!(is_message_filename(filename), "invalid sent record filename"); - let path = root - .join(SENT_MESSAGES) - .join(sent_record_name(filename)); + anyhow::ensure!( + is_message_filename(filename), + "invalid sent record filename" + ); + let path = root.join(SENT_MESSAGES).join(sent_record_name(filename)); let record: SentRecord = serde_json::from_slice(&fs::read(&path)?) .with_context(|| format!("reading sent record {}", path.display()))?; - anyhow::ensure!(record.version == SENT_VERSION, "unsupported sent record version"); - anyhow::ensure!(record.filename == filename, "sent record filename does not match payload"); + anyhow::ensure!( + record.version == SENT_VERSION, + "unsupported sent record version" + ); + anyhow::ensure!( + record.filename == filename, + "sent record filename does not match payload" + ); Ok(record) } fn sent_record_exists(root: &Path, filename: &str) -> anyhow::Result { - anyhow::ensure!(is_message_filename(filename), "invalid sent record filename"); - let path = root - .join(SENT_MESSAGES) - .join(sent_record_name(filename)); + anyhow::ensure!( + is_message_filename(filename), + "invalid sent record filename" + ); + let path = root.join(SENT_MESSAGES).join(sent_record_name(filename)); match fs::metadata(path) { Ok(metadata) => { anyhow::ensure!(metadata.is_file(), "sent record path is not a file"); @@ -1756,17 +1849,38 @@ fn keyed_record(root: &Path, candidate: &SentRecord) -> anyhow::Result return Err(error.into()), }; let receipt: SentKey = serde_json::from_slice(&bytes)?; - anyhow::ensure!(receipt.version == SENT_VERSION, "unsupported sent key version"); - anyhow::ensure!(receipt.to == candidate.to && receipt.key == key, "sent key scope mismatch"); - anyhow::ensure!(is_message_filename(&receipt.filename), "invalid sent key filename"); + anyhow::ensure!( + receipt.version == SENT_VERSION, + "unsupported sent key version" + ); + anyhow::ensure!( + receipt.to == candidate.to && receipt.key == key, + "sent key scope mismatch" + ); + anyhow::ensure!( + is_message_filename(&receipt.filename), + "invalid sent key filename" + ); let record_path = root .join(SENT_MESSAGES) .join(sent_record_name(&receipt.filename)); let record: SentRecord = serde_json::from_slice(&fs::read(record_path)?)?; - anyhow::ensure!(record.version == SENT_VERSION, "unsupported sent record version"); - anyhow::ensure!(record.filename == receipt.filename, "sent key record filename mismatch"); - anyhow::ensure!(digest_json(&record)? == receipt.record_digest, "sent key record mismatch"); - anyhow::ensure!(record.same_operation(candidate), "message idempotency key reused with different content"); + anyhow::ensure!( + record.version == SENT_VERSION, + "unsupported sent record version" + ); + anyhow::ensure!( + record.filename == receipt.filename, + "sent key record filename mismatch" + ); + anyhow::ensure!( + digest_json(&record)? == receipt.record_digest, + "sent key record mismatch" + ); + anyhow::ensure!( + record.same_operation(candidate), + "message idempotency key reused with different content" + ); Ok(Some(record)) } @@ -1776,9 +1890,18 @@ fn head_tip_commits(root: &Path, head: &SentHead, filename: &str) -> anyhow::Res }; let path = root.join(SENT_COMMITS).join(format!("{digest}.json")); let node: SentCommit = serde_json::from_slice(&fs::read(path)?)?; - anyhow::ensure!(node.version == SENT_VERSION, "unsupported sent commit version"); - anyhow::ensure!(is_message_filename(&node.filename), "invalid sent commit filename"); - anyhow::ensure!(digest_json(&node)? == *digest, "sent commit digest mismatch"); + anyhow::ensure!( + node.version == SENT_VERSION, + "unsupported sent commit version" + ); + anyhow::ensure!( + is_message_filename(&node.filename), + "invalid sent commit filename" + ); + anyhow::ensure!( + digest_json(&node)? == *digest, + "sent commit digest mismatch" + ); anyhow::ensure!(node.ordinal == head.count, "sent commit ordinal mismatch"); Ok(node.filename == filename) } diff --git a/src/park.rs b/src/park.rs index d3143aba..87f3d980 100644 --- a/src/park.rs +++ b/src/park.rs @@ -57,15 +57,17 @@ impl SupervisorScope { } fn in_state_root(state_root: &Path, catalog_root: &Path, host: &str) -> anyhow::Result { - let catalog_root = catalog_root - .canonicalize() - .with_context(|| format!("canonicalize supervisor catalog {}", catalog_root.display()))?; + let catalog_root = catalog_root.canonicalize().with_context(|| { + format!("canonicalize supervisor catalog {}", catalog_root.display()) + })?; let mut hash = Sha256::new(); hash.update(b"st2.supervisor-scope.v1"); hash_scope_component(&mut hash, catalog_root.as_os_str().as_bytes()); hash_scope_component(&mut hash, host.as_bytes()); let scope_id = format!("sha256-{:x}", hash.finalize()); - Ok(Self { root: state_root.join("st2/supervisors").join(scope_id) }) + Ok(Self { + root: state_root.join("st2/supervisors").join(scope_id), + }) } pub fn park_dir(&self) -> PathBuf { @@ -153,7 +155,9 @@ impl DirParkObserver { } pub fn for_supervisor(catalog_root: &Path, host: &str) -> anyhow::Result { - Ok(Self::new(SupervisorScope::current(catalog_root, host)?.park_dir())) + Ok(Self::new( + SupervisorScope::current(catalog_root, host)?.park_dir(), + )) } fn observe_with( @@ -477,8 +481,7 @@ mod tests { projection_a.publish(&parked(&["same.task"]), "crash-looped"); projection_b.publish(&BTreeSet::new(), "crash-looped"); - let marker_a = - DirParkObserver::new(channel_a.park_dir()).observe(&desired(&["same.task"])); + let marker_a = DirParkObserver::new(channel_a.park_dir()).observe(&desired(&["same.task"])); request_unpark(&channel_b.unpark_request_dir(), "same.task").unwrap(); let (taken_by_a, errors_a) = take_unpark_requests(&channel_a.unpark_request_dir()); @@ -489,7 +492,10 @@ mod tests { matches!(marker_a.state("same.task"), ParkState::Parked(_)), "catalog B deleted catalog A's same-host marker" ); - assert!(taken_by_a.is_empty(), "catalog A consumed catalog B's request"); + assert!( + taken_by_a.is_empty(), + "catalog A consumed catalog B's request" + ); assert_eq!(taken_by_b, ["same.task"]); } @@ -515,9 +521,16 @@ mod tests { let projection = projection(dir.path()); let observer = DirParkObserver::new(dir.path().to_path_buf()); - assert!(projection.publish(&parked(&["a", "b"]), "crash-looped").is_empty()); + assert!( + projection + .publish(&parked(&["a", "b"]), "crash-looped") + .is_empty() + ); let batch = observer.observe(&desired(&["a", "b", "healthy"])); - assert!(batch.complete, "a parked task is a known fault, not missing evidence"); + assert!( + batch.complete, + "a parked task is a known fault, not missing evidence" + ); assert!(batch.errors.is_empty()); let ParkState::Parked(record) = batch.state("a") else { panic!("'a' was published as parked but does not read back as parked"); @@ -529,7 +542,11 @@ mod tests { assert_eq!(batch.state("healthy"), &ParkState::NotParked); // 'a' recovers: republishing without it must retract its marker, not leave a fault standing. - assert!(projection.publish(&parked(&["b"]), "crash-looped").is_empty()); + assert!( + projection + .publish(&parked(&["b"]), "crash-looped") + .is_empty() + ); let batch = observer.observe(&desired(&["a", "b"])); assert_eq!(batch.state("a"), &ParkState::NotParked); assert!(matches!(batch.state("b"), ParkState::Parked(_))); @@ -559,7 +576,10 @@ mod tests { &ParkState::NotParked, "a park outlived the supervisor run it belongs to" ); - assert!(batch.complete, "a stale marker is a positive absence, not an unknown"); + assert!( + batch.complete, + "a stale marker is a positive absence, not an unknown" + ); assert!(batch.errors.is_empty()); } @@ -570,7 +590,11 @@ mod tests { fn a_supervisor_generation_observation_error_is_indeterminate() { let dir = tempfile::tempdir().unwrap(); let projection = projection(dir.path()); - assert!(projection.publish(&parked(&["a"]), "crash-looped").is_empty()); + assert!( + projection + .publish(&parked(&["a"]), "crash-looped") + .is_empty() + ); let observer = DirParkObserver::new(dir.path().to_path_buf()); let batch = observer.observe_with(&desired(&["a"]), &|_| { @@ -644,12 +668,18 @@ mod tests { ) .unwrap(); - let batch = - DirParkObserver::new(dir.path().to_path_buf()).observe(&desired(&["garbage", "wrong-schema"])); + let batch = DirParkObserver::new(dir.path().to_path_buf()) + .observe(&desired(&["garbage", "wrong-schema"])); assert!(!batch.complete); assert_eq!(batch.errors.len(), 2); - assert!(matches!(batch.state("garbage"), ParkState::Indeterminate(_))); - assert!(matches!(batch.state("wrong-schema"), ParkState::Indeterminate(_))); + assert!(matches!( + batch.state("garbage"), + ParkState::Indeterminate(_) + )); + assert!(matches!( + batch.state("wrong-schema"), + ParkState::Indeterminate(_) + )); } /// The park's age is what tells a fresh crash-loop from one that has been down all day, so a pass @@ -696,7 +726,15 @@ mod tests { #[test] fn a_request_id_cannot_escape_its_dir() { let dir = tempfile::tempdir().unwrap(); - for bad in ["../escaped", "sub/nested", "/absolute", "", ".", "..", ".hidden"] { + for bad in [ + "../escaped", + "sub/nested", + "/absolute", + "", + ".", + "..", + ".hidden", + ] { assert!( request_unpark(dir.path(), bad).is_err(), "{bad:?} was accepted as a task id" diff --git a/src/pretrust.rs b/src/pretrust.rs index cd0d3de4..53f71582 100644 --- a/src/pretrust.rs +++ b/src/pretrust.rs @@ -24,7 +24,8 @@ fn config_path() -> Result { if let Some(dir) = std::env::var_os("CLAUDE_CONFIG_DIR") { Ok(PathBuf::from(dir).join(".claude.json")) } else { - let home = std::env::var_os("HOME").context("neither $CLAUDE_CONFIG_DIR nor $HOME is set")?; + let home = + std::env::var_os("HOME").context("neither $CLAUDE_CONFIG_DIR nor $HOME is set")?; Ok(PathBuf::from(home).join(".claude.json")) } } @@ -65,7 +66,8 @@ fn codex_config_path() -> Result { pub fn pretrust_codex_at(config: &Path, dirs: &[PathBuf]) -> Result { let existing = std::fs::read_to_string(config).unwrap_or_default(); // Parse read-only just to see which dirs are already trusted (never rewrites the file). - let parsed: toml::Value = toml::from_str(&existing).unwrap_or(toml::Value::Table(Default::default())); + let parsed: toml::Value = + toml::from_str(&existing).unwrap_or(toml::Value::Table(Default::default())); let already = |dir: &str| -> bool { parsed .get("projects") @@ -82,7 +84,10 @@ pub fn pretrust_codex_at(config: &Path, dirs: &[PathBuf]) -> Result { if already(&key) { continue; } - appended.push_str(&format!("\n[projects.{}]\ntrust_level = \"trusted\"\n", toml_key(&key))); + appended.push_str(&format!( + "\n[projects.{}]\ntrust_level = \"trusted\"\n", + toml_key(&key) + )); n += 1; } if !appended.is_empty() { @@ -184,9 +189,15 @@ mod tests { assert_eq!(n, 1); let v: Value = serde_json::from_str(&std::fs::read_to_string(&cfg).unwrap()).unwrap(); - let key = std::fs::canonicalize(&ws).unwrap().to_string_lossy().into_owned(); + let key = std::fs::canonicalize(&ws) + .unwrap() + .to_string_lossy() + .into_owned(); assert_eq!(v["projects"][&key]["hasTrustDialogAccepted"], json!(true)); - assert_eq!(v["projects"][&key]["hasCompletedProjectOnboarding"], json!(true)); + assert_eq!( + v["projects"][&key]["hasCompletedProjectOnboarding"], + json!(true) + ); } #[test] @@ -195,7 +206,10 @@ mod tests { let cfg = tmp.path().join(".claude.json"); let ws = tmp.path().join("ws"); std::fs::create_dir_all(&ws).unwrap(); - let key = std::fs::canonicalize(&ws).unwrap().to_string_lossy().into_owned(); + let key = std::fs::canonicalize(&ws) + .unwrap() + .to_string_lossy() + .into_owned(); // Seed a config with a top-level key AND an existing project entry carrying extra fields. let seed = json!({ @@ -212,10 +226,25 @@ mod tests { let v: Value = serde_json::from_str(&std::fs::read_to_string(&cfg).unwrap()).unwrap(); // The trust flag flipped true; the sibling field and the other top-level/project keys survive. assert_eq!(v["projects"][&key]["hasTrustDialogAccepted"], json!(true)); - assert_eq!(v["projects"][&key]["hasCompletedProjectOnboarding"], json!(true)); - assert_eq!(v["projects"][&key]["lastCost"], json!(1.23), "existing field clobbered"); - assert_eq!(v["oauthAccount"]["keep"], json!("me"), "top-level key clobbered"); - assert_eq!(v["projects"]["/other/dir"]["hasTrustDialogAccepted"], json!(true), "sibling clobbered"); + assert_eq!( + v["projects"][&key]["hasCompletedProjectOnboarding"], + json!(true) + ); + assert_eq!( + v["projects"][&key]["lastCost"], + json!(1.23), + "existing field clobbered" + ); + assert_eq!( + v["oauthAccount"]["keep"], + json!("me"), + "top-level key clobbered" + ); + assert_eq!( + v["projects"]["/other/dir"]["hasTrustDialogAccepted"], + json!(true), + "sibling clobbered" + ); } #[test] @@ -224,24 +253,45 @@ mod tests { let cfg = tmp.path().join("config.toml"); let ws = tmp.path().join("ws"); std::fs::create_dir_all(&ws).unwrap(); - let key = std::fs::canonicalize(&ws).unwrap().to_string_lossy().into_owned(); + let key = std::fs::canonicalize(&ws) + .unwrap() + .to_string_lossy() + .into_owned(); // Existing codex config with a COMMENT and an unrelated project entry — must survive. std::fs::write(&cfg, "# my codex config\nmodel = \"gpt-5\"\n\n[projects.\"/other\"]\ntrust_level = \"trusted\"\n").unwrap(); - assert_eq!(pretrust_codex_at(&cfg, std::slice::from_ref(&ws)).unwrap(), 1); + assert_eq!( + pretrust_codex_at(&cfg, std::slice::from_ref(&ws)).unwrap(), + 1 + ); let text = std::fs::read_to_string(&cfg).unwrap(); assert!(text.contains("# my codex config"), "comment clobbered"); - assert!(text.contains("model = \"gpt-5\""), "top-level setting clobbered"); - assert!(text.contains("[projects.\"/other\"]"), "existing project clobbered"); + assert!( + text.contains("model = \"gpt-5\""), + "top-level setting clobbered" + ); + assert!( + text.contains("[projects.\"/other\"]"), + "existing project clobbered" + ); // The new dir is trusted, and the whole file still parses as TOML. let v: toml::Value = toml::from_str(&text).unwrap(); assert_eq!(v["projects"][&key]["trust_level"].as_str(), Some("trusted")); // Idempotent: re-trusting writes nothing (already trusted) and doesn't duplicate the table. - assert_eq!(pretrust_codex_at(&cfg, std::slice::from_ref(&ws)).unwrap(), 0); + assert_eq!( + pretrust_codex_at(&cfg, std::slice::from_ref(&ws)).unwrap(), + 0 + ); let again = std::fs::read_to_string(&cfg).unwrap(); - assert_eq!(again.matches(&format!("[projects.{}]", toml_key(&key))).count(), 1, "duplicated table"); + assert_eq!( + again + .matches(&format!("[projects.{}]", toml_key(&key))) + .count(), + 1, + "duplicated table" + ); } #[test] @@ -259,7 +309,10 @@ mod tests { let v: Value = serde_json::from_str(&std::fs::read_to_string(&cfg).unwrap()).unwrap(); for d in [&a, &b] { - let key = std::fs::canonicalize(d).unwrap().to_string_lossy().into_owned(); + let key = std::fs::canonicalize(d) + .unwrap() + .to_string_lossy() + .into_owned(); assert_eq!(v["projects"][&key]["hasTrustDialogAccepted"], json!(true)); } } diff --git a/src/reconcile.rs b/src/reconcile.rs index 829636c4..a537db5f 100644 --- a/src/reconcile.rs +++ b/src/reconcile.rs @@ -15,7 +15,9 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; -use agent_spec::spec::{AgentSpec, DeliveryTransport, Driver, TaskKind, TaskLifecycle}; +use agent_spec::spec::{ + AgentSpec, DeliveryTransport, Driver, TaskKind, TaskLifecycle, stream_name_of_task, +}; use kdl::KdlValue; /// Immutable inputs captured once before generated tasks are compiled. @@ -173,9 +175,9 @@ pub fn compile_driver_agent_tasks( .tasks .iter_mut() .filter(|task| !task.derived && task.name == "agent"); - let task = candidates.next().with_context(|| { - format!("agent '{bus_id}' driver has no canonical `agent` task") - })?; + let task = candidates + .next() + .with_context(|| format!("agent '{bus_id}' driver has no canonical `agent` task"))?; anyhow::ensure!( candidates.next().is_none(), "agent '{bus_id}' driver has more than one canonical `agent` task" @@ -303,8 +305,13 @@ fn compile_session_wrapped_agent_tasks( Ok(()) } -/// Replace only runner-generated DING markers with exact direct argv. Authored tasks never carry -/// `derived=true`, so source that happens to invoke `st2 ding` remains byte-for-byte unchanged. +/// Replace only runner-generated companion markers with exact direct argv. Authored tasks never +/// carry `derived=true`, so source that happens to invoke `st2 ding` or `st2 stream run` remains +/// byte-for-byte unchanged. +/// +/// The single `ensure!` below is the fail-closed "unsupported derived task" gate. Every derived +/// companion kind must be named in the exhaustive match; an unrecognized derived task refuses the +/// pass rather than reaching a runner with an unbound placeholder command. pub fn compile_generated_ding_tasks( specs: &mut [AgentSpec], this_host: &str, @@ -321,9 +328,11 @@ pub fn compile_generated_ding_tasks( if !task.derived { continue; } + let is_ding = task.name == "ding" || task.name.ends_with(".ding"); + let task_name = task.name.clone(); + let stream_name = stream_name_of_task(&task_name).map(str::to_owned); anyhow::ensure!( - task.kind == TaskKind::Exec - && (task.name == "ding" || task.name.ends_with(".ding")), + task.kind == TaskKind::Exec && (is_ding || stream_name.is_some()), "unsupported derived task: {}", task.name ); @@ -334,8 +343,15 @@ pub fn compile_generated_ding_tasks( .unwrap_or_else(|| context.catalog_root.display().to_string()); anyhow::ensure!( Path::new(&effective_root).is_absolute(), - "derived DING root is not absolute: {effective_root}" + "derived companion root is not absolute: {effective_root}" ); + if let Some(stream_name) = stream_name { + anyhow::ensure!( + task.command.is_some() != task.argv.is_some(), + "derived stream task '{task_name}' for stream '{stream_name}' has no exact launch" + ); + continue; + } task.command = None; task.argv = Some(vec![ st2_executable.clone(), @@ -600,10 +616,7 @@ fn pty_presentation( }) } -fn presentation_matches( - desired: &PtyPresentation, - observed: &ObservedPtyPresentation, -) -> bool { +fn presentation_matches(desired: &PtyPresentation, observed: &ObservedPtyPresentation) -> bool { let display_name_matches = desired .display_name .as_ref() @@ -896,13 +909,13 @@ pub fn reconcile<'a>( let agent_eligible = targets .iter() .find(|(target, _)| target.name == "agent" && !target.derived) - .is_some_and(|(target, lifecycle)| match session_state(&by_id, &target.pty_id) { - SessionState::Alive => true, - SessionState::Dead => { - !target.keep && *lifecycle == TaskLifecycle::Service - } - SessionState::Absent => *lifecycle == TaskLifecycle::Service, - }); + .is_some_and( + |(target, lifecycle)| match session_state(&by_id, &target.pty_id) { + SessionState::Alive => true, + SessionState::Dead => !target.keep && *lifecycle == TaskLifecycle::Service, + SessionState::Absent => *lifecycle == TaskLifecycle::Service, + }, + ); let mut to_launch = Vec::new(); let mut live_derived = Vec::new(); let mut ineligible_derived = Vec::new(); diff --git a/src/run.rs b/src/run.rs index f58f0888..edfdf808 100644 --- a/src/run.rs +++ b/src/run.rs @@ -1110,9 +1110,15 @@ pub fn grant_unpark_requests(cap: &mut FlappingCap, request_dir: &Path, report: /// /// Supervisor loops only, for the same reason as [`grant_unpark_requests`]: publishing an empty /// one-shot cap would wipe the running supervisor's projection and hide every live park. -pub fn publish_parks(cap: &FlappingCap, projection: &crate::park::ParkProjection, report: &mut UpReport) { +pub fn publish_parks( + cap: &FlappingCap, + projection: &crate::park::ParkProjection, + report: &mut UpReport, +) { let parked: std::collections::BTreeSet = cap.parked_ids().cloned().collect(); - report.errors.extend(projection.publish(&parked, PARK_REASON)); + report + .errors + .extend(projection.publish(&parked, PARK_REASON)); } /// A supervisor loop's end of the park channel: the projection it publishes and the request dir it @@ -1135,7 +1141,10 @@ impl ParkChannel { eprintln!( "st2: cannot open the supervisor park channel ({error}); parks remain terminal but cannot be observed or explicitly released." ); - return Self { projection: None, request_dir: None }; + return Self { + projection: None, + request_dir: None, + }; } }; let projection = match crate::park::ParkProjection::current(scope.park_dir()) { @@ -1147,7 +1156,10 @@ impl ParkChannel { None } }; - Self { projection, request_dir: Some(scope.unpark_request_dir()) } + Self { + projection, + request_dir: Some(scope.unpark_request_dir()), + } } fn grant_requests(&self, cap: &mut FlappingCap, report: &mut UpReport) { @@ -1189,9 +1201,9 @@ fn stop_live_derived_companions( for companion_id in &launch.live_derived { match runner.kill(companion_id) { Ok(()) => report.torn_down.push(companion_id.clone()), - Err(error) => report - .errors - .push(format!("kill unavailable derived companion {companion_id}: {error}")), + Err(error) => report.errors.push(format!( + "kill unavailable derived companion {companion_id}: {error}" + )), } } } @@ -2087,7 +2099,8 @@ pub fn surface_crash_loop(catalog_root: &Path, this_host: &str, cl: &CrashLoop) ); return; }; - let Ok(Some(agent_dir)) = message::resolve_agent_dir(catalog_root, supervisor, this_host) else { + let Ok(Some(agent_dir)) = message::resolve_agent_dir(catalog_root, supervisor, this_host) + else { eprintln!( "st2: crash-loop '{}': supervisor '{supervisor}' not found in the catalog to notify.", cl.pty_id @@ -2290,6 +2303,7 @@ mod tests { delivery: None, driver: None, resources: vec![], + streams: Vec::new(), tasks: vec![Task { kind: TaskKind::Pty, derived: false, @@ -2341,6 +2355,7 @@ mod tests { delivery: None, driver: None, resources: vec![], + streams: Vec::new(), tasks: vec![Task { kind: TaskKind::Pty, derived: false, @@ -2599,6 +2614,7 @@ mod tests { delivery: None, driver: None, resources: vec![], + streams: Vec::new(), tasks: vec![], path: std::path::PathBuf::from("/x"), } @@ -2873,19 +2889,14 @@ mod tests { let cli = PtyCli::default(); let mut t = target("hetz.demo", "codex"); t.bus_id = "hetz.demo".to_owned(); - t.tags.insert("unrelated".to_owned(), "preserved".to_owned()); + t.tags + .insert("unrelated".to_owned(), "preserved".to_owned()); t.presentation = Some(PtyPresentation { pty_id: "hetz.demo".to_owned(), display_name: Some(Some("Build owner".to_owned())), tags: BTreeMap::from([ - ( - "agent.presentation.schema".to_owned(), - Some("1".to_owned()), - ), - ( - "agent.actor.path".to_owned(), - Some("hetz.demo".to_owned()), - ), + ("agent.presentation.schema".to_owned(), Some("1".to_owned())), + ("agent.actor.path".to_owned(), Some("hetz.demo".to_owned())), ( "agent.presentation.description".to_owned(), Some(format!("${key}")), @@ -2931,10 +2942,7 @@ mod tests { pty_id: "stable.agent.id".to_owned(), display_name: Some(None), tags: BTreeMap::from([ - ( - "agent.presentation.schema".to_owned(), - Some("1".to_owned()), - ), + ("agent.presentation.schema".to_owned(), Some("1".to_owned())), ("agent.presentation.description".to_owned(), None), ]), }; @@ -2945,10 +2953,9 @@ mod tests { std::fs::read_to_string(executable.with_extension("args")).unwrap(), "metadata\npatch\n--id\nstable.agent.id\n" ); - let payload: serde_json::Value = serde_json::from_slice( - &std::fs::read(executable.with_extension("stdin")).unwrap(), - ) - .unwrap(); + let payload: serde_json::Value = + serde_json::from_slice(&std::fs::read(executable.with_extension("stdin")).unwrap()) + .unwrap(); assert_eq!(payload["displayName"], serde_json::Value::Null); assert_eq!(payload["tags"]["agent.presentation.schema"], "1"); assert_eq!( @@ -3120,7 +3127,10 @@ mod tests { use std::os::fd::{FromRawFd as _, OwnedFd}; let mut pipe_fds = [0; 2]; - assert_eq!(unsafe { libc::pipe2(pipe_fds.as_mut_ptr(), libc::O_CLOEXEC) }, 0); + assert_eq!( + unsafe { libc::pipe2(pipe_fds.as_mut_ptr(), libc::O_CLOEXEC) }, + 0 + ); let reader = unsafe { OwnedFd::from_raw_fd(pipe_fds[0]) }; let writer = unsafe { OwnedFd::from_raw_fd(pipe_fds[1]) }; let pipe = std::fs::read_link(format!("/proc/self/fd/{}", reader.as_raw_fd())).unwrap(); @@ -3154,7 +3164,10 @@ mod tests { use std::os::fd::{FromRawFd as _, OwnedFd}; let mut pipe_fds = [0; 2]; - assert_eq!(unsafe { libc::pipe2(pipe_fds.as_mut_ptr(), libc::O_CLOEXEC) }, 0); + assert_eq!( + unsafe { libc::pipe2(pipe_fds.as_mut_ptr(), libc::O_CLOEXEC) }, + 0 + ); let _reader = unsafe { OwnedFd::from_raw_fd(pipe_fds[0]) }; let writer = unsafe { OwnedFd::from_raw_fd(pipe_fds[1]) }; @@ -3729,7 +3742,10 @@ printf '%s\n' '[{"name":"h.live","status":"running","pid":41,"createdAt":"2026-0 let presentation = sessions[0].presentation.as_ref().unwrap(); assert_eq!(presentation.display_name.as_deref(), Some("Build owner")); assert_eq!( - presentation.tags.get("agent.presentation.schema").map(String::as_str), + presentation + .tags + .get("agent.presentation.schema") + .map(String::as_str), Some("1") ); assert_eq!( diff --git a/src/service.rs b/src/service.rs index 5de29818..40b1ec58 100644 --- a/src/service.rs +++ b/src/service.rs @@ -187,7 +187,10 @@ fn install_systemd_user(_spec: &ServiceSpec) -> Result<()> { #[cfg(target_os = "linux")] fn status_systemd_user() -> Result<()> { - run_command("systemctl", &["--user", "status", SERVICE_NAME, "--no-pager"]) + run_command( + "systemctl", + &["--user", "status", SERVICE_NAME, "--no-pager"], + ) } #[cfg(not(target_os = "linux"))] @@ -238,7 +241,9 @@ fn run_command(program: &str, args: &[&str]) -> Result<()> { #[cfg(target_os = "linux")] fn home_dir() -> Result { - env::var_os("HOME").map(PathBuf::from).context("HOME is not set") + env::var_os("HOME") + .map(PathBuf::from) + .context("HOME is not set") } #[cfg(target_os = "linux")] @@ -340,9 +345,9 @@ mod tests { assert!(unit.contains("RestartSec=5s")); assert!(unit.contains("MemoryMax=1024M")); assert!(unit.contains("WorkingDirectory=/home/user/catalog")); - assert!(unit.contains( - "Environment=PATH=/home/user/.cargo/bin:/home/user/.local/bin:/usr/bin" - )); + assert!( + unit.contains("Environment=PATH=/home/user/.cargo/bin:/home/user/.local/bin:/usr/bin") + ); assert!(!unit.contains("Environment=PTY_ROOT=")); assert!(unit.contains("WantedBy=default.target")); assert!(unit.contains("Description=st2 supervisor (st2 up)")); @@ -383,9 +388,7 @@ mod tests { let unit = render_systemd_user_unit(&spec); - assert!( - unit.contains("ExecStart=\"/opt/st2 tools/st2\" up --catalog \"/srv/cat 100%%\"") - ); + assert!(unit.contains("ExecStart=\"/opt/st2 tools/st2\" up --catalog \"/srv/cat 100%%\"")); assert!(unit.contains("WorkingDirectory=\"/srv/cat 100%%\"")); assert!(unit.contains("Environment=\"PATH=/opt/st2 tools:/usr/bin\"")); assert!(unit.contains("Environment=\"PTY_ROOT=/srv/pty 100%%\"")); diff --git a/src/task_inventory.rs b/src/task_inventory.rs index 067c744a..237ce2a4 100644 --- a/src/task_inventory.rs +++ b/src/task_inventory.rs @@ -400,7 +400,10 @@ pub fn inventory( push_error(&mut errors, error.clone()); } if !parks.complete && parks.errors.is_empty() { - push_error(&mut errors, "park projection reported an incomplete batch".into()); + push_error( + &mut errors, + "park projection reported an incomplete batch".into(), + ); } desired.sort_by(|a, b| { diff --git a/src/validate.rs b/src/validate.rs index 72b2b7b5..c1e784df 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -462,6 +462,9 @@ fn kdl_shape_check(root: &Path, path: &Path, parsed: &DeclaredParse) -> Vec "unsupported-schedule", + DeclaredDiagnosticCode::UnsupportedStreamInterval => { + "unsupported-stream-interval" + } DeclaredDiagnosticCode::UnexpectedTopLevelNode => { "unexpected-top-level-node" } diff --git a/tests/agent_desired_state.rs b/tests/agent_desired_state.rs index 8b808561..3c6f40c1 100644 --- a/tests/agent_desired_state.rs +++ b/tests/agent_desired_state.rs @@ -55,7 +55,11 @@ fn cli_suspends_resumes_and_retires_without_rewriting_unrelated_source() { write(root, "h/worker/agent.kdl", initial); let suspended = author(root, "suspended", Some("Waiting for capacity")); - assert!(suspended.status.success(), "{}", String::from_utf8_lossy(&suspended.stderr)); + assert!( + suspended.status.success(), + "{}", + String::from_utf8_lossy(&suspended.stderr) + ); let receipt: serde_json::Value = serde_json::from_slice(&suspended.stdout).unwrap(); assert_eq!(receipt["result"], "changed"); assert_eq!(receipt["desired_state"], "suspended"); @@ -67,18 +71,31 @@ fn cli_suspends_resumes_and_retires_without_rewriting_unrelated_source() { let repeat = author(root, "suspended", Some("Waiting for capacity")); assert!(repeat.status.success()); - assert_eq!(serde_json::from_slice::(&repeat.stdout).unwrap()["result"], "unchanged"); + assert_eq!( + serde_json::from_slice::(&repeat.stdout).unwrap()["result"], + "unchanged" + ); let running = author(root, "running", None); - assert!(running.status.success(), "{}", String::from_utf8_lossy(&running.stderr)); - assert_eq!(fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(), initial); + assert!( + running.status.success(), + "{}", + String::from_utf8_lossy(&running.stderr) + ); + assert_eq!( + fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(), + initial + ); let retired = author(root, "retired", Some("Mission complete")); assert!(retired.status.success()); let found = st2::discover(root); assert!(found.errors.is_empty(), "{:?}", found.errors); assert!(found.specs[0].desired_state.is_retired()); - assert_eq!(found.specs[0].desired_state.reason(), Some("Mission complete")); + assert_eq!( + found.specs[0].desired_state.reason(), + Some("Mission complete") + ); } #[test] @@ -89,7 +106,11 @@ fn cli_resume_preserves_same_line_leading_comment() { write(root, "h/worker/agent.kdl", initial); let running = author(root, "running", None); - assert!(running.status.success(), "{}", String::from_utf8_lossy(&running.stderr)); + assert!( + running.status.success(), + "{}", + String::from_utf8_lossy(&running.stderr) + ); assert_eq!( fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(), "agent \"worker\" {\n host \"h\"\n /* operator note */\n command \"true\"\n}\n" @@ -104,7 +125,11 @@ fn cli_authors_a_canonical_path_derived_identity() { write(root, "h/worker/agent.kdl", initial); let suspended = author(root, "suspended", Some("Waiting for capacity")); - assert!(suspended.status.success(), "{}", String::from_utf8_lossy(&suspended.stderr)); + assert!( + suspended.status.success(), + "{}", + String::from_utf8_lossy(&suspended.stderr) + ); assert!( fs::read_to_string(root.join("h/worker/agent.kdl")) .unwrap() @@ -112,8 +137,15 @@ fn cli_authors_a_canonical_path_derived_identity() { ); let running = author(root, "running", None); - assert!(running.status.success(), "{}", String::from_utf8_lossy(&running.stderr)); - assert_eq!(fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(), initial); + assert!( + running.status.success(), + "{}", + String::from_utf8_lossy(&running.stderr) + ); + assert_eq!( + fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(), + initial + ); } #[test] @@ -130,8 +162,14 @@ fn cli_rejects_invalid_reason_contract_without_mutation() { ("suspended", Some(" surrounding ")), ] { let output = author(root, state, reason); - assert!(!output.status.success(), "{state} {reason:?} unexpectedly succeeded"); - assert_eq!(fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(), initial); + assert!( + !output.status.success(), + "{state} {reason:?} unexpectedly succeeded" + ); + assert_eq!( + fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(), + initial + ); } } @@ -145,7 +183,11 @@ fn cli_canonicalizes_legacy_retirement_and_refuses_nix_owned_declarations() { "agent \"worker\" { host \"h\"; retired #true; command \"true\" }\n", ); let output = author(root, "suspended", Some("May return")); - assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); let authored = fs::read_to_string(root.join("h/worker/agent.kdl")).unwrap(); assert!(!authored.contains("retired")); assert!(authored.contains("desired-state \"suspended\" reason=\"May return\"")); @@ -182,13 +224,20 @@ fn cli_applies_the_existing_self_or_descendant_authority_guardrail() { ); let allowed = author_as(root, "h.root"); - assert!(allowed.status.success(), "{}", String::from_utf8_lossy(&allowed.stderr)); + assert!( + allowed.status.success(), + "{}", + String::from_utf8_lossy(&allowed.stderr) + ); assert!(author(root, "running", None).status.success()); let refused = author_as(root, "h.sibling"); assert!(!refused.status.success()); let receipt: serde_json::Value = serde_json::from_slice(&refused.stdout).unwrap(); assert_eq!(receipt["code"], "desired-state-not-authorized"); - assert!(st2::discover(root).specs - .iter() - .any(|spec| spec.identity == "worker" && spec.desired_state.is_running())); + assert!( + st2::discover(root) + .specs + .iter() + .any(|spec| spec.identity == "worker" && spec.desired_state.is_running()) + ); } diff --git a/tests/catalog_apply.rs b/tests/catalog_apply.rs index 55b7385d..c23a8d0b 100644 --- a/tests/catalog_apply.rs +++ b/tests/catalog_apply.rs @@ -412,10 +412,9 @@ fn apply_input_fence_survives_source_free_crash_resume() { .output() .unwrap(); assert!(!interrupted.status.success()); - let marker: Value = serde_json::from_slice( - &fs::read(catalog.join(".st2/catalog-apply-incomplete")).unwrap(), - ) - .unwrap(); + let marker: Value = + serde_json::from_slice(&fs::read(catalog.join(".st2/catalog-apply-incomplete")).unwrap()) + .unwrap(); assert_eq!(marker["preparedRootSha256"], input_sha256); fs::remove_dir_all(&prepared).unwrap(); @@ -459,11 +458,7 @@ fn bootstrap_atomically_publishes_an_absent_catalog_and_replays_exactly() { let captured = snapshot(&source, &prepared); let target = temp.path().join("target"); - let first = bootstrap( - &target, - &prepared, - captured["rootSha256"].as_str().unwrap(), - ); + let first = bootstrap(&target, &prepared, captured["rootSha256"].as_str().unwrap()); assert!( first.status.success(), "{}", @@ -487,11 +482,7 @@ fn bootstrap_atomically_publishes_an_absent_catalog_and_replays_exactly() { "state survives replay", ) .unwrap(); - let replay = bootstrap( - &target, - &prepared, - captured["rootSha256"].as_str().unwrap(), - ); + let replay = bootstrap(&target, &prepared, captured["rootSha256"].as_str().unwrap()); assert!( replay.status.success(), "{}", @@ -500,10 +491,8 @@ fn bootstrap_atomically_publishes_an_absent_catalog_and_replays_exactly() { let replay: Value = serde_json::from_slice(&replay.stdout).unwrap(); assert_eq!(replay["status"], "unchanged"); assert_eq!( - fs::read_to_string( - agent_dir(&target, "worker").join("resources/inbox/message.md") - ) - .unwrap(), + fs::read_to_string(agent_dir(&target, "worker").join("resources/inbox/message.md")) + .unwrap(), "state survives replay" ); } @@ -528,11 +517,7 @@ fn bootstrap_rejects_a_different_existing_catalog_without_mutation() { assert!(created.status.success()); let before = fs::read_to_string(agent_dir(&target, "incumbent").join("agent.kdl")).unwrap(); - let rejected = bootstrap( - &target, - &prepared, - captured["rootSha256"].as_str().unwrap(), - ); + let rejected = bootstrap(&target, &prepared, captured["rootSha256"].as_str().unwrap()); assert!(!rejected.status.success()); assert!( String::from_utf8_lossy(&rejected.stderr).contains("already exists with root sha256"), @@ -575,11 +560,7 @@ fn bootstrap_requires_an_explicit_external_pty_root_before_publication() { assert!(captured.status.success()); let captured: Value = serde_json::from_slice(&captured.stdout).unwrap(); let target = temp.path().join(format!("target-{case}")); - let rejected = bootstrap( - &target, - &prepared, - captured["rootSha256"].as_str().unwrap(), - ); + let rejected = bootstrap(&target, &prepared, captured["rootSha256"].as_str().unwrap()); assert!(!rejected.status.success()); assert!(!target.exists()); let stderr = String::from_utf8_lossy(&rejected.stderr); @@ -696,7 +677,7 @@ fn bootstrap_crash_boundaries_replay_from_absent_or_complete_only() { .env( "ST2_TEST_CATALOG_BOOTSTRAP_CRASH_AT", "after-publish-before-parent-sync", - ) + ) .output() .unwrap(); assert!(!after.status.success()); @@ -837,11 +818,7 @@ fn bootstrap_never_touches_the_external_pty_root() { let prepared = temp.path().join("prepared"); let captured = snapshot(&source, &prepared); let target = temp.path().join("target"); - let output = bootstrap( - &target, - &prepared, - captured["rootSha256"].as_str().unwrap(), - ); + let output = bootstrap(&target, &prepared, captured["rootSha256"].as_str().unwrap()); assert!(output.status.success()); assert_eq!( fs::read_to_string(pty_root.join("sentinel")).unwrap(), @@ -881,11 +858,7 @@ fn bootstrap_composes_with_the_next_root_cas_apply_generation() { let prepared = temp.path().join("prepared"); let captured = snapshot(&source, &prepared); let target = temp.path().join("target"); - let created = bootstrap( - &target, - &prepared, - captured["rootSha256"].as_str().unwrap(), - ); + let created = bootstrap(&target, &prepared, captured["rootSha256"].as_str().unwrap()); assert!(created.status.success()); let update = temp.path().join("update"); @@ -895,11 +868,7 @@ fn bootstrap_composes_with_the_next_root_cas_apply_generation() { "agent \"worker\" { host \"host\"; role \"updated\"; argv \"true\" }\n", ) .unwrap(); - let applied = apply( - &target, - &update, - before["rootSha256"].as_str().unwrap(), - ); + let applied = apply(&target, &update, before["rootSha256"].as_str().unwrap()); assert!( applied.status.success(), "{}", @@ -1366,13 +1335,13 @@ fn raw_preimage_refuses_valid_catalogs_and_wrong_cas_without_declaration_writes( "unfinished writer bytes" ); assert!(!invalid.join(".st2/catalog-apply-incomplete").exists()); - assert!(fs::read_dir(invalid.join(".st2")) - .unwrap() - .all(|entry| !entry + assert!(fs::read_dir(invalid.join(".st2")).unwrap().all(|entry| { + !entry .unwrap() .file_name() .to_string_lossy() - .starts_with("catalog-apply-stage-"))); + .starts_with("catalog-apply-stage-") + })); } #[test] @@ -1394,13 +1363,12 @@ fn raw_preimage_requires_a_readable_envelope_and_an_unchanged_pty_root() { let malformed_envelope = temp.path().join("malformed-envelope"); write_invalid_agent(&malformed_envelope, "worker"); fs::write(malformed_envelope.join("catalog.kdl"), "catalog {").unwrap(); - let rejected = raw_snapshot( - &malformed_envelope, - &temp.path().join("malformed-capture"), - ); + let rejected = raw_snapshot(&malformed_envelope, &temp.path().join("malformed-capture")); assert!(!rejected.status.success()); - assert!(String::from_utf8_lossy(&rejected.stderr) - .contains("requires a valid incumbent catalog envelope")); + assert!( + String::from_utf8_lossy(&rejected.stderr) + .contains("requires a valid incumbent catalog envelope") + ); let catalog = temp.path().join("catalog"); write_invalid_agent(&catalog, "worker"); @@ -1425,8 +1393,9 @@ fn raw_preimage_requires_a_readable_envelope_and_an_unchanged_pty_root() { raw_capture["rootSha256"].as_str().unwrap(), ); assert!(!rejected.status.success()); - assert!(String::from_utf8_lossy(&rejected.stderr) - .contains("refuses an effective pty-root change")); + assert!( + String::from_utf8_lossy(&rejected.stderr).contains("refuses an effective pty-root change") + ); assert_eq!( fs::read(agent_dir(&catalog, "worker").join("agent.kdl")).unwrap(), declaration diff --git a/tests/catalog_config.rs b/tests/catalog_config.rs index 509368eb..1b3ae870 100644 --- a/tests/catalog_config.rs +++ b/tests/catalog_config.rs @@ -46,15 +46,25 @@ fn a_declared_pty_root_replaces_the_catalog_default_in_the_bus_env() { ) .unwrap(); - let out = st2(&["env", "--catalog", declared.to_str().unwrap()], tmp.path()); - assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let out = st2( + &["env", "--catalog", declared.to_str().unwrap()], + tmp.path(), + ); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); let env = String::from_utf8_lossy(&out.stdout); assert!( env.contains(&format!("export PTY_ROOT={}\n", shared.display())), "declared root missing from the bus env:\n{env}" ); assert!( - env.contains(&format!("export ST_ROOT={}\n", declared.canonicalize().unwrap().display())), + env.contains(&format!( + "export ST_ROOT={}\n", + declared.canonicalize().unwrap().display() + )), "the flat native ST_ROOT must be unaffected:\n{env}" ); diff --git a/tests/codex_app_server.rs b/tests/codex_app_server.rs index a4757802..0e5e47ff 100644 --- a/tests/codex_app_server.rs +++ b/tests/codex_app_server.rs @@ -102,18 +102,8 @@ fn codex_driver_matches_deliver_after_normalizing_only_the_subcommand_alias() { tmp.path() )); - compile_generated_tasks( - std::slice::from_mut(&mut legacy), - "h", - &compile_context, - ) - .unwrap(); - compile_generated_tasks( - std::slice::from_mut(&mut driver), - "h", - &compile_context, - ) - .unwrap(); + compile_generated_tasks(std::slice::from_mut(&mut legacy), "h", &compile_context).unwrap(); + compile_generated_tasks(std::slice::from_mut(&mut driver), "h", &compile_context).unwrap(); let legacy_task = legacy .tasks diff --git a/tests/doctor.rs b/tests/doctor.rs index 45187f24..b0da7b42 100644 --- a/tests/doctor.rs +++ b/tests/doctor.rs @@ -327,8 +327,15 @@ fn suspended_declaration_is_healthy_when_tasks_are_absent_without_presence() { let output = doctor(&catalog, &bin, &tmp.path().join("state")); let stdout = String::from_utf8_lossy(&output.stdout); - assert!(output.status.success(), "stdout:\n{stdout}\nstderr:\n{}", String::from_utf8_lossy(&output.stderr)); - assert!(stdout.contains("✓ h.idle suspension effective (no live tasks)"), "{stdout}"); + assert!( + output.status.success(), + "stdout:\n{stdout}\nstderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + stdout.contains("✓ h.idle suspension effective (no live tasks)"), + "{stdout}" + ); assert!(!stdout.contains("h.idle presence"), "{stdout}"); } @@ -353,8 +360,15 @@ fn suspended_declaration_distinguishes_live_dead_keep_and_dead_nonkeep() { let output = doctor(&catalog, &bin, &tmp.path().join("state")); let stdout = String::from_utf8_lossy(&output.stdout); - assert_eq!(output.status.success(), healthy, "status={status} keep={keep}\n{stdout}"); - assert!(stdout.contains("h.idle suspension effective (no live tasks)"), "{stdout}"); + assert_eq!( + output.status.success(), + healthy, + "status={status} keep={keep}\n{stdout}" + ); + assert!( + stdout.contains("h.idle suspension effective (no live tasks)"), + "{stdout}" + ); if !detail.is_empty() { assert!(stdout.contains(detail), "{stdout}"); } @@ -423,5 +437,8 @@ fn missing_delivery_is_advisory_while_an_invalid_delivery_is_a_catalog_problem() let invalid = doctor(&catalog, &bin, &tmp.path().join("state")); let stdout = String::from_utf8_lossy(&invalid.stdout); assert!(!invalid.status.success(), "{stdout}"); - assert!(stdout.contains("unsupported `deliver` value 'mpc'"), "{stdout}"); + assert!( + stdout.contains("unsupported `deliver` value 'mpc'"), + "{stdout}" + ); } diff --git a/tests/driver_expansion.rs b/tests/driver_expansion.rs index b291532d..1e86bb84 100644 --- a/tests/driver_expansion.rs +++ b/tests/driver_expansion.rs @@ -15,7 +15,9 @@ fn assert_snapshot(input: &str, expected: &str) { let found = discover(temp.path()); assert!(found.errors.is_empty(), "{:?}", found.errors); assert_eq!(found.specs.len(), 1); - let actual = expand_driver(&found.specs[0], "unused").unwrap().to_string(); + let actual = expand_driver(&found.specs[0], "unused") + .unwrap() + .to_string(); assert_eq!(actual, expected); expected.parse::().unwrap(); } @@ -209,11 +211,7 @@ fn claude_driver_matches_deliver_after_normalizing_the_legacy_command_namespace( let (driver, _) = st2::discover_file(&catalog, &driver_path).unwrap(); let mut legacy = legacy.into_iter().next().unwrap(); let mut driver = driver.into_iter().next().unwrap(); - assert!(!st2::hooks::required_by_codex_agent( - &driver, - "h", - &catalog - )); + assert!(!st2::hooks::required_by_codex_agent(&driver, "h", &catalog)); let executable = catalog.join("bin/st2"); fs::create_dir_all(executable.parent().unwrap()).unwrap(); fs::write(&executable, "test binary").unwrap(); @@ -390,7 +388,11 @@ fn ambiguous_driver_source_neither_compiles_nor_materializes() { let compile_error = compile_generated_tasks(std::slice::from_mut(&mut spec), "h", &context).unwrap_err(); - assert!(compile_error.to_string().contains("choose one launch source")); + assert!( + compile_error + .to_string() + .contains("choose one launch source") + ); assert_eq!(spec, before); let materialize_error = materialize_agent(&catalog, &spec, "h").unwrap_err(); assert!( diff --git a/tests/eval_run_e2e.rs b/tests/eval_run_e2e.rs index 76c28b62..9b819eef 100644 --- a/tests/eval_run_e2e.rs +++ b/tests/eval_run_e2e.rs @@ -7,12 +7,16 @@ //! Needs `pty` on PATH — HARD failure if absent unless ST2_ALLOW_PTY_SKIP is set. use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; fn pty_available() -> bool { - Command::new("pty").arg("--help").output().map(|o| o.status.success()).unwrap_or(false) + Command::new("pty") + .arg("--help") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) } struct RemoveDirOnDrop(std::path::PathBuf); @@ -25,7 +29,11 @@ impl Drop for RemoveDirOnDrop { #[allow(dead_code)] fn preserved_eval_catalog(output: &std::process::Output) -> std::path::PathBuf { - let text = format!("{}\n{}", String::from_utf8_lossy(&output.stderr), String::from_utf8_lossy(&output.stdout)); + let text = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stderr), + String::from_utf8_lossy(&output.stdout) + ); text.lines() .find_map(|line| line.strip_prefix("catalog preserved (--keep): ")) .map(std::path::PathBuf::from) @@ -134,7 +142,11 @@ sleep 60 ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let out = Command::new(bin) .args(["eval"]) .arg(&cell) @@ -151,10 +163,16 @@ sleep 60 stdout.contains("team signalled done"), "the flow didn't reach done (kick → worker report → sup confirm):\n--stdout--\n{stdout}\n--stderr--\n{stderr}" ); - assert!(stdout.contains("VERDICT: PASS"), "expected PASS:\n--stdout--\n{stdout}\n--stderr--\n{stderr}"); + assert!( + stdout.contains("VERDICT: PASS"), + "expected PASS:\n--stdout--\n{stdout}\n--stderr--\n{stderr}" + ); assert!(out.status.success(), "exit non-zero:\n{stdout}\n{stderr}"); // Both judges (bash + declarative) passed. - assert!(stdout.contains("SCORE: 5 PASS / 0 FAIL"), "expected 5/0:\n{stdout}"); + assert!( + stdout.contains("SCORE: 5 PASS / 0 FAIL"), + "expected 5/0:\n{stdout}" + ); let json_out = Command::new(bin) .args(["eval", "--json"]) @@ -166,27 +184,46 @@ sleep 60 .env("XDG_STATE_HOME", tmp.path().join("xdg-json")) .output() .unwrap(); - assert!(json_out.status.success(), "json eval should preserve exit 0: {}", String::from_utf8_lossy(&json_out.stderr)); - let json: serde_json::Value = serde_json::from_slice(&json_out.stdout).expect("--json emits EvalReport"); + assert!( + json_out.status.success(), + "json eval should preserve exit 0: {}", + String::from_utf8_lossy(&json_out.stderr) + ); + let json: serde_json::Value = + serde_json::from_slice(&json_out.stdout).expect("--json emits EvalReport"); assert_eq!(json["done"], true); assert!(json["judges"].as_array().is_some()); let fail_cell = cell.join("fail.kdl"); - let fail_spec = std::fs::read_to_string(cell.join("cell.kdl")).unwrap() - .replace("test -f $CATALOG/worker/DONE", "test -f $CATALOG/worker/NOPE") + let fail_spec = std::fs::read_to_string(cell.join("cell.kdl")) + .unwrap() + .replace( + "test -f $CATALOG/worker/DONE", + "test -f $CATALOG/worker/NOPE", + ) .replace("field \"count\" is 7", "field \"count\" is 8"); std::fs::write(&fail_cell, fail_spec).unwrap(); let fail_out = Command::new(bin) .args(["eval", "--json"]) .arg(&fail_cell) .env("PATH", &path) - .env_remove("CATALOG").env_remove("ST_ROOT").env_remove("PTY_ROOT") + .env_remove("CATALOG") + .env_remove("ST_ROOT") + .env_remove("PTY_ROOT") .env("XDG_STATE_HOME", tmp.path().join("xdg-fail")) - .output().unwrap(); + .output() + .unwrap(); assert!(!fail_out.status.success()); - let fail_json: serde_json::Value = serde_json::from_slice(&fail_out.stdout).expect("failed eval report"); + let fail_json: serde_json::Value = + serde_json::from_slice(&fail_out.stdout).expect("failed eval report"); assert_eq!(fail_json["judges"][0]["passed"], false); - assert!(fail_json["judges"].as_array().unwrap().iter().any(|j| j["detail"].as_str().unwrap_or("").contains("count"))); + assert!( + fail_json["judges"] + .as_array() + .unwrap() + .iter() + .any(|j| j["detail"].as_str().unwrap_or("").contains("count")) + ); assert!(!String::from_utf8_lossy(&fail_out.stdout).contains("==")); let invalid = Command::new(bin) @@ -194,7 +231,10 @@ sleep 60 .arg(tmp.path().join("missing.kdl")) .output() .unwrap(); - assert!(!invalid.status.success(), "invalid eval input must retain nonzero exit"); + assert!( + !invalid.status.success(), + "invalid eval input must retain nonzero exit" + ); } #[test] @@ -204,7 +244,9 @@ fn canonical_agents_run_from_the_hermetic_catalog_with_one_root_and_native_bus() std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1" ); - eprintln!("SKIP canonical_agents_run_from_the_hermetic_catalog_with_one_root_and_native_bus"); + eprintln!( + "SKIP canonical_agents_run_from_the_hermetic_catalog_with_one_root_and_native_bus" + ); return; } @@ -329,7 +371,11 @@ exec sleep 60 ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let poison = tmp.path().join("ambient-poison"); let child = Command::new(bin) .args(["eval", "--keep", "--host", "evalhost"]) @@ -364,7 +410,11 @@ exec sleep 60 ); for id in ["canonical-sup-main", "canonical-worker-main"] { let log = catalog.join("logs").join(format!("{id}.log")); - assert!(log.exists(), "custom main id did not flow into log capture: {}", log.display()); + assert!( + log.exists(), + "custom main id did not flow into log capture: {}", + log.display() + ); } let sessions = Command::new("pty") .args(["ls", "--json"]) @@ -443,7 +493,11 @@ eval { "#, ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let child = Command::new(bin) .args(["eval", "--keep", "--host", "evalhost"]) .arg(&cell) @@ -553,7 +607,11 @@ eval { "#, ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let child = Command::new(bin) .args(["eval", "--keep", "--host", "evalhost"]) .arg(&cell) @@ -630,13 +688,18 @@ eval { String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr) ); - assert!(!out.status.success(), "unknown kickoff target was accepted:\n{combined}"); assert!( - combined.contains("kickoff target `evalhost.missing`") - && combined.contains("found 0"), + !out.status.success(), + "unknown kickoff target was accepted:\n{combined}" + ); + assert!( + combined.contains("kickoff target `evalhost.missing`") && combined.contains("found 0"), "wrong refusal:\n{combined}" ); - assert!(!catalog.join("SPAWNED").exists(), "task spawned before kickoff admission"); + assert!( + !catalog.join("SPAWNED").exists(), + "task spawned before kickoff admission" + ); } #[test] @@ -706,7 +769,11 @@ eval { "#, ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let child = Command::new(bin) .args(["eval", "--keep", "--host", "evalhost"]) .arg(&cell) @@ -877,7 +944,10 @@ eval {{ String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr) ); - assert!(!out.status.success(), "`{expected}` case launched:\n{combined}"); + assert!( + !out.status.success(), + "`{expected}` case launched:\n{combined}" + ); assert!( combined.contains(expected), "`{expected}` case produced the wrong refusal:\n{combined}" @@ -932,7 +1002,10 @@ eval { String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr) ); - assert!(!out.status.success(), "malformed catalog config launched:\n{combined}"); + assert!( + !out.status.success(), + "malformed catalog config launched:\n{combined}" + ); assert!( combined.contains("catalog-config") && combined.contains("pty_root"), "malformed catalog config produced the wrong refusal:\n{combined}" @@ -961,7 +1034,10 @@ eval { fn supervise_teardown_reaps_a_runtime_spawned_seat_case(judge_command: &str, expect_success: bool) { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); eprintln!("SKIP supervise_teardown_reaps_a_runtime_spawned_seat: `pty` not on PATH"); return; } @@ -978,8 +1054,19 @@ fn supervise_teardown_reaps_a_runtime_spawned_seat_case(judge_command: &str, exp assert_eq!(spec_text.len(), 434); let sentinel = "judge \"trivial\" { exec \"exit 0\" }"; assert_eq!(spec_text.matches(sentinel).count(), 1); - std::fs::write(cell.join("cell.kdl"), spec_text.replace(sentinel, &format!("judge \"trivial\" {{ exec \"{judge_command}\" }}"))).unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + std::fs::write( + cell.join("cell.kdl"), + spec_text.replace( + sentinel, + &format!("judge \"trivial\" {{ exec \"{judge_command}\" }}"), + ), + ) + .unwrap(); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let child = Command::new(bin) .args(["eval", "--keep"]) .arg(&cell) @@ -1006,9 +1093,15 @@ fn supervise_teardown_reaps_a_runtime_spawned_seat_case(judge_command: &str, exp ); assert_eq!(out.status.success(), expect_success); if expect_success { - assert!(stdout.contains("VERDICT: PASS"), "expected human PASS verdict:\n{stdout}\n{stderr}"); + assert!( + stdout.contains("VERDICT: PASS"), + "expected human PASS verdict:\n{stdout}\n{stderr}" + ); } else { - assert!(stderr.contains("VERDICT: FAIL"), "expected human FAIL verdict on stderr:\n{stdout}\n{stderr}"); + assert!( + stderr.contains("VERDICT: FAIL"), + "expected human FAIL verdict on stderr:\n{stdout}\n{stderr}" + ); assert!(!out.status.success(), "human FAIL must be nonzero"); } @@ -1018,7 +1111,10 @@ fn supervise_teardown_reaps_a_runtime_spawned_seat_case(judge_command: &str, exp .parse() .unwrap(); let peer_alive = unsafe { libc::kill(peer_pid, 0) == 0 }; - assert!(!peer_alive, "runtime peer still alive before post-teardown assertions (pid {peer_pid})"); + assert!( + !peer_alive, + "runtime peer still alive before post-teardown assertions (pid {peer_pid})" + ); let sessions = Command::new("pty") .args(["--root"]) .arg(catalog.join("pty")) @@ -1034,17 +1130,23 @@ fn supervise_teardown_reaps_a_runtime_spawned_seat_case(judge_command: &str, exp assert!( !catalog.join("pty/rtpeer.pid").exists() && !catalog.join("pty/rtpeer.sock").exists() - && session_json.as_array().is_some_and(|sessions| sessions.is_empty()), + && session_json + .as_array() + .is_some_and(|sessions| sessions.is_empty()), "runtime-spawned seat leaked after supervise teardown (pid {peer_pid}, registry {session_json}):\n\ --stdout--\n{stdout}\n--stderr--\n{stderr}" ); } #[test] -fn supervise_teardown_reaps_a_runtime_spawned_seat() { supervise_teardown_reaps_a_runtime_spawned_seat_case("exit 0", true); } +fn supervise_teardown_reaps_a_runtime_spawned_seat() { + supervise_teardown_reaps_a_runtime_spawned_seat_case("exit 0", true); +} #[test] -fn supervise_teardown_runtime_peer_human_failure() { supervise_teardown_reaps_a_runtime_spawned_seat_case("exit 1", false); } +fn supervise_teardown_runtime_peer_human_failure() { + supervise_teardown_reaps_a_runtime_spawned_seat_case("exit 1", false); +} struct SignalCaseFailureGuard { child: Option, @@ -1056,39 +1158,107 @@ struct SignalCaseFailureGuard { } fn pty_session_pid(root: &Path, id: &str) -> (i32, serde_json::Value) { - let out = Command::new("pty").args(["--root"]).arg(root).args(["stats", "--json", id]).output().unwrap(); - let raw: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap_or_else(|e| panic!("stats parse failed: {e}; raw={}", String::from_utf8_lossy(&out.stdout))); - fn find(v: &serde_json::Value) -> Option { match v { serde_json::Value::Object(m) => m.get("process").and_then(|p| p.get("pid")).and_then(|p| p.as_i64()).map(|p| p as i32).or_else(|| m.values().find_map(find)), serde_json::Value::Array(a) => a.iter().find_map(find), _ => None } } - let pid = find(&raw).unwrap_or_else(|| panic!("stats missing process.pid: {raw}")); (pid, raw) + let out = Command::new("pty") + .args(["--root"]) + .arg(root) + .args(["stats", "--json", id]) + .output() + .unwrap(); + let raw: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap_or_else(|e| { + panic!( + "stats parse failed: {e}; raw={}", + String::from_utf8_lossy(&out.stdout) + ) + }); + fn find(v: &serde_json::Value) -> Option { + match v { + serde_json::Value::Object(m) => m + .get("process") + .and_then(|p| p.get("pid")) + .and_then(|p| p.as_i64()) + .map(|p| p as i32) + .or_else(|| m.values().find_map(find)), + serde_json::Value::Array(a) => a.iter().find_map(find), + _ => None, + } + } + let pid = find(&raw).unwrap_or_else(|| panic!("stats missing process.pid: {raw}")); + (pid, raw) } #[derive(Clone, Debug, Default)] -struct SignalCleanupReceipt { success: bool, diagnostics: String, child_pid: Option, child_dead: bool, peer_pid: Option, peer_dead: bool, registry_empty: bool, pid_absent: bool, socket_absent: bool } +struct SignalCleanupReceipt { + success: bool, + diagnostics: String, + child_pid: Option, + child_dead: bool, + peer_pid: Option, + peer_dead: bool, + registry_empty: bool, + pid_absent: bool, + socket_absent: bool, +} impl SignalCaseFailureGuard { - fn disarm(&mut self) { self.armed = false; } - fn child_mut(&mut self) -> Option<&mut std::process::Child> { self.child.as_mut() } - fn take_child(&mut self) -> Option { self.child.take() } + fn disarm(&mut self) { + self.armed = false; + } + fn child_mut(&mut self) -> Option<&mut std::process::Child> { + self.child.as_mut() + } + fn take_child(&mut self) -> Option { + self.child.take() + } } impl Drop for SignalCaseFailureGuard { fn drop(&mut self) { - if !self.armed { return; } + if !self.armed { + return; + } if let Some(child) = self.child.as_mut() { let _ = child.kill(); let _ = child.wait(); } - let child_pid = self.child.as_ref().map(|c| c.id()); let peer_pid = self.peer_pid; - let mut ok = false; let mut diagnostics = String::new(); + let child_pid = self.child.as_ref().map(|c| c.id()); + let peer_pid = self.peer_pid; + let mut ok = false; + let mut diagnostics = String::new(); for _ in 0..5 { - let _ = Command::new("pty").args(["--root"]).arg(&self.pty_root).args(["kill", &self.peer_id]).status(); - if let Some(pid) = peer_pid && unsafe { libc::kill(pid, 0) == 0 } { unsafe { libc::kill(pid, libc::SIGKILL); } } - let _ = Command::new("pty").args(["--root"]).arg(&self.pty_root).args(["rm", &self.peer_id]).status(); - match Command::new("pty").args(["--root"]).arg(&self.pty_root).args(["list", "--json"]).output() { - Ok(out) => if let Ok(json) = serde_json::from_slice::(&out.stdout) { - if json.as_array().is_some_and(|v| v.is_empty()) { ok = true; break; } - diagnostics = json.to_string(); - } else { diagnostics = String::from_utf8_lossy(&out.stderr).into_owned(); }, + let _ = Command::new("pty") + .args(["--root"]) + .arg(&self.pty_root) + .args(["kill", &self.peer_id]) + .status(); + if let Some(pid) = peer_pid + && unsafe { libc::kill(pid, 0) == 0 } + { + unsafe { + libc::kill(pid, libc::SIGKILL); + } + } + let _ = Command::new("pty") + .args(["--root"]) + .arg(&self.pty_root) + .args(["rm", &self.peer_id]) + .status(); + match Command::new("pty") + .args(["--root"]) + .arg(&self.pty_root) + .args(["list", "--json"]) + .output() + { + Ok(out) => { + if let Ok(json) = serde_json::from_slice::(&out.stdout) { + if json.as_array().is_some_and(|v| v.is_empty()) { + ok = true; + break; + } + diagnostics = json.to_string(); + } else { + diagnostics = String::from_utf8_lossy(&out.stderr).into_owned(); + } + } Err(e) => diagnostics = e.to_string(), } std::thread::sleep(Duration::from_millis(20)); @@ -1096,57 +1266,149 @@ impl Drop for SignalCaseFailureGuard { let child_dead = child_pid.is_some_and(|p| unsafe { libc::kill(p as i32, 0) != 0 }); let peer_dead = peer_pid.is_none_or(|p| unsafe { libc::kill(p, 0) != 0 }); let pid_absent = !self.pty_root.join(format!("{}.pid", self.peer_id)).exists(); - let socket_absent = !self.pty_root.join(format!("{}.sock", self.peer_id)).exists(); - if !ok && diagnostics.is_empty() { diagnostics = "registry did not converge empty".into(); } - if let Ok(mut receipt) = self.receipt.lock() { receipt.child_pid=child_pid; receipt.child_dead=child_dead; receipt.peer_pid=peer_pid; receipt.peer_dead=peer_dead; receipt.registry_empty=ok; receipt.pid_absent=pid_absent; receipt.socket_absent=socket_absent; receipt.success=child_dead&&peer_dead&&ok&&pid_absent&&socket_absent; receipt.diagnostics=diagnostics; } + let socket_absent = !self + .pty_root + .join(format!("{}.sock", self.peer_id)) + .exists(); + if !ok && diagnostics.is_empty() { + diagnostics = "registry did not converge empty".into(); + } + if let Ok(mut receipt) = self.receipt.lock() { + receipt.child_pid = child_pid; + receipt.child_dead = child_dead; + receipt.peer_pid = peer_pid; + receipt.peer_dead = peer_dead; + receipt.registry_empty = ok; + receipt.pid_absent = pid_absent; + receipt.socket_absent = socket_absent; + receipt.success = child_dead && peer_dead && ok && pid_absent && socket_absent; + receipt.diagnostics = diagnostics; + } } } fn runtime_peer_signal_case(sig: libc::c_int) { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "pty not on PATH; set ST2_ALLOW_PTY_SKIP=1"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "pty not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); eprintln!("SKIP runtime_peer_signal_case: pty not on PATH"); return; } - let bin = env!("CARGO_BIN_EXE_st2"); let bin_dir = Path::new(bin).parent().unwrap(); - let tmp = tempfile::tempdir().unwrap(); let cell = tmp.path().join("cell"); std::fs::create_dir_all(&cell).unwrap(); + let bin = env!("CARGO_BIN_EXE_st2"); + let bin_dir = Path::new(bin).parent().unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let cell = tmp.path().join("cell"); + std::fs::create_dir_all(&cell).unwrap(); std::fs::write(cell.join("cell.kdl"), RUNTIME_PEER_SPEC).unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); - let child = Command::new(bin).args(["eval", "--keep"]).arg(&cell).env("PATH", path) - .env("XDG_STATE_HOME", tmp.path().join("xdg")).env_remove("CATALOG").env_remove("ST_ROOT").env_remove("PTY_ROOT") - .stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); - let catalog = std::env::temp_dir().join(format!("st2e-{}", child.id())); let _guard = RemoveDirOnDrop(catalog.clone()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); + let child = Command::new(bin) + .args(["eval", "--keep"]) + .arg(&cell) + .env("PATH", path) + .env("XDG_STATE_HOME", tmp.path().join("xdg")) + .env_remove("CATALOG") + .env_remove("ST_ROOT") + .env_remove("PTY_ROOT") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let catalog = std::env::temp_dir().join(format!("st2e-{}", child.id())); + let _guard = RemoveDirOnDrop(catalog.clone()); let receipt = Arc::new(Mutex::new(SignalCleanupReceipt::default())); - let mut failure = SignalCaseFailureGuard { child: Some(child), pty_root: catalog.join("pty"), peer_id: "rtpeer".into(), peer_pid: None, armed: true, receipt }; - let marker = catalog.join("runtime-peer.pid"); let deadline = Instant::now() + Duration::from_secs(15); - while !marker.exists() && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(100)); } - if !marker.exists() { let status = failure.child_mut().and_then(|c| c.try_wait().ok()).flatten(); panic!("marker timeout status={status:?} catalog={}", catalog.display()); } + let mut failure = SignalCaseFailureGuard { + child: Some(child), + pty_root: catalog.join("pty"), + peer_id: "rtpeer".into(), + peer_pid: None, + armed: true, + receipt, + }; + let marker = catalog.join("runtime-peer.pid"); + let deadline = Instant::now() + Duration::from_secs(15); + while !marker.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(100)); + } + if !marker.exists() { + let status = failure + .child_mut() + .and_then(|c| c.try_wait().ok()) + .flatten(); + panic!( + "marker timeout status={status:?} catalog={}", + catalog.display() + ); + } assert!(catalog.is_dir()); let (session_pid, _stats) = pty_session_pid(&catalog.join("pty"), "rtpeer"); let child_id = failure.child_mut().unwrap().id(); assert_eq!(unsafe { libc::kill(child_id as i32, sig) }, 0); - let out = failure.take_child().unwrap().wait_with_output().unwrap(); assert!(!out.status.success(), "status={:?} stdout={} stderr={}", out.status, String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); - let combined = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); - assert!(combined.contains("eval interrupted by SIGINT/SIGTERM"), "missing interruption contract: {combined}"); - let peer: i32 = std::fs::read_to_string(&marker).unwrap().trim().parse().unwrap(); failure.peer_pid = Some(peer); assert!(unsafe { libc::kill(peer, 0) != 0 }); + let out = failure.take_child().unwrap().wait_with_output().unwrap(); + assert!( + !out.status.success(), + "status={:?} stdout={} stderr={}", + out.status, + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert!( + combined.contains("eval interrupted by SIGINT/SIGTERM"), + "missing interruption contract: {combined}" + ); + let peer: i32 = std::fs::read_to_string(&marker) + .unwrap() + .trim() + .parse() + .unwrap(); + failure.peer_pid = Some(peer); + assert!(unsafe { libc::kill(peer, 0) != 0 }); assert!(unsafe { libc::kill(session_pid, 0) != 0 }); - let listed = Command::new("pty").args(["--root"]).arg(catalog.join("pty")).args(["list", "--json"]).output().unwrap(); - let registry: serde_json::Value = serde_json::from_slice(&listed.stdout).unwrap(); assert!(registry.as_array().is_some_and(|v| v.is_empty())); - assert!(!catalog.join("pty/rtpeer.pid").exists()); assert!(!catalog.join("pty/rtpeer.sock").exists()); + let listed = Command::new("pty") + .args(["--root"]) + .arg(catalog.join("pty")) + .args(["list", "--json"]) + .output() + .unwrap(); + let registry: serde_json::Value = serde_json::from_slice(&listed.stdout).unwrap(); + assert!(registry.as_array().is_some_and(|v| v.is_empty())); + assert!(!catalog.join("pty/rtpeer.pid").exists()); + assert!(!catalog.join("pty/rtpeer.sock").exists()); failure.disarm(); } -#[test] fn supervise_runtime_peer_sigterm_reaps() { runtime_peer_signal_case(libc::SIGTERM); } -#[test] fn supervise_runtime_peer_sigint_reaps() { runtime_peer_signal_case(libc::SIGINT); } +#[test] +fn supervise_runtime_peer_sigterm_reaps() { + runtime_peer_signal_case(libc::SIGTERM); +} +#[test] +fn supervise_runtime_peer_sigint_reaps() { + runtime_peer_signal_case(libc::SIGINT); +} #[test] fn signal_case_failure_guard_reaps_on_unwind() { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "pty not on PATH; set ST2_ALLOW_PTY_SKIP=1"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "pty not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); eprintln!("SKIP signal_case_failure_guard_reaps_on_unwind: pty not on PATH"); return; } - let tmp = tempfile::tempdir().unwrap(); let catalog = tmp.path().join("catalog"); let receipt = Arc::new(Mutex::new(SignalCleanupReceipt::default())); + let tmp = tempfile::tempdir().unwrap(); + let catalog = tmp.path().join("catalog"); + let receipt = Arc::new(Mutex::new(SignalCleanupReceipt::default())); let receipt_out = receipt.clone(); let session_pid_out: Arc>> = Arc::new(Mutex::new(None)); let session_pid_capture = session_pid_out.clone(); @@ -1154,16 +1416,60 @@ fn signal_case_failure_guard_reaps_on_unwind() { std::fs::create_dir_all(catalog.join("pty")).unwrap(); let _catalog = RemoveDirOnDrop(catalog.clone()); let root = catalog.join("pty"); - let peer = Command::new("pty").args(["--root"]).arg(&root).args(["run", "-d", "--id", "rtpeer", "--", "sleep", "1000"]).status().unwrap(); assert!(peer.success()); - let pid_path = root.join("rtpeer.pid"); let deadline = Instant::now() + Duration::from_secs(5); - while !pid_path.exists() && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(50)); } - let peer_pid: i32 = std::fs::read_to_string(&pid_path).unwrap().trim().parse().unwrap(); - let (session_pid, _) = pty_session_pid(&root, "rtpeer"); *session_pid_capture.lock().unwrap() = Some(session_pid); - let child = Command::new("sh").args(["-c", "sleep 1000"]).spawn().unwrap(); - let guard = SignalCaseFailureGuard { child: Some(child), pty_root: root, peer_id: "rtpeer".into(), peer_pid: Some(peer_pid), armed: true, receipt: receipt.clone() }; - let _guard = guard; panic!("representative post-signal assertion"); + let peer = Command::new("pty") + .args(["--root"]) + .arg(&root) + .args(["run", "-d", "--id", "rtpeer", "--", "sleep", "1000"]) + .status() + .unwrap(); + assert!(peer.success()); + let pid_path = root.join("rtpeer.pid"); + let deadline = Instant::now() + Duration::from_secs(5); + while !pid_path.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(50)); + } + let peer_pid: i32 = std::fs::read_to_string(&pid_path) + .unwrap() + .trim() + .parse() + .unwrap(); + let (session_pid, _) = pty_session_pid(&root, "rtpeer"); + *session_pid_capture.lock().unwrap() = Some(session_pid); + let child = Command::new("sh") + .args(["-c", "sleep 1000"]) + .spawn() + .unwrap(); + let guard = SignalCaseFailureGuard { + child: Some(child), + pty_root: root, + peer_id: "rtpeer".into(), + peer_pid: Some(peer_pid), + armed: true, + receipt: receipt.clone(), + }; + let _guard = guard; + panic!("representative post-signal assertion"); })); - assert!(result.is_err()); let receipt = receipt_out.lock().unwrap(); assert!(receipt.success, "cleanup diagnostics: {}", receipt.diagnostics); assert!(receipt.diagnostics.is_empty()); assert!(receipt.child_pid.is_some() && receipt.child_dead && receipt.peer_pid.is_some() && receipt.peer_dead && receipt.registry_empty && receipt.pid_absent && receipt.socket_absent); let session_pid = session_pid_out.lock().unwrap().unwrap(); assert!(unsafe { libc::kill(session_pid, 0) != 0 }); assert!(!catalog.exists()); + assert!(result.is_err()); + let receipt = receipt_out.lock().unwrap(); + assert!( + receipt.success, + "cleanup diagnostics: {}", + receipt.diagnostics + ); + assert!(receipt.diagnostics.is_empty()); + assert!( + receipt.child_pid.is_some() + && receipt.child_dead + && receipt.peer_pid.is_some() + && receipt.peer_dead + && receipt.registry_empty + && receipt.pid_absent + && receipt.socket_absent + ); + let session_pid = session_pid_out.lock().unwrap().unwrap(); + assert!(unsafe { libc::kill(session_pid, 0) != 0 }); + assert!(!catalog.exists()); } /// A TEAM-LESS eval (no agents, no kickoff) runs its `run` steps to completion, captures each step's @@ -1199,7 +1505,11 @@ eval { "#, ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let out = Command::new(bin) .args(["eval"]) .arg(&cell) @@ -1212,13 +1522,22 @@ eval { .unwrap(); let stdout = String::from_utf8_lossy(&out.stdout); let stderr = String::from_utf8_lossy(&out.stderr); - assert!(stdout.contains("team-less eval: 2 run step(s)"), "not the team-less path:\n{stdout}\n{stderr}"); - assert!(stdout.contains("run step probe → exit 3"), "probe's non-zero exit not captured:\n{stdout}"); + assert!( + stdout.contains("team-less eval: 2 run step(s)"), + "not the team-less path:\n{stdout}\n{stderr}" + ); + assert!( + stdout.contains("run step probe → exit 3"), + "probe's non-zero exit not captured:\n{stdout}" + ); assert!( stdout.contains("SCORE: 6 PASS / 0 FAIL"), "team-less run-stage eval should be 6/0 (make must-exit-0 gate + 5 judges incl. $LOGS_DIR):\n--stdout--\n{stdout}\n--stderr--\n{stderr}" ); - assert!(stdout.contains("VERDICT: PASS") && out.status.success(), "expected PASS:\n{stdout}\n{stderr}"); + assert!( + stdout.contains("VERDICT: PASS") && out.status.success(), + "expected PASS:\n{stdout}\n{stderr}" + ); } /// crash-ding: under `supervise`, a seat that CRASHES (non-zero/killed/vanished) is respawned AND its @@ -1229,8 +1548,13 @@ eval { #[test] fn supervise_crash_dings_up_the_chain_and_is_silent_on_clean_exit() { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1"); - eprintln!("SKIP supervise_crash_dings_up_the_chain_and_is_silent_on_clean_exit: `pty` not on PATH"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); + eprintln!( + "SKIP supervise_crash_dings_up_the_chain_and_is_silent_on_clean_exit: `pty` not on PATH" + ); return; } let bin = env!("CARGO_BIN_EXE_st2"); @@ -1326,7 +1650,11 @@ sleep 100000 "#, ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let child = Command::new(bin) .args(["eval", "--keep", "--host", "evalhost"]) .arg(&cell) @@ -1373,7 +1701,10 @@ sleep 100000 #[test] fn st2_eval_fails_fast_when_a_seat_exits_at_boot() { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); eprintln!("SKIP st2_eval_fails_fast_when_a_seat_exits_at_boot: `pty` not on PATH"); return; } @@ -1398,7 +1729,11 @@ eval { "#, ) .unwrap(); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let start = Instant::now(); let out = Command::new(bin) .args(["eval"]) @@ -1411,8 +1746,21 @@ eval { .output() .unwrap(); let elapsed = start.elapsed(); - let combined = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); - assert!(!out.status.success(), "a dead-at-boot seat must fail the eval:\n{combined}"); - assert!(combined.contains("exited at boot"), "expected a clear fail-fast error:\n{combined}"); - assert!(elapsed < Duration::from_secs(60), "must fail FAST, not wait max-timeout — took {elapsed:?}"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert!( + !out.status.success(), + "a dead-at-boot seat must fail the eval:\n{combined}" + ); + assert!( + combined.contains("exited at boot"), + "expected a clear fail-fast error:\n{combined}" + ); + assert!( + elapsed < Duration::from_secs(60), + "must fail FAST, not wait max-timeout — took {elapsed:?}" + ); } diff --git a/tests/eval_up.rs b/tests/eval_up.rs index a6fba86e..7eeeb127 100644 --- a/tests/eval_up.rs +++ b/tests/eval_up.rs @@ -10,19 +10,34 @@ use std::process::Command; use std::time::{Duration, Instant}; fn pty_available() -> bool { - Command::new("pty").arg("--help").output().map(|o| o.status.success()).unwrap_or(false) + Command::new("pty") + .arg("--help") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) } fn pty_ids(pty_root: &Path) -> Vec<(String, String)> { - let out = Command::new("pty").args(["list", "--json"]).env("PTY_ROOT", pty_root).output().unwrap(); - let v: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap_or(serde_json::Value::Array(vec![])); + let out = Command::new("pty") + .args(["list", "--json"]) + .env("PTY_ROOT", pty_root) + .output() + .unwrap(); + let v: serde_json::Value = + serde_json::from_slice(&out.stdout).unwrap_or(serde_json::Value::Array(vec![])); v.as_array() .map(|rows| { rows.iter() .map(|s| { ( - s.get("name").and_then(|x| x.as_str()).unwrap_or("").to_string(), - s.get("status").and_then(|x| x.as_str()).unwrap_or("").to_string(), + s.get("name") + .and_then(|x| x.as_str()) + .unwrap_or("") + .to_string(), + s.get("status") + .and_then(|x| x.as_str()) + .unwrap_or("") + .to_string(), ) }) .collect() @@ -60,7 +75,11 @@ team "t" { .unwrap(); let pty_root = tmp.path().join("pty"); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let out = Command::new(bin) .args(["up"]) .arg(&spec_dir) @@ -76,25 +95,49 @@ team "t" { "st2 up failed.\n--- stdout ---\n{stdout}\n--- stderr ---\n{}", String::from_utf8_lossy(&out.stderr) ); - assert!(stdout.contains("booted team from spec"), "no boot line:\n{stdout}"); + assert!( + stdout.contains("booted team from spec"), + "no boot line:\n{stdout}" + ); // The team is running under its team-prefixed ids, in the isolated PTY_ROOT. std::thread::sleep(std::time::Duration::from_millis(500)); let ids = pty_ids(&pty_root); - let running: Vec<&str> = ids.iter().filter(|(_, s)| s == "running").map(|(n, _)| n.as_str()).collect(); - assert!(running.contains(&"t.a"), "t.a not running; sessions={ids:?}"); - assert!(running.contains(&"t.b"), "t.b not running; sessions={ids:?}"); + let running: Vec<&str> = ids + .iter() + .filter(|(_, s)| s == "running") + .map(|(n, _)| n.as_str()) + .collect(); + assert!( + running.contains(&"t.a"), + "t.a not running; sessions={ids:?}" + ); + assert!( + running.contains(&"t.b"), + "t.b not running; sessions={ids:?}" + ); // Teardown — the team persists after st2 exits (nomad-decoupled), so clean it up ourselves. for id in ["t.a", "t.b"] { - let _ = Command::new("pty").args(["kill", id]).env("PTY_ROOT", &pty_root).status(); - let _ = Command::new("pty").args(["rm", id]).env("PTY_ROOT", &pty_root).status(); + let _ = Command::new("pty") + .args(["kill", id]) + .env("PTY_ROOT", &pty_root) + .status(); + let _ = Command::new("pty") + .args(["rm", id]) + .env("PTY_ROOT", &pty_root) + .status(); } // Stop any lingering per-task scopes (this spec's ids only). - if let Ok(o) = Command::new("systemctl").args(["--user", "list-units", "--no-legend", "st2-t.*"]).output() { + if let Ok(o) = Command::new("systemctl") + .args(["--user", "list-units", "--no-legend", "st2-t.*"]) + .output() + { for line in String::from_utf8_lossy(&o.stdout).lines() { if let Some(unit) = line.split_whitespace().next() { - let _ = Command::new("systemctl").args(["--user", "stop", unit]).status(); + let _ = Command::new("systemctl") + .args(["--user", "stop", unit]) + .status(); } } } @@ -119,7 +162,10 @@ fn st2_up_refuses_an_eval_only_file() { .env("XDG_STATE_HOME", tmp.path().join("xdg")) .output() .unwrap(); - assert!(!out.status.success(), "st2 up on an eval-only file must refuse"); + assert!( + !out.status.success(), + "st2 up on an eval-only file must refuse" + ); let stderr = String::from_utf8_lossy(&out.stderr); assert!( stderr.contains("eval-only file") && stderr.contains("st2 eval"), @@ -133,7 +179,10 @@ fn st2_up_refuses_an_eval_only_file() { #[test] fn st2_down_tears_down_a_spec_fleet() { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); eprintln!("SKIP st2_down_tears_down_a_spec_fleet: `pty` not on PATH"); return; } @@ -154,7 +203,11 @@ team "down" { ) .unwrap(); let pty_root = tmp.path().join("pty"); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let run = |args: &[&str]| { Command::new(bin) .args(args) @@ -168,36 +221,73 @@ team "down" { // Boot the team, confirm it's running. let up = run(&["up", "--once"]); - assert!(up.status.success(), "up failed: {}", String::from_utf8_lossy(&up.stderr)); + assert!( + up.status.success(), + "up failed: {}", + String::from_utf8_lossy(&up.stderr) + ); std::thread::sleep(Duration::from_millis(500)); let running = |ids: &[(String, String)]| -> Vec { - ids.iter().filter(|(_, s)| s == "running").map(|(n, _)| n.clone()).collect() + ids.iter() + .filter(|(_, s)| s == "running") + .map(|(n, _)| n.clone()) + .collect() }; let before = running(&pty_ids(&pty_root)); - assert!(before.contains(&"down.a".to_string()), "t.a not running before down; {before:?}"); - assert!(before.contains(&"down.b".to_string()), "t.b not running before down; {before:?}"); + assert!( + before.contains(&"down.a".to_string()), + "t.a not running before down; {before:?}" + ); + assert!( + before.contains(&"down.b".to_string()), + "t.b not running before down; {before:?}" + ); // `st2 down ` tears down the declared team. let down = run(&["down"]); let dstdout = String::from_utf8_lossy(&down.stdout); - assert!(down.status.success(), "down failed: {}", String::from_utf8_lossy(&down.stderr)); - assert!(dstdout.contains("teardown of spec"), "no spec-teardown line:\n{dstdout}"); - assert!(dstdout.contains("down.a") && dstdout.contains("down.b"), "down did not report tearing down down.a/down.b:\n{dstdout}"); + assert!( + down.status.success(), + "down failed: {}", + String::from_utf8_lossy(&down.stderr) + ); + assert!( + dstdout.contains("teardown of spec"), + "no spec-teardown line:\n{dstdout}" + ); + assert!( + dstdout.contains("down.a") && dstdout.contains("down.b"), + "down did not report tearing down down.a/down.b:\n{dstdout}" + ); // The sessions are no longer running. std::thread::sleep(Duration::from_millis(500)); let after = running(&pty_ids(&pty_root)); - assert!(!after.contains(&"down.a".to_string()), "t.a still running after down; {after:?}"); - assert!(!after.contains(&"down.b".to_string()), "t.b still running after down; {after:?}"); + assert!( + !after.contains(&"down.a".to_string()), + "t.a still running after down; {after:?}" + ); + assert!( + !after.contains(&"down.b".to_string()), + "t.b still running after down; {after:?}" + ); // Clean up the (now-stopped) sessions + any per-task scopes. for id in ["down.a", "down.b"] { - let _ = Command::new("pty").args(["rm", id]).env("PTY_ROOT", &pty_root).status(); + let _ = Command::new("pty") + .args(["rm", id]) + .env("PTY_ROOT", &pty_root) + .status(); } - if let Ok(o) = Command::new("systemctl").args(["--user", "list-units", "--no-legend", "st2-down.*"]).output() { + if let Ok(o) = Command::new("systemctl") + .args(["--user", "list-units", "--no-legend", "st2-down.*"]) + .output() + { for line in String::from_utf8_lossy(&o.stdout).lines() { if let Some(unit) = line.split_whitespace().next() { - let _ = Command::new("systemctl").args(["--user", "stop", unit]).status(); + let _ = Command::new("systemctl") + .args(["--user", "stop", unit]) + .status(); } } } @@ -210,7 +300,10 @@ team "down" { #[test] fn st2_up_once_atomically_respawns_a_hard_killed_agent() { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); eprintln!("SKIP st2_up_once_atomically_respawns_a_hard_killed_agent: `pty` not on PATH"); return; } @@ -225,7 +318,11 @@ fn st2_up_once_atomically_respawns_a_hard_killed_agent() { ) .unwrap(); let pty_root = tmp.path().join("pty"); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); let once = || { Command::new(bin) .args(["up", "--once"]) @@ -237,7 +334,11 @@ fn st2_up_once_atomically_respawns_a_hard_killed_agent() { .unwrap() }; let pid_of = |id: &str| -> Option { - let out = Command::new("pty").args(["list", "--json"]).env("PTY_ROOT", &pty_root).output().ok()?; + let out = Command::new("pty") + .args(["list", "--json"]) + .env("PTY_ROOT", &pty_root) + .output() + .ok()?; let v: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?; v.as_array()? .iter() @@ -252,24 +353,44 @@ fn st2_up_once_atomically_respawns_a_hard_killed_agent() { assert!(once().status.success(), "initial boot failed"); std::thread::sleep(Duration::from_millis(700)); let pid1 = pid_of("raceonce").expect("agent 'raceonce' should be running after boot"); - let _ = Command::new("kill").args(["-9", &pid1.to_string()]).status(); + let _ = Command::new("kill") + .args(["-9", &pid1.to_string()]) + .status(); // A single `--once` pass right after the hard-kill must atomically reap + respawn — no "in use". let out = once(); let stderr = String::from_utf8_lossy(&out.stderr); - assert!(out.status.success(), "up --once failed after hard-kill:\n{stderr}"); - assert!(!stderr.contains("already in use"), "respawn hit the reap race:\n{stderr}"); + assert!( + out.status.success(), + "up --once failed after hard-kill:\n{stderr}" + ); + assert!( + !stderr.contains("already in use"), + "respawn hit the reap race:\n{stderr}" + ); std::thread::sleep(Duration::from_millis(500)); - let pid2 = pid_of("raceonce").expect("agent 'raceonce' should be respawned by the same --once pass"); + let pid2 = + pid_of("raceonce").expect("agent 'raceonce' should be respawned by the same --once pass"); assert_ne!(pid1, pid2, "respawn must be a NEW process"); // Clean up. - let _ = Command::new("pty").args(["kill", "raceonce"]).env("PTY_ROOT", &pty_root).status(); - let _ = Command::new("pty").args(["rm", "raceonce"]).env("PTY_ROOT", &pty_root).status(); - if let Ok(o) = Command::new("systemctl").args(["--user", "list-units", "--no-legend", "st2-raceonce*"]).output() { + let _ = Command::new("pty") + .args(["kill", "raceonce"]) + .env("PTY_ROOT", &pty_root) + .status(); + let _ = Command::new("pty") + .args(["rm", "raceonce"]) + .env("PTY_ROOT", &pty_root) + .status(); + if let Ok(o) = Command::new("systemctl") + .args(["--user", "list-units", "--no-legend", "st2-raceonce*"]) + .output() + { for line in String::from_utf8_lossy(&o.stdout).lines() { if let Some(unit) = line.split_whitespace().next() { - let _ = Command::new("systemctl").args(["--user", "stop", unit]).status(); + let _ = Command::new("systemctl") + .args(["--user", "stop", unit]) + .status(); } } } @@ -281,7 +402,10 @@ fn st2_up_once_atomically_respawns_a_hard_killed_agent() { #[test] fn st2_up_spec_supervises_and_respawns_a_killed_agent() { if !pty_available() { - assert!(std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1"); + assert!( + std::env::var_os("ST2_ALLOW_PTY_SKIP").is_some(), + "`pty` not on PATH; set ST2_ALLOW_PTY_SKIP=1" + ); eprintln!("SKIP st2_up_spec_supervises_and_respawns_a_killed_agent: `pty` not on PATH"); return; } @@ -296,11 +420,17 @@ fn st2_up_spec_supervises_and_respawns_a_killed_agent() { ) .unwrap(); let pty_root = tmp.path().join("pty"); - let path = format!("{}:{}", bin_dir.display(), std::env::var("PATH").unwrap_or_default()); + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); // Supervise in the background with a fast reconcile interval. let mut child = std::process::Command::new(bin) - .args(["up"]).arg(&spec_dir).args(["--interval", "1"]) + .args(["up"]) + .arg(&spec_dir) + .args(["--interval", "1"]) .env("PATH", &path) .env("XDG_STATE_HOME", tmp.path().join("xdg")) .env("PTY_ROOT", &pty_root) @@ -310,37 +440,67 @@ fn st2_up_spec_supervises_and_respawns_a_killed_agent() { .unwrap(); let pid_of = |id: &str| -> Option { - let out = std::process::Command::new("pty").args(["list", "--json"]).env("PTY_ROOT", &pty_root).output().ok()?; + let out = std::process::Command::new("pty") + .args(["list", "--json"]) + .env("PTY_ROOT", &pty_root) + .output() + .ok()?; let v: serde_json::Value = serde_json::from_slice(&out.stdout).ok()?; - v.as_array()?.iter().find(|s| s.get("name").and_then(|x| x.as_str()) == Some(id) && s.get("status").and_then(|x| x.as_str()) == Some("running")) + v.as_array()? + .iter() + .find(|s| { + s.get("name").and_then(|x| x.as_str()) == Some(id) + && s.get("status").and_then(|x| x.as_str()) == Some("running") + }) .and_then(|s| s.get("pid").and_then(|p| p.as_i64())) }; let wait_for = |id: &str, secs: u64| -> Option { let deadline = Instant::now() + Duration::from_secs(secs); loop { - if let Some(pid) = pid_of(id) { return Some(pid); } - if Instant::now() > deadline { return None; } + if let Some(pid) = pid_of(id) { + return Some(pid); + } + if Instant::now() > deadline { + return None; + } std::thread::sleep(Duration::from_millis(300)); } }; let pid1 = wait_for("a", 15).expect("agent 'a' should boot under supervision"); // Kill it out from under the supervisor. - let _ = std::process::Command::new("pty").args(["kill", "a"]).env("PTY_ROOT", &pty_root).status(); + let _ = std::process::Command::new("pty") + .args(["kill", "a"]) + .env("PTY_ROOT", &pty_root) + .status(); // The supervise loop must bring it back (new pid) within a few reconcile intervals. let pid2 = wait_for("a", 15).expect("supervisor should RESPAWN the killed agent"); - assert_ne!(pid1, pid2, "respawn must be a NEW process, not the killed one"); + assert_ne!( + pid1, pid2, + "respawn must be a NEW process, not the killed one" + ); // Stop the supervisor; the session persists (nomad-decoupled). Clean up. let _ = child.kill(); let _ = child.wait(); - let _ = std::process::Command::new("pty").args(["kill", "a"]).env("PTY_ROOT", &pty_root).status(); - let _ = std::process::Command::new("pty").args(["rm", "a"]).env("PTY_ROOT", &pty_root).status(); + let _ = std::process::Command::new("pty") + .args(["kill", "a"]) + .env("PTY_ROOT", &pty_root) + .status(); + let _ = std::process::Command::new("pty") + .args(["rm", "a"]) + .env("PTY_ROOT", &pty_root) + .status(); for u in ["st2-a"] { - if let Ok(o) = std::process::Command::new("systemctl").args(["--user", "list-units", "--no-legend", &format!("{u}*")]).output() { + if let Ok(o) = std::process::Command::new("systemctl") + .args(["--user", "list-units", "--no-legend", &format!("{u}*")]) + .output() + { for line in String::from_utf8_lossy(&o.stdout).lines() { if let Some(unit) = line.split_whitespace().next() { - let _ = std::process::Command::new("systemctl").args(["--user", "stop", unit]).status(); + let _ = std::process::Command::new("systemctl") + .args(["--user", "stop", unit]) + .status(); } } } diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs new file mode 100644 index 00000000..92e85948 --- /dev/null +++ b/tests/event_e2e.rs @@ -0,0 +1,333 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Barrier}; + +use sha2::{Digest as _, Sha256}; +use st2::event::{self, EventReceiptStatus, RING_CAPACITY}; +use st2::message; + +fn declare_agent(root: &Path, desired: &str, streams: &str) -> PathBuf { + let directory = root.join("agents/hetz/worker"); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("agent.kdl"), + format!( + "agent \"worker\" {{\n host \"hetz\"\n desired-state {desired}\n command \"agent\"\n{streams}}}\n" + ), + ) + .unwrap(); + directory +} + +fn emit(root: &Path, id: &str, key: Option<&str>, supersede: bool) -> event::EventReceipt { + event::emit( + root, + "hetz", + "hetz.worker", + "gh-ci", + id, + key, + Some(&format!("CI {id}")), + &format!("{{\"id\":\"{id}\"}}"), + supersede, + ) + .unwrap() +} + +#[test] +fn stable_event_identity_publishes_exactly_one_canonical_message() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + + let first = emit(catalog.path(), "run-812", None, false); + let replay = emit(catalog.path(), "run-812", None, false); + + assert_eq!(first.status, EventReceiptStatus::Created); + assert_eq!(replay.status, EventReceiptStatus::Deduplicated); + assert_eq!(first.filename, replay.filename); + let inbox = message::list_inbox(&message::inbox_dir(&agent)).unwrap(); + assert_eq!(inbox.len(), 1); + assert_eq!(inbox[0].from.as_deref(), Some("hetz.worker/gh-ci")); + assert_eq!(inbox[0].stream.as_deref(), Some("gh-ci")); + assert_eq!(inbox[0].event_id.as_deref(), Some("run-812")); + assert!(!agent.join("resources/sent").exists()); +} + +#[test] +fn concurrent_replays_publish_exactly_one_event() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let root = Arc::new(catalog.path().to_path_buf()); + let barrier = Arc::new(Barrier::new(12)); + let threads = (0..12) + .map(|_| { + let root = Arc::clone(&root); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + emit(&root, "delivery-1", None, false) + }) + }) + .collect::>(); + let receipts = threads + .into_iter() + .map(|thread| thread.join().unwrap()) + .collect::>(); + assert_eq!( + receipts + .iter() + .filter(|receipt| receipt.status == EventReceiptStatus::Created) + .count(), + 1 + ); + assert!( + receipts + .windows(2) + .all(|pair| pair[0].filename == pair[1].filename) + ); + assert_eq!( + message::list_inbox(&message::inbox_dir(&agent)) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn conflicting_reuse_and_undeclared_or_suspended_ingress_fail_closed() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + emit(catalog.path(), "same", None, false); + let conflict = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "same", + None, + Some("different"), + "different", + false, + ) + .unwrap_err() + .to_string(); + assert!( + conflict.contains("reused with different content"), + "{conflict}" + ); + let undeclared = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "other", + "1", + None, + None, + "x", + false, + ) + .unwrap_err() + .to_string(); + assert!( + undeclared.contains("does not declare stream 'other'"), + "{undeclared}" + ); + + fs::write( + agent.join("agent.kdl"), + "agent \"worker\" {\n host \"hetz\"\n desired-state \"suspended\" reason=\"hold\"\n command \"agent\"\n stream \"gh-ci\" {}\n}\n", + ) + .unwrap(); + let suspended = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "2", + None, + None, + "x", + false, + ) + .unwrap_err() + .to_string(); + assert!(suspended.contains("eyes are closed"), "{suspended}"); + assert_eq!( + message::list_inbox(&message::inbox_dir(&agent)) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn supersede_collapses_only_the_matching_key_and_preserves_archive_receipts() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let pr_1_old = emit(catalog.path(), "pr1-fail", Some("pr-1"), true); + let pr_2 = emit(catalog.path(), "pr2-fail", Some("pr-2"), true); + let pr_1_new = emit(catalog.path(), "pr1-pass", Some("pr-1"), true); + + let inbox = message::inbox_dir(&agent); + let archive = message::archive_dir(&agent); + assert!(!inbox.join(&pr_1_old.filename).exists()); + assert!(archive.join(&pr_1_old.filename).exists()); + assert!(inbox.join(&pr_2.filename).exists()); + assert!(inbox.join(&pr_1_new.filename).exists()); + assert_eq!( + pr_1_new.superseded.as_deref(), + Some(pr_1_old.filename.as_str()) + ); + let replay = emit(catalog.path(), "pr1-fail", Some("pr-1"), true); + assert_eq!(replay.status, EventReceiptStatus::Deduplicated); + assert!(!inbox.join(&pr_1_old.filename).exists()); +} + +#[test] +fn keyless_supersede_replaces_the_stream_wide_head() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let old = emit(catalog.path(), "old", None, true); + let new = emit(catalog.path(), "new", None, true); + assert_eq!(new.superseded.as_deref(), Some(old.filename.as_str())); + assert!(!message::inbox_dir(&agent).join(old.filename).exists()); +} + +#[test] +fn crash_replay_honors_an_archive_receipt_and_never_restores_the_inbox_copy() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let filename = "1784649988123-proof1.md"; + let rendered = event::render_event( + "hetz.worker/gh-ci", + Some("CI archived"), + "gh-ci", + "archived", + None, + "payload", + ); + let archive = message::archive_dir(&agent); + fs::create_dir_all(&archive).unwrap(); + fs::write(archive.join(filename), &rendered).unwrap(); + let state_dir = agent.join("resources/streams/gh-ci"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + serde_json::to_vec(&serde_json::json!({ + "version": 1, + "stream": "gh-ci", + "recipient": "hetz.worker", + "pending": { + "eventId": "archived", + "filename": filename, + "key": null, + "renderedSha256": format!("{:x}", Sha256::digest(rendered.as_bytes())), + "supersede": false + }, + "recent": [] + })) + .unwrap(), + ) + .unwrap(); + + let receipt = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "archived", + None, + Some("CI archived"), + "payload", + false, + ) + .unwrap(); + assert_eq!(receipt.status, EventReceiptStatus::Deduplicated); + assert!(!message::inbox_dir(&agent).join(filename).exists()); + assert!(archive.join(filename).exists()); +} + +#[test] +fn subject_frontmatter_injection_is_refused_before_any_write() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "safe-id", + None, + Some("safe\nevent-id: forged"), + "body", + false, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("event subject"), "{error}"); + assert!(!message::inbox_dir(&agent).exists()); + assert!(!agent.join("resources/streams").exists()); +} + +#[test] +fn stream_state_is_bounded_and_forgets_only_beyond_its_honest_horizon() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let first = emit(catalog.path(), "event-0", None, false); + for index in 1..=RING_CAPACITY { + emit(catalog.path(), &format!("event-{index}"), None, false); + } + let state = fs::read_to_string(agent.join("resources/streams/gh-ci/state.json")).unwrap(); + let state: serde_json::Value = serde_json::from_str(&state).unwrap(); + assert_eq!(state["recent"].as_array().unwrap().len(), RING_CAPACITY); + let replay = emit(catalog.path(), "event-0", None, false); + assert_eq!(replay.status, EventReceiptStatus::Created); + assert_ne!(replay.filename, first.filename); +} + +#[test] +fn event_emit_cli_returns_a_stable_json_receipt_and_ding_marks_the_record() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let mut child = Command::new(env!("CARGO_BIN_EXE_st2")) + .args([ + "--catalog", + catalog.path().to_str().unwrap(), + "event", + "emit", + "hetz.worker", + "--stream", + "gh-ci", + "--event-id", + "cli-1", + "--subject", + "CLI proof", + "--host", + "hetz", + "--json", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write as _; + child.stdin.take().unwrap().write_all(b"payload").unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let receipt: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(receipt["status"], "created"); + assert_eq!(receipt["recipient"], "hetz.worker"); + let message = message::list_inbox(&message::inbox_dir(&agent)) + .unwrap() + .remove(0); + let ding = st2::ding::poke_text(catalog.path(), "hetz", "hetz.worker", &message); + assert!( + ding.starts_with("[DING] » hetz.worker/gh-ci: CLI proof"), + "{ding}" + ); +} diff --git a/tests/hooks.rs b/tests/hooks.rs index 6f874820..563931ad 100644 --- a/tests/hooks.rs +++ b/tests/hooks.rs @@ -190,11 +190,13 @@ fn codex_materialization_verifies_before_writing_and_renders_a_versioned_path() "materialization verification must not install hooks" ); - assert!(command(&hooks_root) - .args(["hooks", "install"]) - .status() - .unwrap() - .success()); + assert!( + command(&hooks_root) + .args(["hooks", "install"]) + .status() + .unwrap() + .success() + ); let materialized = command(&hooks_root) .arg("up") .arg(&catalog) @@ -231,7 +233,10 @@ fn claude_hook_materialization_rejects_a_missing_receipt_before_writing() { .unwrap(); let stderr = String::from_utf8_lossy(&blocked.stderr); assert!(!blocked.status.success(), "{stderr}"); - assert!(stderr.contains("render plan references $ST_HOOKS"), "{stderr}"); + assert!( + stderr.contains("render plan references $ST_HOOKS"), + "{stderr}" + ); assert!(!workspace.join(".claude/settings.local.json").exists()); assert!( !hooks_root.exists(), @@ -308,10 +313,9 @@ fn claude_hook_materialization_accepts_a_valid_receipt_and_renders_a_versioned_p "{}", String::from_utf8_lossy(&materialized.stderr) ); - let settings: serde_json::Value = serde_json::from_slice( - &fs::read(workspace.join(".claude/settings.local.json")).unwrap(), - ) - .unwrap(); + let settings: serde_json::Value = + serde_json::from_slice(&fs::read(workspace.join(".claude/settings.local.json")).unwrap()) + .unwrap(); let hook = settings["hooks"]["SessionStart"][0]["hooks"][0]["command"] .as_str() .unwrap(); diff --git a/tests/materialize.rs b/tests/materialize.rs index 30766795..49ce33b0 100644 --- a/tests/materialize.rs +++ b/tests/materialize.rs @@ -585,14 +585,11 @@ fn suspended_declaration_does_not_materialize_workspace_content() { fs::create_dir_all(&workspace).unwrap(); write(&workspace.join("AGENTS.md"), "existing\n"); write(&catalog.join("_templates/AGENTS.md"), "new\n"); - let declaration = agent_kdl( - &workspace, - r#" copy "_templates/AGENTS.md" "AGENTS.md""#, - ) - .replace( - " host \"Silber\"\n", - " host \"Silber\"\n desired-state \"suspended\" reason=\"Waiting for capacity\"\n", - ); + let declaration = agent_kdl(&workspace, r#" copy "_templates/AGENTS.md" "AGENTS.md""#) + .replace( + " host \"Silber\"\n", + " host \"Silber\"\n desired-state \"suspended\" reason=\"Waiting for capacity\"\n", + ); write(&catalog.join("agents/Silber/cos/agent.kdl"), declaration); let found = discover(&catalog); diff --git a/tests/message_cli.rs b/tests/message_cli.rs index e85829e9..40dc3012 100644 --- a/tests/message_cli.rs +++ b/tests/message_cli.rs @@ -126,11 +126,8 @@ fn send_persists_canonical_sender_history_independent_of_every_recipient_box() { String::from_utf8_lossy(&output.stderr) ); let filename = String::from_utf8(output.stdout).unwrap().trim().to_string(); - let delivered = st2::message::read_msg( - &tmp.path().join("h/recipient/resources/inbox"), - &filename, - ) - .unwrap(); + let delivered = + st2::message::read_msg(&tmp.path().join("h/recipient/resources/inbox"), &filename).unwrap(); assert_eq!(delivered.from.as_deref(), Some("h.sender")); fs::remove_file( @@ -174,7 +171,10 @@ fn replies_are_indexed_with_the_canonical_recipient_and_thread_relation() { let original = send_message(tmp.path(), "recipient", "sender", "question", &[]); assert!(original.status.success()); - let original = String::from_utf8(original.stdout).unwrap().trim().to_string(); + let original = String::from_utf8(original.stdout) + .unwrap() + .trim() + .to_string(); let reply = Command::new(env!("CARGO_BIN_EXE_st2")) .args(["message", "reply", &original, "--root"]) .arg(tmp.path()) @@ -212,7 +212,10 @@ fn keyed_reply_retry_reads_an_archived_source_and_returns_the_committed_reply() let original = send_message(tmp.path(), "recipient", "sender", "question", &[]); assert!(original.status.success()); - let original = String::from_utf8(original.stdout).unwrap().trim().to_string(); + let original = String::from_utf8(original.stdout) + .unwrap() + .trim() + .to_string(); let first = Command::new(env!("CARGO_BIN_EXE_st2")) .args(["message", "reply", &original, "--root"]) .arg(tmp.path()) @@ -229,7 +232,10 @@ fn keyed_reply_retry_reads_an_archived_source_and_returns_the_committed_reply() .env("ST2_TEST_MESSAGE_SEND_FAIL_AFTER", "active-cleanup") .output() .unwrap(); - assert!(!first.status.success(), "first reply must simulate lost output"); + assert!( + !first.status.success(), + "first reply must simulate lost output" + ); let recipient_inbox = tmp.path().join("h/recipient/resources/inbox"); let reply_filename = fs::read_dir(&recipient_inbox) .unwrap() @@ -262,8 +268,15 @@ fn keyed_reply_retry_reads_an_archived_source_and_returns_the_committed_reply() ]) .output() .unwrap(); - assert!(retry.status.success(), "{}", String::from_utf8_lossy(&retry.stderr)); - assert_eq!(String::from_utf8_lossy(&retry.stdout).trim(), reply_filename); + assert!( + retry.status.success(), + "{}", + String::from_utf8_lossy(&retry.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&retry.stdout).trim(), + reply_filename + ); assert_eq!(fs::read_dir(recipient_inbox).unwrap().count(), 1); assert_eq!( String::from_utf8_lossy(&sent(tmp.path(), "sender", &["--count"]).stdout).trim(), @@ -304,12 +317,14 @@ fn keyed_retry_recovers_every_crash_boundary_without_false_sent_or_duplicates() .env("ST2_TEST_MESSAGE_SEND_FAIL_AFTER", crash_after) .output() .unwrap(); - assert!(!failed.status.success(), "{crash_after} must inject failure"); + assert!( + !failed.status.success(), + "{crash_after} must inject failure" + ); let interrupted = sent(tmp.path(), "sender", &["--json"]); assert!(interrupted.status.success()); - let interrupted: serde_json::Value = - serde_json::from_slice(&interrupted.stdout).unwrap(); + let interrupted: serde_json::Value = serde_json::from_slice(&interrupted.stdout).unwrap(); assert_ne!(interrupted["coverage"]["_tag"], "unavailable"); if matches!( crash_after, @@ -387,21 +402,24 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva ] { let tmp = prepare(); let messages = tmp.path().join("h/sender/resources/sent/messages"); - let row = fs::read_dir(&messages).unwrap().next().unwrap().unwrap().path(); + let row = fs::read_dir(&messages) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); match mutate { "missing-directory" => fs::remove_dir_all(&messages).unwrap(), "missing-row" => fs::remove_file(&row).unwrap(), "extra-row" => { let extra = messages.join("1700000000000-aaaaaa.md.json"); - let value = fs::read_to_string(&row) - .unwrap() - .replace( - serde_json::from_slice::(&fs::read(&row).unwrap()) - .unwrap()["filename"] - .as_str() - .unwrap(), - "1700000000000-aaaaaa.md", - ); + let value = fs::read_to_string(&row).unwrap().replace( + serde_json::from_slice::(&fs::read(&row).unwrap()).unwrap() + ["filename"] + .as_str() + .unwrap(), + "1700000000000-aaaaaa.md", + ); fs::write(extra, value).unwrap(); } "unexpected-row-entry" => fs::write(messages.join("unexpected"), "junk").unwrap(), @@ -440,7 +458,12 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva ] { let tmp = prepare(); let commits = tmp.path().join("h/sender/resources/sent/commits"); - let node = fs::read_dir(&commits).unwrap().next().unwrap().unwrap().path(); + let node = fs::read_dir(&commits) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); match mutate { "missing-node" => fs::remove_file(&node).unwrap(), "extra-node" => { @@ -465,8 +488,11 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva } let digest = json_digest(&value); fs::remove_file(&node).unwrap(); - fs::write(commits.join(format!("{digest}.json")), serde_json::to_vec(&value).unwrap()) - .unwrap(); + fs::write( + commits.join(format!("{digest}.json")), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); let head = tmp.path().join("h/sender/resources/sent/index.json"); let mut head_value: serde_json::Value = serde_json::from_slice(&fs::read(&head).unwrap()).unwrap(); @@ -530,7 +556,10 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva assert!(output.stdout.is_empty()); if mutate == "tip-format" { let retry = send_message(tmp.path(), "sender", "recipient", "next", &[]); - assert!(!retry.status.success(), "invalid tip must fail before node lookup"); + assert!( + !retry.status.success(), + "invalid tip must fail before node lookup" + ); } } @@ -580,7 +609,10 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva "durable", &["--idempotency-key", "stable"], ); - assert!(!retry.status.success(), "key filename must fail before row lookup"); + assert!( + !retry.status.success(), + "key filename must fail before row lookup" + ); } } @@ -591,7 +623,10 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva fs::write(path.join(".message.tmp-999-0"), "interrupted atomic write").unwrap(); } let output = sent(tmp.path(), "sender", &["--json"]); - assert!(output.status.success(), "atomic-write temporary siblings stay invisible"); + assert!( + output.status.success(), + "atomic-write temporary siblings stay invisible" + ); let tmp = tempfile::tempdir().unwrap(); write_agent(tmp.path(), "sender"); @@ -605,17 +640,28 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva .unwrap(); assert!(!interrupted.status.success()); let pending = tmp.path().join("h/sender/resources/sent/pending"); - let pending_record = fs::read_dir(&pending).unwrap().next().unwrap().unwrap().path(); + let pending_record = fs::read_dir(&pending) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); let mut value: serde_json::Value = serde_json::from_slice(&fs::read(&pending_record).unwrap()).unwrap(); value["body"] = "substituted pending\n".into(); value["renderedMessage"] = "---\nfrom: h.sender\n---\nsubstituted pending\n".into(); fs::write(&pending_record, serde_json::to_vec(&value).unwrap()).unwrap(); let output = sent(tmp.path(), "sender", &["--json"]); - assert!(!output.status.success(), "substituted pending intent must fail closed"); + assert!( + !output.status.success(), + "substituted pending intent must fail closed" + ); assert!(output.stdout.is_empty()); let retry = send_message(tmp.path(), "sender", "recipient", "next", &[]); - assert!(!retry.status.success(), "next sender operation must reject substituted pending"); + assert!( + !retry.status.success(), + "next sender operation must reject substituted pending" + ); let tmp = tempfile::tempdir().unwrap(); write_agent(tmp.path(), "sender"); @@ -629,13 +675,21 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva .unwrap(); assert!(!interrupted.status.success()); let messages = tmp.path().join("h/sender/resources/sent/messages"); - let row = fs::read_dir(&messages).unwrap().next().unwrap().unwrap().path(); + let row = fs::read_dir(&messages) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); let mut value: serde_json::Value = serde_json::from_slice(&fs::read(&row).unwrap()).unwrap(); value["body"] = "substituted row\n".into(); value["renderedMessage"] = "---\nfrom: h.sender\n---\nsubstituted row\n".into(); fs::write(row, serde_json::to_vec(&value).unwrap()).unwrap(); let output = sent(tmp.path(), "sender", &["--json"]); - assert!(!output.status.success(), "active-owned substituted row must fail closed"); + assert!( + !output.status.success(), + "active-owned substituted row must fail closed" + ); assert!(output.stdout.is_empty()); let tmp = tempfile::tempdir().unwrap(); @@ -660,7 +714,10 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva ) .unwrap(); let output = sent(tmp.path(), "sender", &["--json"]); - assert!(!output.status.success(), "missing pending intent must fail closed"); + assert!( + !output.status.success(), + "missing pending intent must fail closed" + ); assert!(output.stdout.is_empty()); let tmp = tempfile::tempdir().unwrap(); @@ -679,7 +736,10 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva value["recordDigest"] = "substituted".into(); fs::write(active, serde_json::to_vec(&value).unwrap()).unwrap(); let output = sent(tmp.path(), "sender", &["--json"]); - assert!(!output.status.success(), "substituted active digest must fail closed"); + assert!( + !output.status.success(), + "substituted active digest must fail closed" + ); assert!(output.stdout.is_empty()); let tmp = tempfile::tempdir().unwrap(); @@ -695,20 +755,30 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva assert!(!interrupted.status.success()); fs::remove_file(tmp.path().join("h/sender/resources/sent/active.json")).unwrap(); let output = sent(tmp.path(), "sender", &["--json"]); - assert!(!output.status.success(), "committed pending without active must fail closed"); + assert!( + !output.status.success(), + "committed pending without active must fail closed" + ); assert!(output.stdout.is_empty()); let retry = send_message(tmp.path(), "sender", "recipient", "committed", &[]); - assert!(!retry.status.success(), "recovery must not commit the same row twice"); + assert!( + !retry.status.success(), + "recovery must not commit the same row twice" + ); let tmp = tempfile::tempdir().unwrap(); write_agent(tmp.path(), "sender"); write_agent(tmp.path(), "recipient"); - assert!(send_message(tmp.path(), "sender", "recipient", "older", &[]) - .status - .success()); - assert!(send_message(tmp.path(), "sender", "recipient", "newer", &[]) - .status - .success()); + assert!( + send_message(tmp.path(), "sender", "recipient", "older", &[]) + .status + .success() + ); + assert!( + send_message(tmp.path(), "sender", "recipient", "newer", &[]) + .status + .success() + ); let messages = tmp.path().join("h/sender/resources/sent/messages"); let mut rows = fs::read_dir(&messages) .unwrap() @@ -717,16 +787,26 @@ fn sent_ledger_fails_closed_when_head_nodes_or_rows_are_lost_substituted_or_inva rows.sort(); let pending = tmp.path().join("h/sender/resources/sent/pending"); let older = fs::read(&rows[0]).unwrap(); - fs::write(pending.join(format!("{}.json", bytes_digest(&older))), older).unwrap(); + fs::write( + pending.join(format!("{}.json", bytes_digest(&older))), + older, + ) + .unwrap(); let output = sent(tmp.path(), "sender", &["--json"]); - assert!(!output.status.success(), "older committed row cannot become pending again"); + assert!( + !output.status.success(), + "older committed row cannot become pending again" + ); assert!(output.stdout.is_empty()); assert!( String::from_utf8_lossy(&output.stderr) .contains("committed pending intent is missing its active marker") ); let retry = send_message(tmp.path(), "sender", "recipient", "next", &[]); - assert!(!retry.status.success(), "recovery must not recommit an older row"); + assert!( + !retry.status.success(), + "recovery must not recommit an older row" + ); assert!( String::from_utf8_lossy(&retry.stderr) .contains("committed pending intent is missing its active marker") @@ -1064,10 +1144,16 @@ fn send_routes_only_by_stable_identity_in_a_catalog_and_preserves_catalogless_bu ); fs::create_dir_all(catalog.path().join("requester/inbox")).unwrap(); - assert!(!send(catalog.path(), "requester", "--catalog", None).status.success()); - assert!(!send(catalog.path(), "requester", "--catalog", Some("other")) - .status - .success()); + assert!( + !send(catalog.path(), "requester", "--catalog", None) + .status + .success() + ); + assert!( + !send(catalog.path(), "requester", "--catalog", Some("other")) + .status + .success() + ); let external = send(catalog.path(), "requester", "--catalog", Some("requester")); assert!( external.status.success(), diff --git a/tests/parked_recovery.rs b/tests/parked_recovery.rs index 632013ff..98b5c9df 100644 --- a/tests/parked_recovery.rs +++ b/tests/parked_recovery.rs @@ -87,7 +87,12 @@ impl Fleet { } /// Poll the inventory until `row` satisfies `done`, or fail with what it last looked like. - fn until(&self, runtime_id: &str, what: &str, done: impl Fn(&serde_json::Value) -> bool) -> serde_json::Value { + fn until( + &self, + runtime_id: &str, + what: &str, + done: impl Fn(&serde_json::Value) -> bool, + ) -> serde_json::Value { let deadline = Instant::now() + DEADLINE; let mut last = self.row(runtime_id); while Instant::now() < deadline { @@ -144,7 +149,10 @@ fn repaired_flapper(catalog: &Path) { } fn generation(row: &serde_json::Value) -> (serde_json::Value, serde_json::Value) { - (row["runtime"]["pid"].clone(), row["runtime"]["generationId"].clone()) + ( + row["runtime"]["pid"].clone(), + row["runtime"]["generationId"].clone(), + ) } #[test] @@ -251,8 +259,14 @@ fn a_real_supervisor_parks_a_crash_looper_and_unpark_recovers_only_that_task() { // 6. And it stays recovered past the policy's own observation window, rather than for one pass. while recovered_at.elapsed() < INTERVAL + Duration::from_secs(1) { let row = fleet.row(FLAPPER); - assert_eq!(row["runtime"]["state"], "running", "the recovered task fell over again"); - assert!(row["parked"].is_null(), "the recovered task re-parked: {row}"); + assert_eq!( + row["runtime"]["state"], "running", + "the recovered task fell over again" + ); + assert!( + row["parked"].is_null(), + "the recovered task re-parked: {row}" + ); std::thread::sleep(Duration::from_millis(100)); } assert_eq!( diff --git a/tests/reconcile.rs b/tests/reconcile.rs index a4a208fd..fea00863 100644 --- a/tests/reconcile.rs +++ b/tests/reconcile.rs @@ -3,9 +3,9 @@ use std::collections::BTreeMap; use std::path::PathBuf; +use st2::reconcile::ObservedPtyPresentation; use st2::reconcile::reconcile_selected; use st2::reconcile::resolve_task; -use st2::reconcile::ObservedPtyPresentation; use st2::spec::{AgentDesiredState, AgentSpec, JobType, Resource, Task, TaskKind, TaskLifecycle}; use st2::{Session, reconcile as reconcile_result}; @@ -138,9 +138,22 @@ fn selected_reconcile_launches_missing_and_adopts_live_without_siblings() { assert_eq!(plan.launch.len(), 1); assert_eq!(plan.launch[0].tasks.len(), 1); assert_eq!(plan.launch[0].tasks[0].pty_id, "host.a.x"); - let plan2 = reconcile_selected(&specs, &[live("host.a.x"), live("host.a.y"), live("host.b.z")], "host", "host.a.x").unwrap(); + let plan2 = reconcile_selected( + &specs, + &[live("host.a.x"), live("host.a.y"), live("host.b.z")], + "host", + "host.a.x", + ) + .unwrap(); assert!(plan2.launch.is_empty() && plan2.gc.is_empty() && plan2.teardown.is_empty()); - assert_eq!(plan2.adopt.iter().map(|s| s.identity.as_str()).collect::>(), vec!["a"]); + assert_eq!( + plan2 + .adopt + .iter() + .map(|s| s.identity.as_str()) + .collect::>(), + vec!["a"] + ); } #[test] @@ -234,7 +247,13 @@ fn selected_reconcile_action_ids_are_exact_and_refusals_immutable() { let sessions = vec![live("host.a.y"), live("host.b.z")]; let before = (specs.clone(), sessions.clone()); let p = reconcile_selected(&specs, &sessions, "host", "host.a.x").unwrap(); - assert_eq!(p.launch.iter().flat_map(|l| l.tasks.iter().map(|t| t.pty_id.as_str())).collect::>(), vec!["host.a.x"]); + assert_eq!( + p.launch + .iter() + .flat_map(|l| l.tasks.iter().map(|t| t.pty_id.as_str())) + .collect::>(), + vec!["host.a.x"] + ); assert!(p.gc.is_empty() && p.teardown.is_empty()); assert!(reconcile_selected(&specs, &sessions, "host", "host.a.missing").is_err()); assert_eq!((specs, sessions), before); @@ -291,9 +310,19 @@ fn selected_dead_non_keep_gc_and_relaunch_only_selected() { } #[test] fn selected_retired_live_tears_down_only_selected() { - let mut s = svc("a", None, vec![task(TaskKind::Exec, "x", None, Some("a")), task(TaskKind::Exec, "sib", None, Some("b"))]); + let mut s = svc( + "a", + None, + vec![ + task(TaskKind::Exec, "x", None, Some("a")), + task(TaskKind::Exec, "sib", None, Some("b")), + ], + ); s.desired_state = AgentDesiredState::Retired { reason: None }; - let specs = [s, svc("b", None, vec![task(TaskKind::Exec, "z", None, Some("c"))])]; + let specs = [ + s, + svc("b", None, vec![task(TaskKind::Exec, "z", None, Some("c"))]), + ]; let p = reconcile_selected( &specs, &[live("host.a.x"), live("host.a.sib"), live("host.b.z")], @@ -301,7 +330,13 @@ fn selected_retired_live_tears_down_only_selected() { "host.a.x", ) .unwrap(); - assert_eq!(p.teardown.iter().flat_map(|t| t.pty_ids.iter().map(String::as_str)).collect::>(), vec!["host.a.x"]); + assert_eq!( + p.teardown + .iter() + .flat_map(|t| t.pty_ids.iter().map(String::as_str)) + .collect::>(), + vec!["host.a.x"] + ); assert!(p.launch.is_empty() && p.gc.is_empty()); } #[test] @@ -401,6 +436,7 @@ fn spec( delivery: None, driver: None, resources: Vec::new(), + streams: Vec::new(), tasks, path: PathBuf::from(format!( "/cat/agents/{}/{identity}/agent.kdl", @@ -493,7 +529,11 @@ fn resuming_uses_ordinary_reconcile_and_does_not_override_keep() { let specs = [spec]; let plan = reconcile(&specs, &[dead("host.idle.agent")], "host"); assert!(plan.launch.is_empty()); - assert_eq!(plan.adopt.len(), 1, "resume preserves the existing keep contract"); + assert_eq!( + plan.adopt.len(), + 1, + "resume preserves the existing keep contract" + ); } fn live(id: &str) -> Session { @@ -565,12 +605,7 @@ fn live_pty_presentation_is_exact_id_metadata_and_not_lifecycle_drift() { "worker", Some(HOST), vec![ - task( - TaskKind::Pty, - "agent", - Some("hetz.worker"), - Some("codex"), - ), + task(TaskKind::Pty, "agent", Some("hetz.worker"), Some("codex")), task( TaskKind::Pty, "shell", @@ -602,7 +637,10 @@ fn live_pty_presentation_is_exact_id_metadata_and_not_lifecycle_drift() { primary.tags, BTreeMap::from([ ("agent.presentation.schema".to_owned(), Some("1".to_owned())), - ("agent.actor.path".to_owned(), Some("hetz.worker".to_owned())), + ( + "agent.actor.path".to_owned(), + Some("hetz.worker".to_owned()) + ), ( "agent.presentation.description".to_owned(), Some("Owns build delivery".to_owned()), @@ -700,7 +738,12 @@ fn live_pty_presentation_only_queues_observed_drift() { }), }; assert_eq!(reconcile(&specs, &[drifted], HOST).presentation.len(), 1); - assert_eq!(reconcile(&specs, &[live("hetz.worker")], HOST).presentation.len(), 1); + assert_eq!( + reconcile(&specs, &[live("hetz.worker")], HOST) + .presentation + .len(), + 1 + ); } #[test] @@ -710,13 +753,8 @@ fn resource_only_changes_do_not_replace_or_relaunch_a_live_task() { Some(HOST), vec![task(TaskKind::Pty, "agent", Some("hetz.a"), Some("x"))], ); - spec.resources.push( - Resource::new( - "work".into(), - "github-issue://example/project/41".into(), - ) - .unwrap(), - ); + spec.resources + .push(Resource::new("work".into(), "github-issue://example/project/41".into()).unwrap()); let specs = [spec]; let plan = reconcile(&specs, &[live("hetz.a")], HOST); @@ -954,7 +992,9 @@ fn adopting_a_live_task_proves_it_alive_to_the_restart_cap() { vec![task(TaskKind::Pty, "agent", None, Some("run"))], )]; // Take the runtime id from the launch plan itself rather than restating the derivation. - let runtime = reconcile(&specs, &[], HOST).launch[0].tasks[0].pty_id.clone(); + let runtime = reconcile(&specs, &[], HOST).launch[0].tasks[0] + .pty_id + .clone(); let plan = reconcile(&specs, &[live(&runtime)], HOST); @@ -979,7 +1019,9 @@ fn selecting_a_live_task_proves_it_alive_to_the_restart_cap() { None, vec![task(TaskKind::Pty, "agent", None, Some("run"))], )]; - let runtime = reconcile(&specs, &[], HOST).launch[0].tasks[0].pty_id.clone(); + let runtime = reconcile(&specs, &[], HOST).launch[0].tasks[0] + .pty_id + .clone(); let plan = reconcile_selected(&specs, &[live(&runtime)], HOST, &runtime).unwrap(); diff --git a/tests/request_cli.rs b/tests/request_cli.rs index 0dadb7f5..87369a74 100644 --- a/tests/request_cli.rs +++ b/tests/request_cli.rs @@ -450,9 +450,7 @@ fn request_status_propagates_non_not_found_message_directory_errors() { ); assert!(sent.status.success()); - let inbox = tmp - .path() - .join("principals/h/example-ci/resources/inbox"); + let inbox = tmp.path().join("principals/h/example-ci/resources/inbox"); fs::create_dir_all(inbox.parent().unwrap()).unwrap(); fs::write(&inbox, "not a directory").unwrap(); assert!(inbox.is_file()); diff --git a/tests/run.rs b/tests/run.rs index c1da5842..290b38d8 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -8,7 +8,8 @@ use std::time::Instant; use st2::message; use st2::reconcile::{ - Launch, PtyPresentation, ReconcilePlan, Session, TaskLaunch, TaskTarget, Teardown, + Launch, PtyPresentation, ReconcilePlan, Session, TaskCompileContext, TaskLaunch, TaskTarget, + Teardown, compile_generated_tasks, }; use st2::run::Runner; use st2::run::{CrashLoop, surface_crash_loop, up_once_selected, up_once_selected_specs}; @@ -239,6 +240,7 @@ fn task_spec(identity: &str, host: Option<&str>, id: &str) -> AgentSpec { delivery: None, driver: None, resources: vec![], + streams: Vec::new(), tasks: vec![Task { kind: TaskKind::Exec, derived: false, @@ -1062,11 +1064,8 @@ command = "st2 ding hetz.demo" #[test] fn retired_compact_agent_stops_agent_and_derived_ding() { let tmp = tempfile::tempdir().unwrap(); - let retired = COMPACT_AGENT_WITH_DING.replacen( - " host \"hetz\"", - " host \"hetz\"\n retired #true", - 1, - ); + let retired = + COMPACT_AGENT_WITH_DING.replacen(" host \"hetz\"", " host \"hetz\"\n retired #true", 1); write(tmp.path(), "agents/hetz/demo/agent.kdl", &retired); let runner = FakeRunner { sessions: vec![live("hetz.demo"), live("hetz.demo.ding")], @@ -1112,10 +1111,11 @@ fn suspend_and_resume_cover_derived_ding_sibling_continuity_and_inbox_retention( assert_eq!(suspended_report.torn_down, ["hetz.demo", "hetz.demo.ding"]); assert_eq!(suspended_report.adopted, ["sibling"]); assert!(suspended_report.launched.is_empty()); - assert!(tmp - .path() - .join("agents/hetz/demo/resources/inbox/1234567890000-proof.md") - .is_file()); + assert!( + tmp.path() + .join("agents/hetz/demo/resources/inbox/1234567890000-proof.md") + .is_file() + ); write(tmp.path(), "agents/hetz/demo/agent.kdl", running); let resume_runner = FakeRunner { @@ -1129,10 +1129,11 @@ fn suspend_and_resume_cover_derived_ding_sibling_continuity_and_inbox_retention( let resumed_report = up_once(tmp.path(), "hetz", &resume_runner).unwrap(); assert_eq!(resumed_report.restarted, ["hetz.demo", "hetz.demo.ding"]); assert_eq!(resumed_report.adopted, ["sibling"]); - assert!(tmp - .path() - .join("agents/hetz/demo/resources/inbox/1234567890000-proof.md") - .is_file()); + assert!( + tmp.path() + .join("agents/hetz/demo/resources/inbox/1234567890000-proof.md") + .is_file() + ); } #[test] @@ -1454,11 +1455,7 @@ fn an_operator_recovers_one_parked_task_without_disturbing_a_healthy_peer() { // Phase 1 — the flapper dies before every pass; the peer is up and stays up. let crashing = FakeRunner { - sessions: vec![ - dead("hetz.demo"), - live("hetz.demo.ding"), - live("hetz.peer"), - ], + sessions: vec![dead("hetz.demo"), live("hetz.demo.ding"), live("hetz.peer")], ..Default::default() }; let mut parked_report = UpReport::default(); @@ -1474,7 +1471,10 @@ fn an_operator_recovers_one_parked_task_without_disturbing_a_healthy_peer() { // The park is legible to a separate reader — the entire point of #204. let observer = DirParkObserver::new(state.path().join("parked")); let batch = observer.observe(&["hetz.demo".to_string(), "hetz.peer".to_string()]); - assert!(batch.complete, "a park is a known fault, not missing evidence"); + assert!( + batch.complete, + "a park is a known fault, not missing evidence" + ); let ParkState::Parked(record) = batch.state("hetz.demo") else { panic!("the parked task is not visible in the projection"); }; @@ -1576,7 +1576,11 @@ fn an_unpark_request_for_a_task_that_is_not_parked_says_so() { assert!(report.unparked.is_empty()); assert_eq!(report.warnings.len(), 1); - assert!(report.warnings[0].contains("hetz.typo"), "{:?}", report.warnings); + assert!( + report.warnings[0].contains("hetz.typo"), + "{:?}", + report.warnings + ); } /// A parked crash-loop is surfaced to the agent's supervisor over the native bus: a `crash-loop`-tagged @@ -1722,3 +1726,410 @@ fn up_once_marks_a_list_failure_as_a_skipped_pass() { vec!["list sessions (pass skipped): simulated list failure"] ); } + +// --------------------------------------------------------------------------------------------- +// Streams (DQ1 spike) — a declared event SOURCE lowers to a derived exec companion, so it inherits +// the derived-companion lifecycle wholesale. Every test below is the derived-DING proof for the +// same guarantee, re-run against `stream-gh-ci`, plus the two claims a stream adds that DING does not: +// it must not make an otherwise-empty agent runnable, and it must not disturb its agent when it +// crash-loops. +// --------------------------------------------------------------------------------------------- + +/// An agent with BOTH companions. Every stream test carries the ding too, so a claim about the stream +/// is also a claim that the two derived siblings stay independent. +const COMPACT_AGENT_WITH_STREAM: &str = r#" +agent "demo" { + host "hetz" + supervisor "cos-claude" + command "true" + ding + stream "gh-ci" { command "poll-gh-ci.sh" } + restart { attempts 1; interval "60s"; delay "0s"; mode "fail" } +} +"#; + +const COMPACT_STREAM_ONLY_AGENT: &str = r#" +agent "sourceless" { + host "hetz" + stream "gh-ci" { command "poll-gh-ci.sh" } +} +"#; + +/// 4a. One reconcile pass launches the agent and BOTH derived companions — no second pass, no +/// ordering ceremony at the call site, with the declared adapter launch carried through verbatim. +#[test] +fn fresh_compact_agent_launches_with_its_derived_stream() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/hetz/demo/agent.kdl", + COMPACT_AGENT_WITH_STREAM, + ); + + let runner = FakeRunner::default(); + let report = up_once(tmp.path(), "hetz", &runner).unwrap(); + + assert_eq!( + report.launched, + ["hetz.demo", "hetz.demo.ding", "hetz.demo.stream-gh-ci"] + ); + let targets = runner.spawned_targets.borrow(); + let stream = targets + .iter() + .find(|target| target.pty_id == "hetz.demo.stream-gh-ci") + .unwrap(); + assert_eq!( + stream.kind, + TaskKind::Exec, + "a stream source needs no terminal" + ); + assert!(stream.derived, "a stream companion is runner-generated"); + assert_eq!(&stream.launch, &TaskLaunch::Shell("poll-gh-ci.sh".into())); + // Runner-owned task identity reaches the stream exactly as it reaches every other task. + assert_eq!( + stream.env.get("ST_AGENT").map(String::as_str), + Some("hetz.demo") + ); +} + +/// 4b (retire). Retirement tears down the agent and BOTH companions in the same pass. +#[test] +fn retired_compact_agent_stops_agent_and_derived_stream() { + let tmp = tempfile::tempdir().unwrap(); + let retired = COMPACT_AGENT_WITH_STREAM.replacen( + " host \"hetz\"", + " host \"hetz\"\n retired #true", + 1, + ); + write(tmp.path(), "agents/hetz/demo/agent.kdl", &retired); + let runner = FakeRunner { + sessions: vec![ + live("hetz.demo"), + live("hetz.demo.ding"), + live("hetz.demo.stream-gh-ci"), + ], + ..Default::default() + }; + + let report = up_once(tmp.path(), "hetz", &runner).unwrap(); + + assert_eq!( + report.torn_down, + ["hetz.demo", "hetz.demo.ding", "hetz.demo.stream-gh-ci"] + ); + assert!(report.launched.is_empty()); +} + +/// 4b (suspend). A suspended agent stops its stream with it, and a sibling agent is untouched. +#[test] +fn suspended_compact_agent_stops_its_derived_stream_without_touching_a_sibling() { + let tmp = tempfile::tempdir().unwrap(); + let suspended = COMPACT_AGENT_WITH_STREAM.replacen( + " host \"hetz\"", + " host \"hetz\"\n desired-state \"suspended\" reason=\"Waiting for CI budget\"", + 1, + ); + write(tmp.path(), "agents/hetz/demo/agent.kdl", &suspended); + write( + tmp.path(), + "agents/hetz/sibling/agent.kdl", + "agent \"sibling\" { host \"hetz\"; command \"true\" }\n", + ); + let runner = FakeRunner { + sessions: vec![ + live("hetz.demo"), + live("hetz.demo.ding"), + live("hetz.demo.stream-gh-ci"), + live("hetz.sibling"), + ], + ..Default::default() + }; + + let report = up_once(tmp.path(), "hetz", &runner).unwrap(); + + assert_eq!( + report.torn_down, + ["hetz.demo", "hetz.demo.ding", "hetz.demo.stream-gh-ci"] + ); + assert_eq!(report.adopted, ["sibling"]); + assert!(report.launched.is_empty()); +} + +#[test] +fn suspend_and_resume_relaunch_the_agent_and_stream_together() { + let tmp = tempfile::tempdir().unwrap(); + let suspended = COMPACT_AGENT_WITH_STREAM.replacen( + " host \"hetz\"", + " host \"hetz\"\n desired-state \"suspended\" reason=\"Waiting for capacity\"", + 1, + ); + write(tmp.path(), "agents/hetz/demo/agent.kdl", &suspended); + let suspend_runner = FakeRunner { + sessions: vec![ + live("hetz.demo"), + live("hetz.demo.ding"), + live("hetz.demo.stream-gh-ci"), + ], + ..Default::default() + }; + let suspended_report = up_once(tmp.path(), "hetz", &suspend_runner).unwrap(); + assert_eq!( + suspended_report.torn_down, + ["hetz.demo", "hetz.demo.ding", "hetz.demo.stream-gh-ci"] + ); + + write( + tmp.path(), + "agents/hetz/demo/agent.kdl", + COMPACT_AGENT_WITH_STREAM, + ); + let resume_runner = FakeRunner { + sessions: vec![ + dead("hetz.demo"), + dead("hetz.demo.ding"), + dead("hetz.demo.stream-gh-ci"), + ], + ..Default::default() + }; + let resumed_report = up_once(tmp.path(), "hetz", &resume_runner).unwrap(); + assert_eq!( + resumed_report.restarted, + ["hetz.demo", "hetz.demo.ding", "hetz.demo.stream-gh-ci"] + ); +} + +/// A held (adopt-only) agent stops a live stream, exactly as it stops a live ding. +#[test] +fn held_adopt_only_compact_agent_stops_its_live_derived_stream() { + let tmp = tempfile::tempdir().unwrap(); + let held = COMPACT_AGENT_WITH_STREAM.replacen( + " command \"true\"", + " command \"true\"\n lifecycle \"adopt-only\"", + 1, + ); + write(tmp.path(), "agents/hetz/demo/agent.kdl", &held); + let runner = FakeRunner { + sessions: vec![dead("hetz.demo"), live("hetz.demo.stream-gh-ci")], + ..Default::default() + }; + + let report = up_once(tmp.path(), "hetz", &runner).unwrap(); + + assert_eq!(report.held, ["hetz.demo"]); + assert_eq!(report.torn_down, ["hetz.demo.stream-gh-ci"]); + assert!(report.launched.is_empty()); +} + +/// 4c. THE claim the hypothesis rests on: a stream source that keeps dying exhausts the agent's +/// `mode = fail` budget, parks, and is surfaced as a crash-loop record — while its agent and its +/// ding sibling are never touched. +/// +/// `up_once` cannot express this (both single-pass entry points build a fresh `FlappingCap`, so a +/// one-shot reconcile can never park anything), so this drives `discover` + `reconcile`/`execute` +/// with ONE cap across passes, exactly as `flapping_cap_parks_a_fail_mode_task_that_keeps_dying` +/// does. The live agent and live ding are the negative controls: a host-wide or agent-wide reaction +/// would name them in the runner's op log, and nothing does. +#[test] +fn a_crash_looping_stream_parks_and_surfaces_without_disturbing_its_agent() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/hetz/demo/agent.kdl", + COMPACT_AGENT_WITH_STREAM, + ); + let found = discover(tmp.path()); + let runner = FakeRunner { + sessions: vec![ + live("hetz.demo"), + live("hetz.demo.ding"), + dead("hetz.demo.stream-gh-ci"), + ], + ..Default::default() + }; + let mut cap = FlappingCap::default(); + + let mut last = UpReport::default(); + for _ in 0..4 { + let plan = reconcile(&found.specs, &runner.sessions, "hetz"); + last = UpReport::default(); + execute(&plan, &runner, &mut cap, &mut last); + } + + assert_eq!(last.flapping, ["hetz.demo.stream-gh-ci"]); + assert!(last.launched.is_empty()); + assert!( + last.gc.is_empty(), + "a parked stream keeps its corpse as evidence" + ); + assert_eq!( + runner.spawned.borrow().len(), + 1, + "attempts = 1 bounds the relaunches" + ); + + // The agent and its ding sibling take NO lifecycle op — not spawned, not killed, not reaped. + // This is the "without affecting the agent" half of the claim, and it is a stronger check than + // comparing a pid because it also catches a kill followed by an identical relaunch. + // + // `patch:` is deliberately excluded: presentation patching is the pass's ordinary cosmetic + // batch for every live task and carries no lifecycle meaning. Measured, not assumed — the + // agent DOES appear in the raw op log, as `patch:hetz.demo` and nothing else. + let ops = runner.ops.borrow(); + let lifecycle_ops = ops + .iter() + .filter(|op| !op.starts_with("patch:")) + .cloned() + .collect::>(); + assert_eq!( + lifecycle_ops, + [ + "reap:hetz.demo.stream-gh-ci", + "spawn:hetz.demo.stream-gh-ci" + ], + "only the stream may be touched while it crash-loops" + ); + assert!( + ops.iter().filter(|op| !op.starts_with("patch:")).count() < ops.len(), + "the agent is still presented; this test's claim is about lifecycle ops" + ); + + // …and it surfaces. The crash-loop record carries the parked TASK, its owning agent, and the + // supervisor to notify, so `surface_crash_loop` can deliver it over the bus unchanged. + assert_eq!(last.crash_loops.len(), 1); + let cl = &last.crash_loops[0]; + assert_eq!(cl.pty_id, "hetz.demo.stream-gh-ci"); + assert_eq!(cl.identity, "demo"); + assert_eq!(cl.supervisor.as_deref(), Some("cos-claude")); + assert_eq!(cl.agent_bus_id("hetz"), "hetz.demo"); + + write( + tmp.path(), + "agents/hetz/cos-claude/agent.kdl", + "agent \"cos-claude\" { host \"hetz\"; command \"true\" }\n", + ); + surface_crash_loop(tmp.path(), "hetz", cl); + let inbox = message::inbox_dir(&tmp.path().join("agents/hetz/cos-claude")); + let msgs = message::list_dir(&inbox).unwrap(); + assert_eq!(msgs.len(), 1); + assert!(msgs[0].tags.contains(&"crash-loop".to_string())); + assert!( + msgs[0].body.contains("hetz.demo.stream-gh-ci"), + "the supervisor is told WHICH task parked, not just which agent" + ); +} + +/// A parked AGENT still stops its live stream — the coupling runs in both directions. +#[test] +fn parked_compact_agent_stops_its_live_derived_stream() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/hetz/demo/agent.kdl", + COMPACT_AGENT_WITH_STREAM, + ); + let found = discover(tmp.path()); + let runner = FakeRunner { + sessions: vec![dead("hetz.demo"), live("hetz.demo.stream-gh-ci")], + ..Default::default() + }; + let mut cap = FlappingCap::default(); + + let first = reconcile(&found.specs, &runner.sessions, "hetz"); + execute(&first, &runner, &mut cap, &mut UpReport::default()); + let second = reconcile(&found.specs, &runner.sessions, "hetz"); + let mut report = UpReport::default(); + execute(&second, &runner, &mut cap, &mut report); + + assert_eq!(report.flapping, ["hetz.demo"]); + assert!( + runner + .killed + .borrow() + .contains(&"hetz.demo.stream-gh-ci".to_string()) + ); +} + +/// A missing stream under targeted reconciliation is HELD, never broadened to its agent — the same +/// guarantee `selected_missing_derived_ding_is_held_without_broadening_to_its_agent` states. +#[test] +fn selected_missing_derived_stream_is_held_without_broadening_to_its_agent() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/hetz/demo/agent.kdl", + COMPACT_AGENT_WITH_STREAM, + ); + let runner = FakeRunner::default(); + + let report = up_once_selected(tmp.path(), "hetz.demo.stream-gh-ci", "hetz", &runner).unwrap(); + + assert_eq!(runner.list_calls.get(), 1); + assert_eq!(report.held, ["hetz.demo.stream-gh-ci"]); + assert!(report.launched.is_empty()); + assert!( + runner.spawned.borrow().is_empty(), + "targeted reconciliation must not broaden to the agent" + ); + assert!(runner.reaped.borrow().is_empty()); +} + +/// A stream is a companion, not work. `is_runnable` filters derived tasks, so a declaration whose +/// only launch is a stream is unrunnable — correct, and asserted here rather than bent. +#[test] +fn a_stream_alone_does_not_make_an_agent_runnable() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/hetz/sourceless/agent.kdl", + COMPACT_STREAM_ONLY_AGENT, + ); + let runner = FakeRunner::default(); + + let report = up_once(tmp.path(), "hetz", &runner).unwrap(); + + assert_eq!(report.unrunnable, ["sourceless"]); + assert!(report.launched.is_empty()); + assert!(runner.spawned.borrow().is_empty()); +} + +/// A stream must not accidentally claim a delivery transport: `has_delivery_transport` is scoped to +/// the derived `ding` companion, and an agent with a stream but no `ding` still has no transport. +#[test] +fn a_stream_does_not_claim_a_delivery_transport() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/hetz/demo/agent.kdl", + "agent \"demo\" { host \"hetz\"; command \"true\"; stream \"gh-ci\" { command \"poll.sh\" } }\n", + ); + let found = discover(tmp.path()); + let spec = &found.specs[0]; + + assert!(!spec.has_delivery_transport()); + assert!(spec.is_runnable()); + let names = spec + .tasks + .iter() + .map(|task| task.name.as_str()) + .collect::>(); + assert_eq!(names, ["agent", "stream-gh-ci"]); +} + +/// The "unsupported derived task" gate stays fail-closed: extending it for streams must not have +/// turned it into a permissive fall-through. +#[test] +fn an_unknown_derived_task_still_refuses_the_pass() { + let tmp = tempfile::tempdir().unwrap(); + let mut spec = task_spec("demo", Some("hetz"), "hetz.demo.mystery"); + spec.tasks[0].derived = true; + spec.tasks[0].name = "mystery".to_string(); + let context = TaskCompileContext::current(tmp.path().to_path_buf()).unwrap(); + let mut specs = vec![spec]; + + let error = compile_generated_tasks(&mut specs, "hetz", &context).unwrap_err(); + + assert!( + format!("{error:#}").contains("unsupported derived task: mystery"), + "got: {error:#}" + ); +} diff --git a/tests/service.rs b/tests/service.rs index 945aabb3..814e0210 100644 --- a/tests/service.rs +++ b/tests/service.rs @@ -13,12 +13,19 @@ fn st2() -> std::process::Command { fn install_rejects_a_missing_catalog_before_touching_systemd() { // canonicalize() is the first thing install() does, so this errors out before any systemctl call. let out = st2() - .args(["service", "install", "/definitely/not/a/real/catalog/st2test"]) + .args([ + "service", + "install", + "/definitely/not/a/real/catalog/st2test", + ]) .output() .unwrap(); assert!(!out.status.success(), "should fail on a missing catalog"); let err = String::from_utf8_lossy(&out.stderr); - assert!(err.contains("does not exist"), "expected a missing-catalog error, got: {err}"); + assert!( + err.contains("does not exist"), + "expected a missing-catalog error, got: {err}" + ); } #[test] @@ -33,7 +40,10 @@ fn service_exposes_install_status_uninstall() { #[test] fn install_exposes_a_machine_local_pty_root() { - let out = st2().args(["service", "install", "--help"]).output().unwrap(); + let out = st2() + .args(["service", "install", "--help"]) + .output() + .unwrap(); assert!(out.status.success()); let help = String::from_utf8_lossy(&out.stdout); assert!(help.contains("--pty-root"), "{help}"); @@ -46,7 +56,10 @@ fn ping_is_an_alias_for_ding() { let help = st2().args(["ping", "--help"]).output().unwrap(); assert!(help.status.success()); let text = String::from_utf8_lossy(&help.stdout); - assert!(text.contains("inbox"), "ping --help should be the ding help; got: {text}"); + assert!( + text.contains("inbox"), + "ping --help should be the ding help; got: {text}" + ); // And a bare `st2 ping` dispatches INTO the ding handler (past clap). Catalog selection now has // an XDG default, so the next required runtime input is the acting identity. diff --git a/tests/stream_authoring_cli.rs b/tests/stream_authoring_cli.rs new file mode 100644 index 00000000..824e01dd --- /dev/null +++ b/tests/stream_authoring_cli.rs @@ -0,0 +1,166 @@ +use std::fs; +use std::path::Path; +use std::process::Command; + +fn write_agent(root: &Path) { + let directory = root.join("agents/hetz/worker"); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("agent.kdl"), + "agent \"worker\" {\n host \"hetz\"\n command \"agent\"\n}\n", + ) + .unwrap(); +} + +fn st2(root: &Path, args: &[&str]) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_st2")) + .arg("--catalog") + .arg(root) + .args(args) + .env_remove("ST_AGENT") + .output() + .unwrap() +} + +#[test] +fn stream_add_emit_and_rm_are_one_real_cli_workflow() { + let catalog = tempfile::tempdir().unwrap(); + write_agent(catalog.path()); + + let add = st2( + catalog.path(), + &[ + "stream", + "add", + "webhook", + "--agent", + "hetz.worker", + "--host", + "hetz", + "--json", + ], + ); + assert!( + add.status.success(), + "{}", + String::from_utf8_lossy(&add.stderr) + ); + let receipt: serde_json::Value = serde_json::from_slice(&add.stdout).unwrap(); + assert_eq!(receipt["result"], "changed"); + assert_eq!(receipt["name"], "webhook"); + assert!(receipt["launch"].is_null()); + + let emit = st2( + catalog.path(), + &[ + "event", + "emit", + "hetz.worker", + "--stream", + "webhook", + "--event-id", + "delivery-1", + "--message", + "payload", + "--host", + "hetz", + "--json", + ], + ); + assert!( + emit.status.success(), + "{}", + String::from_utf8_lossy(&emit.stderr) + ); + let receipt: serde_json::Value = serde_json::from_slice(&emit.stdout).unwrap(); + assert_eq!(receipt["status"], "created"); + + let remove = st2( + catalog.path(), + &[ + "stream", + "rm", + "webhook", + "--agent", + "hetz.worker", + "--host", + "hetz", + "--json", + ], + ); + assert!( + remove.status.success(), + "{}", + String::from_utf8_lossy(&remove.stderr) + ); + let receipt: serde_json::Value = serde_json::from_slice(&remove.stdout).unwrap(); + assert_eq!(receipt["result"], "changed"); + let declaration = + fs::read_to_string(catalog.path().join("agents/hetz/worker/agent.kdl")).unwrap(); + assert!(!declaration.contains("stream \"webhook\"")); +} + +#[test] +fn a_direct_adapter_launch_executes_the_exact_event_cli_contract() { + let catalog = tempfile::tempdir().unwrap(); + write_agent(catalog.path()); + let binary = env!("CARGO_BIN_EXE_st2"); + let add = st2( + catalog.path(), + &[ + "stream", + "add", + "adapter", + "--agent", + "hetz.worker", + "--host", + "hetz", + "--", + binary, + "--catalog", + catalog.path().to_str().unwrap(), + "event", + "emit", + "hetz.worker", + "--stream", + "adapter", + "--event-id", + "adapter-1", + "--message", + "from-adapter", + "--host", + "hetz", + "--json", + ], + ); + assert!( + add.status.success(), + "{}", + String::from_utf8_lossy(&add.stderr) + ); + + let spec = st2::discover(catalog.path()).specs.remove(0); + let adapter = spec + .tasks + .iter() + .find(|task| task.name == "stream-adapter") + .unwrap(); + assert!(adapter.derived); + let argv = adapter.argv.as_ref().unwrap(); + let output = Command::new(&argv[0]).args(&argv[1..]).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let receipt: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(receipt["status"], "created"); + assert_eq!( + st2::message::list_inbox(&st2::message::inbox_dir( + &catalog.path().join("agents/hetz/worker") + )) + .unwrap() + .len(), + 1 + ); +} diff --git a/tests/task_inventory_cli.rs b/tests/task_inventory_cli.rs index 5a512354..b7589a12 100644 --- a/tests/task_inventory_cli.rs +++ b/tests/task_inventory_cli.rs @@ -1,7 +1,7 @@ use std::collections::BTreeSet; use std::fs; -use std::os::unix::fs::symlink; use std::os::unix::fs::PermissionsExt; +use std::os::unix::fs::symlink; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::time::Duration; @@ -80,7 +80,9 @@ fn real_tasks(catalog: &Path, state: &Path) -> Output { fn seeded_supervisor_scope(catalog: &Path, host: Option<&str>, state: &Path) -> PathBuf { let mut command = Command::new(env!("CARGO_BIN_EXE_st2")); - command.args(["unpark", "h.worker", "--catalog"]).arg(catalog); + command + .args(["unpark", "h.worker", "--catalog"]) + .arg(catalog); if let Some(host) = host { command.args(["--host", host]); } @@ -89,7 +91,11 @@ fn seeded_supervisor_scope(catalog: &Path, host: Option<&str>, state: &Path) -> .env_remove("CATALOG") .output() .unwrap(); - assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); let supervisors = state.join("st2/supervisors"); let scope = fs::read_dir(&supervisors) @@ -108,7 +114,9 @@ fn execute_recovery(action: &serde_json::Value, bin: &Path, ambient_catalog: &Pa command.args(["-c", shell]); command } else { - let argv = action["argv"].as_array().expect("recovery action carries argv"); + let argv = action["argv"] + .as_array() + .expect("recovery action carries argv"); let mut command = Command::new(env!("CARGO_BIN_EXE_st2")); command.args(argv.iter().skip(1).map(|arg| arg.as_str().unwrap())); command @@ -119,7 +127,11 @@ fn execute_recovery(action: &serde_json::Value, bin: &Path, ambient_catalog: &Pa .env("XDG_STATE_HOME", state) .output() .unwrap(); - assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); } #[test] @@ -135,24 +147,35 @@ fn projected_recovery_targets_its_exact_catalog_and_host_despite_ambient_default assert_ne!(selected_scope, ambient_scope); for scope in [&selected_scope, &ambient_scope] { let projection = st2::park::ParkProjection::current(scope.join("parked")).unwrap(); - assert!(projection - .publish(&BTreeSet::from(["h.worker".to_string()]), "same-id collision") - .is_empty()); + assert!( + projection + .publish( + &BTreeSet::from(["h.worker".to_string()]), + "same-id collision" + ) + .is_empty() + ); } let output = tasks(&selected_catalog, &bin, &state); - assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); let inventory: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); let recovery = &inventory["tasks"][0]["parked"]["recovery"]; execute_recovery(recovery, &bin, &ambient_catalog, &state); let (selected, selected_errors) = st2::park::take_unpark_requests(&selected_scope.join("unpark")); - let (ambient, ambient_errors) = - st2::park::take_unpark_requests(&ambient_scope.join("unpark")); + let (ambient, ambient_errors) = st2::park::take_unpark_requests(&ambient_scope.join("unpark")); assert!(selected_errors.is_empty() && ambient_errors.is_empty()); assert_eq!(selected, ["h.worker"]); - assert!(ambient.is_empty(), "emitted recovery targeted the ambient supervisor scope"); + assert!( + ambient.is_empty(), + "emitted recovery targeted the ambient supervisor scope" + ); } #[test] @@ -249,16 +272,18 @@ agent "worker" { fn suspended_agent_projects_task_absence_and_agent_rationale_separately() { let (tmp, catalog, bin) = fixture("[]"); let declaration = catalog.join("agents/h/worker/agent.kdl"); - let authored = fs::read_to_string(&declaration) - .unwrap() - .replace( - " host \"h\"\n", - " host \"h\"\n desired-state \"suspended\" reason=\"Waiting for capacity\"\n", - ); + let authored = fs::read_to_string(&declaration).unwrap().replace( + " host \"h\"\n", + " host \"h\"\n desired-state \"suspended\" reason=\"Waiting for capacity\"\n", + ); fs::write(declaration, authored).unwrap(); let output = tasks(&catalog, &bin, &tmp.path().join("state")); - assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); let row = &value["tasks"][0]; assert_eq!(row["retired"], false); @@ -822,3 +847,39 @@ fn packaged_tasks_tracks_real_pty_generation_replacement() { first_generation ); } + +#[test] +fn a_derived_stream_task_is_reported_honestly_alongside_its_agent() { + let (tmp, catalog, bin) = fixture( + r#"[{"name":"h.streamed","status":"running","pid":91,"createdAt":"2026-08-20T10:00:00.000Z"}]"#, + ); + fs::create_dir_all(catalog.join("agents/h/streamed")).unwrap(); + fs::write( + catalog.join("agents/h/streamed/agent.kdl"), + r#" +agent "streamed" { + host "h" + command "true" + stream "gh-ci" { command "poll-gh-ci.sh" } +} +"#, + ) + .unwrap(); + let output = tasks(&catalog, &bin, &tmp.path().join("state")); + let inventory: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let rows = inventory["tasks"].as_array().unwrap(); + let stream = rows + .iter() + .find(|row| row["task"] == "stream-gh-ci") + .unwrap_or_else(|| { + panic!( + "no stream row in {}", + String::from_utf8_lossy(&output.stdout) + ) + }); + assert_eq!(stream["agent"], "h.streamed"); + assert_eq!(stream["runtimeId"], "h.streamed.stream-gh-ci"); + assert_eq!(stream["kind"], "exec"); + assert_eq!(stream["runtime"]["state"], "absent"); + assert!(stream.get("parked").is_none() || stream["parked"].is_null()); +} diff --git a/tests/transport_isolation.rs b/tests/transport_isolation.rs index c1d11868..a8bfb657 100644 --- a/tests/transport_isolation.rs +++ b/tests/transport_isolation.rs @@ -63,7 +63,8 @@ impl Fixture { fn new() -> Self { let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - let (catalog, xdg, pty_root) = (root.join("catalog"), root.join("xdg"), root.join("ptyroot")); + let (catalog, xdg, pty_root) = + (root.join("catalog"), root.join("xdg"), root.join("ptyroot")); for d in [&catalog, &xdg, &pty_root] { std::fs::create_dir_all(d).unwrap(); } @@ -80,7 +81,9 @@ impl Fixture { /// One service agent whose single task is `kind` = "exec" | "pty". fn write_agent(&self, identity: &str, kind: &str) { if kind == "pty" { - self.pty_sessions.borrow_mut().push(format!("{HOST}.{identity}.task")); + self.pty_sessions + .borrow_mut() + .push(format!("{HOST}.{identity}.task")); } let kdl = format!( "agent \"{identity}\" {{\n identity \"{identity}\"\n host \"{HOST}\"\n \ @@ -95,7 +98,12 @@ impl Fixture { /// daemon's pid (which owns the session). Both must survive the transport cascade. fn task_pidfile(&self, kind: &str, identity: &str) -> PathBuf { match kind { - "exec" => self.xdg.join("st2").join(HOST).join("exec").join(format!("{HOST}.{identity}.task.pid")), + "exec" => self + .xdg + .join("st2") + .join(HOST) + .join("exec") + .join(format!("{HOST}.{identity}.task.pid")), "pty" => self.pty_root.join(format!("{HOST}.{identity}.task.pid")), other => panic!("unknown task kind {other}"), } @@ -145,7 +153,11 @@ impl Fixture { impl Drop for Fixture { fn drop(&mut self) { let quiet = |args: &[&str]| { - let _ = Command::new("systemctl").args(args).stdout(Stdio::null()).stderr(Stdio::null()).status(); + let _ = Command::new("systemctl") + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); }; // Stop only the transport scopes WE registered — never a broad `st2-` sweep, which would // stop a concurrently-running sibling test's task scope (same host). The task scopes this @@ -162,7 +174,12 @@ impl Drop for Fixture { && let Some(pid) = read_pid(&e.path()) { for t in [format!("-{pid}"), pid.to_string()] { - let _ = Command::new("kill").arg("-KILL").arg(t).stdout(Stdio::null()).stderr(Stdio::null()).status(); + let _ = Command::new("kill") + .arg("-KILL") + .arg(t) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); } } } @@ -207,13 +224,23 @@ fn read_alive(pidfile: &Path) -> bool { /// A pid's cgroup line (`0::/…`), or empty if unreadable. fn cgroup_of(pid: i32) -> String { - std::fs::read_to_string(format!("/proc/{pid}/cgroup")).unwrap_or_default().trim().to_string() + std::fs::read_to_string(format!("/proc/{pid}/cgroup")) + .unwrap_or_default() + .trim() + .to_string() } /// The live pids inside a scope's cgroup (empty once the scope is drained/gone). fn scope_pids(unit: &str) -> Vec { let out = match Command::new("systemctl") - .args(["--user", "show", &format!("{unit}.scope"), "-p", "ControlGroup", "--value"]) + .args([ + "--user", + "show", + &format!("{unit}.scope"), + "-p", + "ControlGroup", + "--value", + ]) .output() { Ok(o) => o, @@ -248,7 +275,13 @@ fn poll_until(timeout: Duration, mut cond: impl FnMut() -> bool) -> bool { /// is untouched — which is the whole point. fn cascade_kill_scope(unit: &str) { let ok = Command::new("systemctl") - .args(["--user", "kill", "--kill-whom=all", "--signal=SIGKILL", &format!("{unit}.scope")]) + .args([ + "--user", + "kill", + "--kill-whom=all", + "--signal=SIGKILL", + &format!("{unit}.scope"), + ]) .status() .map(|s| s.success()) .unwrap_or(false); @@ -256,7 +289,13 @@ fn cascade_kill_scope(unit: &str) { } fn have(bin: &str, args: &[&str]) -> bool { - Command::new(bin).args(args).stdout(Stdio::null()).stderr(Stdio::null()).status().map(|s| s.success()).unwrap_or(false) + Command::new(bin) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) } /// This gate needs `systemd-run`/`systemctl --user` (the isolation mechanism) AND `pty` on PATH (the @@ -264,7 +303,8 @@ fn have(bin: &str, args: &[&str]) -> bool { /// UNPROVEN — a HARD FAILURE, never a silent green skip, unless a dev explicitly opts out with /// `ST2_ALLOW_ISOLATION_SKIP` on a box without them. CI/gating MUST provide both. fn isolation_gate(test: &str) -> bool { - let systemd = have("systemd-run", &["--user", "--version"]) && std::env::var_os("XDG_RUNTIME_DIR").is_some(); + let systemd = have("systemd-run", &["--user", "--version"]) + && std::env::var_os("XDG_RUNTIME_DIR").is_some(); let pty = have("pty", &["--help"]); if systemd && pty { return true; @@ -301,7 +341,8 @@ fn task_survives_transport_cgroup_cascade(kind: &str) { let _tr = Handle(fx.spawn_transport(&transport)); let task_pidfile = fx.task_pidfile(kind, &identity); assert!( - poll_until(SPAWN_TIMEOUT, || read_alive(&task_pidfile) && !scope_pids(&transport).is_empty()), + poll_until(SPAWN_TIMEOUT, || read_alive(&task_pidfile) + && !scope_pids(&transport).is_empty()), "st2 up --once never brought up a live task (task pidfile {})", task_pidfile.display() ); @@ -324,7 +365,10 @@ fn task_survives_transport_cgroup_cascade(kind: &str) { cgroup_of(task_pid) ); // The control (a naive `sleep`) is live in the transport cgroup right now. - assert!(!scope_pids(&transport).is_empty(), "transport scope unexpectedly empty before the cascade"); + assert!( + !scope_pids(&transport).is_empty(), + "transport scope unexpectedly empty before the cascade" + ); // 3) Fire the cascade: SIGKILL the transport scope's cgroup — the supervisor-restart failure // restart (its SIGTERM would be trapped by a supervisor; the SIGKILL escalation is what kills). diff --git a/tests/transport_isolation_macos.rs b/tests/transport_isolation_macos.rs index 2a1855a6..96fbb2f1 100644 --- a/tests/transport_isolation_macos.rs +++ b/tests/transport_isolation_macos.rs @@ -42,11 +42,17 @@ impl Fixture { fn new() -> Self { let tmp = tempfile::tempdir().unwrap(); let root = tmp.path(); - let (catalog, xdg, pty_root) = (root.join("catalog"), root.join("xdg"), root.join("ptyroot")); + let (catalog, xdg, pty_root) = + (root.join("catalog"), root.join("xdg"), root.join("ptyroot")); for d in [&catalog, &xdg, &pty_root] { std::fs::create_dir_all(d).unwrap(); } - Fixture { catalog, xdg, pty_root, _tmp: tmp } + Fixture { + catalog, + xdg, + pty_root, + _tmp: tmp, + } } fn write_exec_agent(&self, identity: &str) { @@ -60,7 +66,11 @@ impl Fixture { } fn task_pidfile(&self, identity: &str) -> PathBuf { - self.xdg.join("st2").join(HOST).join("exec").join(format!("{HOST}.{identity}.task.pid")) + self.xdg + .join("st2") + .join(HOST) + .join("exec") + .join(format!("{HOST}.{identity}.task.pid")) } fn supervisor_pidfile(&self) -> PathBuf { @@ -107,7 +117,12 @@ impl Drop for Fixture { && let Some(pid) = read_pid(&e.path()) { for t in [format!("-{pid}"), pid.to_string()] { - let _ = Command::new("kill").arg("-KILL").arg(t).stdout(Stdio::null()).stderr(Stdio::null()).status(); + let _ = Command::new("kill") + .arg("-KILL") + .arg(t) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); } } } @@ -130,7 +145,10 @@ fn read_alive(pidfile: &Path) -> bool { /// A pid's parent pid via `ps` (macOS has no `/proc`). `Some(1)` == reparented to launchd/init. fn ppid_of(pid: i32) -> Option { - let out = Command::new("ps").args(["-o", "ppid=", "-p", &pid.to_string()]).output().ok()?; + let out = Command::new("ps") + .args(["-o", "ppid=", "-p", &pid.to_string()]) + .output() + .ok()?; String::from_utf8_lossy(&out.stdout).trim().parse().ok() } @@ -148,7 +166,13 @@ fn poll_until(timeout: Duration, mut cond: impl FnMut() -> bool) -> bool { /// This gate needs `pty` on PATH (the real `st2 up` lists pty every reconcile pass). Missing it means /// the gate cannot run and is UNPROVEN — a HARD FAILURE, never a silent skip, unless a dev opts out. fn isolation_gate(test: &str) -> bool { - let pty = Command::new("pty").arg("--help").stdout(Stdio::null()).stderr(Stdio::null()).status().map(|s| s.success()).unwrap_or(false); + let pty = Command::new("pty") + .arg("--help") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false); if pty { return true; } @@ -163,7 +187,12 @@ fn isolation_gate(test: &str) -> bool { /// Kill a process group (leader `pgid`) with SIGKILL: `kill -KILL -`. fn kill_group(pgid: i32) { - let ok = Command::new("kill").arg("-KILL").arg(format!("-{pgid}")).status().map(|s| s.success()).unwrap_or(false); + let ok = Command::new("kill") + .arg("-KILL") + .arg(format!("-{pgid}")) + .status() + .map(|s| s.success()) + .unwrap_or(false); assert!(ok, "failed to kill process group {pgid}"); } @@ -182,7 +211,8 @@ fn task_survives_spawner_group_kill() { let mut sup = Handle(fx.spawn_supervisor_in_own_group()); let task_pidfile = fx.task_pidfile("survivor"); assert!( - poll_until(SPAWN_TIMEOUT, || read_alive(&task_pidfile) && read_alive(&fx.supervisor_pidfile())), + poll_until(SPAWN_TIMEOUT, || read_alive(&task_pidfile) + && read_alive(&fx.supervisor_pidfile())), "supervisor never brought up a live task (task pidfile {})", task_pidfile.display() ); @@ -209,7 +239,10 @@ fn task_survives_spawner_group_kill() { ); // 5) THE PROPERTY: the task outlived the group kill and reparented to launchd/init (ppid == 1). - assert!(process_alive(task_pid), "task pid {task_pid} died with the spawner's group — NOT detached"); + assert!( + process_alive(task_pid), + "task pid {task_pid} died with the spawner's group — NOT detached" + ); assert!( poll_until(DEATH_TIMEOUT, || ppid_of(task_pid) == Some(1)), "task pid {task_pid} survived but did not reparent to launchd/init (ppid {:?})", diff --git a/tests/validate.rs b/tests/validate.rs index 86f1c6d4..1b1586e7 100644 --- a/tests/validate.rs +++ b/tests/validate.rs @@ -431,7 +431,12 @@ fn fleet_validation_compiles_remote_driver_launches() { for report in [validate(c.path()), validate_for_host(c.path(), "droppy")] { assert_eq!(report.errors(), 0, "unexpected issues: {:?}", report.issues); - assert_eq!(report.warnings(), 0, "unexpected issues: {:?}", report.issues); + assert_eq!( + report.warnings(), + 0, + "unexpected issues: {:?}", + report.issues + ); } } From b00e2c2c8f2a571c2fe873c86f46248027e99462 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 11:14:10 +0200 Subject: [PATCH 02/26] fix(stream): fail closed at event boundaries --- crates/agent-spec/src/kdl_format.rs | 40 ++++++++++++++++---- crates/agent-spec/tests/discovery.rs | 55 ++++++++++++++++++++++++++++ crates/st2-wire/src/message.rs | 32 ++++++++++++++++ src/event.rs | 24 ++++++++++-- src/main.rs | 18 +++++++++ tests/event_e2e.rs | 34 +++++++++++++++++ tests/message_cli.rs | 41 +++++++++++++++++++++ 7 files changed, 233 insertions(+), 11 deletions(-) diff --git a/crates/agent-spec/src/kdl_format.rs b/crates/agent-spec/src/kdl_format.rs index 472eed6e..22e6e4e3 100644 --- a/crates/agent-spec/src/kdl_format.rs +++ b/crates/agent-spec/src/kdl_format.rs @@ -178,13 +178,22 @@ fn agent_node_to_raw(node: &DeclaredNode) -> anyhow::Result { } } "stream" => { - if let Some(name) = arg_string(child) { - let stream = stream_node_to_raw(child, &name)?; - anyhow::ensure!( - raw.stream.insert(name.clone(), stream).is_none(), - "agent declares `stream \"{name}\"` more than once" - ); - } + anyhow::ensure!( + child.type_name.is_none() + && child.entries.len() == 1 + && child.entries[0].name.is_none(), + "agent `stream` must contain exactly one positional name string and no properties" + ); + let name = arg_string(child).ok_or_else(|| { + anyhow::anyhow!( + "agent `stream` must contain exactly one positional name string and no properties" + ) + })?; + let stream = stream_node_to_raw(child, &name)?; + anyhow::ensure!( + raw.stream.insert(name.clone(), stream).is_none(), + "agent declares `stream \"{name}\"` more than once" + ); } // meta, harness, model, persona, permissions, transport, strategy, … — ignored. _ => {} @@ -479,8 +488,17 @@ fn stream_node_to_raw(node: &DeclaredNode, name: &str) -> anyhow::Result { @@ -488,6 +506,12 @@ fn stream_node_to_raw(node: &DeclaredNode, name: &str) -> anyhow::Result anyhow::bail!( diff --git a/crates/agent-spec/tests/discovery.rs b/crates/agent-spec/tests/discovery.rs index ac9c3774..d7dd2e0e 100644 --- a/crates/agent-spec/tests/discovery.rs +++ b/crates/agent-spec/tests/discovery.rs @@ -1906,3 +1906,58 @@ fn stream_names_launches_and_task_collisions_fail_closed() { ); } } + +#[test] +fn malformed_stream_kdl_shapes_fail_closed() { + for (identity, declaration, expected) in [ + ( + "extra-name-argument", + "stream \"ci\" \"extra\" {}", + "exactly one positional name string", + ), + ( + "stream-property", + "stream \"ci\" bogus=#true {}", + "no properties", + ), + ( + "typed-stream", + "(typed)stream \"ci\" {}", + "exactly one positional name string", + ), + ( + "extra-command-argument", + "stream \"ci\" { command \"watch\" \"typo\" }", + "exactly one positional string", + ), + ( + "nested-command", + "stream \"ci\" { command \"watch\" { typo \"value\" } }", + "exactly one positional string", + ), + ( + "argv-property", + "stream \"ci\" { argv \"watch\" typo=#true }", + "only positional string arguments", + ), + ( + "nested-argv", + "stream \"ci\" { argv \"watch\" { typo \"value\" } }", + "only positional string arguments", + ), + ] { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + &format!("agents/h/{identity}/agent.kdl"), + &format!("agent \"{identity}\" {{ host \"h\"; command \"agent\"; {declaration} }}"), + ); + let found = discover(tmp.path()); + assert_eq!(found.errors.len(), 1, "{identity}: {:?}", found.errors); + assert!( + found.errors[0].message.contains(expected), + "{identity}: {:?}", + found.errors + ); + } +} diff --git a/crates/st2-wire/src/message.rs b/crates/st2-wire/src/message.rs index c09a2942..285d01c1 100644 --- a/crates/st2-wire/src/message.rs +++ b/crates/st2-wire/src/message.rs @@ -40,6 +40,15 @@ pub struct MessageRow { skip_serializing_if = "Option::is_none" )] pub idempotency_key: Option, + /// The declared stream that produced this inbox event, when this is an event record. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + /// The producer-owned stable identity of this event. + #[serde(rename = "eventId", default, skip_serializing_if = "Option::is_none")] + pub event_id: Option, + /// The optional stream-local supersession key. + #[serde(rename = "eventKey", default, skip_serializing_if = "Option::is_none")] + pub event_key: Option, /// The markdown body. /// /// Absent — the key omitted entirely, not `null` — from a `ls --json` row unless @@ -106,6 +115,9 @@ mod tests { tags: Vec::new(), priority: None, idempotency_key: None, + stream: None, + event_id: None, + event_key: None, body: None, } } @@ -170,6 +182,26 @@ mod tests { assert!(json.contains(r#""body":"""#), "{json}"); } + #[test] + fn event_identity_round_trips_without_changing_ordinary_message_rows() { + let ordinary = serde_json::to_value(row()).unwrap(); + assert!(ordinary.get("stream").is_none()); + assert!(ordinary.get("eventId").is_none()); + assert!(ordinary.get("eventKey").is_none()); + + let event = MessageRow { + stream: Some("gh-ci".to_string()), + event_id: Some("run-812".to_string()), + event_key: Some("main".to_string()), + ..row() + }; + let json = serde_json::to_value(&event).unwrap(); + assert_eq!(json["stream"], "gh-ci"); + assert_eq!(json["eventId"], "run-812"); + assert_eq!(json["eventKey"], "main"); + assert_eq!(serde_json::from_value::(json).unwrap(), event); + } + /// An ABSENT optional key and an explicit `null` both mean "not carried", and both must parse. /// /// This is the distinction the defect turned on: `#[serde(default)]` covers only the absent diff --git a/src/event.rs b/src/event.rs index b1e3244d..94eaa12b 100644 --- a/src/event.rs +++ b/src/event.rs @@ -99,14 +99,32 @@ fn resolve_stream( .collect::>() .join("; ") ); - let spec = discovered + let mut matches = discovered .specs .into_iter() - .find(|spec| { + .filter(|spec| { spec.bus_id(this_host) == recipient || (spec.resolved_host(this_host) == this_host && spec.identity == recipient) }) - .with_context(|| format!("no agent '{recipient}' found in catalog {}", root.display()))?; + .collect::>(); + anyhow::ensure!( + !matches.is_empty(), + "no agent '{recipient}' found in catalog {}", + root.display() + ); + anyhow::ensure!( + matches.len() == 1, + "agent recipient '{recipient}' is ambiguous; matched {} declarations: {}", + matches.len(), + matches + .iter() + .map(|spec| spec.path.display().to_string()) + .collect::>() + .join(", ") + ); + let spec = matches + .pop() + .context("exactly one matching agent expected")?; anyhow::ensure!( spec.streams.iter().any(|declared| declared.name == stream), "agent '{}' does not declare stream '{stream}'", diff --git a/src/main.rs b/src/main.rs index 8b754de5..c5c49a10 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2781,6 +2781,12 @@ struct LsItemJson<'a> { #[serde(rename = "idempotencyKey", skip_serializing_if = "Option::is_none")] idempotency_key: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] + stream: Option<&'a str>, + #[serde(rename = "eventId", skip_serializing_if = "Option::is_none")] + event_id: Option<&'a str>, + #[serde(rename = "eventKey", skip_serializing_if = "Option::is_none")] + event_key: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] body: Option<&'a str>, } @@ -2795,6 +2801,9 @@ impl<'a> From<&'a st2::message::Message> for LsItemJson<'a> { tags: &m.tags, priority: m.priority.as_deref(), idempotency_key: m.idempotency_key.as_deref(), + stream: m.stream.as_deref(), + event_id: m.event_id.as_deref(), + event_key: m.event_key.as_deref(), body: None, } } @@ -2823,6 +2832,12 @@ struct MessageJson<'a> { priority: Option<&'a str>, #[serde(rename = "idempotencyKey", skip_serializing_if = "Option::is_none")] idempotency_key: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + stream: Option<&'a str>, + #[serde(rename = "eventId", skip_serializing_if = "Option::is_none")] + event_id: Option<&'a str>, + #[serde(rename = "eventKey", skip_serializing_if = "Option::is_none")] + event_key: Option<&'a str>, body: &'a str, } @@ -2837,6 +2852,9 @@ impl<'a> From<&'a st2::message::Message> for MessageJson<'a> { tags: &m.tags, priority: m.priority.as_deref(), idempotency_key: m.idempotency_key.as_deref(), + stream: m.stream.as_deref(), + event_id: m.event_id.as_deref(), + event_key: m.event_key.as_deref(), body: &m.body, } } diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index 92e85948..d0fd5026 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -161,6 +161,40 @@ fn conflicting_reuse_and_undeclared_or_suspended_ingress_fail_closed() { ); } +#[test] +fn ambiguous_recipient_matching_a_bus_id_and_local_identity_fails_closed() { + let catalog = tempfile::tempdir().unwrap(); + let canonical = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let ambiguous = catalog.path().join("agents/hetz/ambiguous"); + fs::create_dir_all(&ambiguous).unwrap(); + fs::write( + ambiguous.join("agent.kdl"), + "agent \"hetz.worker\" {\n host \"hetz\"\n desired-state \"running\"\n command \"agent\"\n stream \"gh-ci\" {}\n}\n", + ) + .unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "ambiguous", + None, + None, + "payload", + false, + ) + .unwrap_err() + .to_string(); + + assert!( + error.contains("recipient 'hetz.worker' is ambiguous"), + "{error}" + ); + assert!(!message::inbox_dir(&canonical).exists()); + assert!(!message::inbox_dir(&ambiguous).exists()); +} + #[test] fn supersede_collapses_only_the_matching_key_and_preserves_archive_receipts() { let catalog = tempfile::tempdir().unwrap(); diff --git a/tests/message_cli.rs b/tests/message_cli.rs index 40dc3012..7838d91b 100644 --- a/tests/message_cli.rs +++ b/tests/message_cli.rs @@ -79,6 +79,47 @@ fn sent(root: &Path, identity: &str, extra: &[&str]) -> std::process::Output { .unwrap() } +#[test] +fn event_metadata_is_exposed_by_list_and_read_json() { + let tmp = tempfile::tempdir().unwrap(); + write_agent(tmp.path(), "bob"); + let inbox = tmp.path().join("h/bob/resources/inbox"); + fs::create_dir_all(&inbox).unwrap(); + let filename = "1785000000000-abcdef.md"; + fs::write( + inbox.join(filename), + "---\nfrom: h.bob/gh-ci\nsubject: CI result\nstream: gh-ci\nevent-id: run-812\nkey: main\n---\npayload\n", + ) + .unwrap(); + + let listed = list(tmp.path(), &["--json"]); + assert!( + listed.status.success(), + "{}", + String::from_utf8_lossy(&listed.stderr) + ); + let listed: serde_json::Value = serde_json::from_slice(&listed.stdout).unwrap(); + assert_eq!(listed[0]["stream"], "gh-ci"); + assert_eq!(listed[0]["eventId"], "run-812"); + assert_eq!(listed[0]["eventKey"], "main"); + + let read = Command::new(env!("CARGO_BIN_EXE_st2")) + .args(["message", "read", "bob", filename, "--root"]) + .arg(tmp.path()) + .args(["--host", "h", "--json"]) + .output() + .unwrap(); + assert!( + read.status.success(), + "{}", + String::from_utf8_lossy(&read.stderr) + ); + let read: serde_json::Value = serde_json::from_slice(&read.stdout).unwrap(); + assert_eq!(read["stream"], "gh-ci"); + assert_eq!(read["eventId"], "run-812"); + assert_eq!(read["eventKey"], "main"); +} + #[test] fn uninitialized_sent_history_is_explicitly_unavailable_not_a_complete_empty_list() { let tmp = tempfile::tempdir().unwrap(); From 27692bc312bbea21978e816ed21d0e6bc2a1c8d9 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 11:33:20 +0200 Subject: [PATCH 03/26] fix(stream): retain no-follow ingress capabilities --- crates/agent-spec/src/spec.rs | 1 + crates/agent-spec/tests/discovery.rs | 92 +++++++++++ src/event.rs | 231 ++++++++++++++------------- src/message.rs | 31 ++++ tests/event_e2e.rs | 90 +++++++++++ 5 files changed, 333 insertions(+), 112 deletions(-) diff --git a/crates/agent-spec/src/spec.rs b/crates/agent-spec/src/spec.rs index 9deb8555..6261027c 100644 --- a/crates/agent-spec/src/spec.rs +++ b/crates/agent-spec/src/spec.rs @@ -515,6 +515,7 @@ pub(crate) struct RawSpec { /// A declared event source. Exactly one of `command` / `argv` launches the source process. #[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] pub(crate) struct RawStream { pub command: Option, pub argv: Option>, diff --git a/crates/agent-spec/tests/discovery.rs b/crates/agent-spec/tests/discovery.rs index d7dd2e0e..96f71f9e 100644 --- a/crates/agent-spec/tests/discovery.rs +++ b/crates/agent-spec/tests/discovery.rs @@ -1871,6 +1871,98 @@ fn streams_are_typed_and_only_launched_streams_lower_to_derived_exec_tasks() { assert_eq!(shell.command.as_deref(), Some("watch-ci")); } +#[test] +fn streams_have_toml_and_json_parity_and_reject_unknown_fields() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/h/toml/agent.toml", + r#"identity = "toml" +host = "h" +command = "agent" + +[stream.external] + +[stream.direct] +argv = ["watch", "--json"] +"#, + ); + write( + tmp.path(), + "agents/h/json/agent.json", + r#"{"identity":"json","host":"h","command":"agent","stream":{"external":{},"shell":{"command":"watch-ci"}}}"#, + ); + + for (identity, extension, stream) in [ + ("toml-every", "toml", "every = \"1m\""), + ("toml-misspelled", "toml", "commmand = \"watch-ci\""), + ("json-every", "json", r#""every":"1m""#), + ("json-misspelled", "json", r#""commmand":"watch-ci""#), + ] { + let contents = if extension == "toml" { + format!( + "identity = \"{identity}\"\nhost = \"h\"\ncommand = \"agent\"\n[stream.ci]\n{stream}\n" + ) + } else { + format!( + r#"{{"identity":"{identity}","host":"h","command":"agent","stream":{{"ci":{{{stream}}}}}}}"# + ) + }; + write( + tmp.path(), + &format!("agents/h/{identity}/agent.{extension}"), + &contents, + ); + } + + let found = discover(tmp.path()); + assert_eq!(found.specs.len(), 2, "specs: {:?}", found.specs); + assert_eq!(found.errors.len(), 4, "errors: {:?}", found.errors); + + let toml = find(&found.specs, "toml"); + assert_eq!(toml.streams.len(), 2); + assert!(toml.tasks.iter().all(|task| task.name != "stream-external")); + assert_eq!( + argv( + toml.tasks + .iter() + .find(|task| task.name == "stream-direct") + .unwrap() + ), + ["watch", "--json"] + ); + + let json = find(&found.specs, "json"); + assert_eq!(json.streams.len(), 2); + assert!(json.tasks.iter().all(|task| task.name != "stream-external")); + assert_eq!( + json.tasks + .iter() + .find(|task| task.name == "stream-shell") + .unwrap() + .command + .as_deref(), + Some("watch-ci") + ); + + for identity in [ + "toml-every", + "toml-misspelled", + "json-every", + "json-misspelled", + ] { + let error = found + .errors + .iter() + .find(|error| error.path.to_string_lossy().contains(identity)) + .unwrap_or_else(|| panic!("missing error for {identity}: {:?}", found.errors)); + assert!( + error.message.contains("unknown field"), + "{identity}: {error:?}" + ); + } +} + #[test] fn stream_names_launches_and_task_collisions_fail_closed() { for (identity, body, expected) in [ diff --git a/src/event.rs b/src/event.rs index 94eaa12b..c064292a 100644 --- a/src/event.rs +++ b/src/event.rs @@ -4,7 +4,7 @@ //! own bounded, agent-local receipt ring rather than writing the agent's immutable Sent ledger. use std::fs::{self, File, OpenOptions}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; use anyhow::Context as _; @@ -79,7 +79,6 @@ pub enum EventReceiptStatus { struct ResolvedStream { recipient: String, - agent_dir: PathBuf, } fn resolve_stream( @@ -88,7 +87,7 @@ fn resolve_stream( recipient: &str, stream: &str, ) -> anyhow::Result { - let discovered = crate::discover(root); + let discovered = crate::discover_strict(root); anyhow::ensure!( discovered.errors.is_empty(), "catalog has errors; refusing event publication: {}", @@ -136,14 +135,8 @@ fn resolve_stream( spec.bus_id(this_host), spec.desired_state.as_str() ); - let agent_dir = spec - .path - .parent() - .context("agent declaration has no parent")? - .to_path_buf(); Ok(ResolvedStream { recipient: spec.bus_id(this_host), - agent_dir, }) } @@ -197,116 +190,130 @@ pub fn emit( // suspension edit owns this lock, no later emit can publish from a stale running observation. let _catalog_lock = crate::catalog_lock::CatalogLock::exclusive(root)?; let resolved = resolve_stream(root, this_host, recipient, stream)?; - let from = format!("{}/{}", resolved.recipient, stream); + let canonical_recipient = resolved.recipient; + let from = format!("{canonical_recipient}/{stream}"); let rendered = render_event(&from, subject, stream, event_id, key, body); - let state_dir = resolved.agent_dir.join("resources/streams").join(stream); - fs::create_dir_all(&state_dir)?; - let _lock = StreamLock::exclusive(&state_dir)?; - let record_path = state_dir.join("state.json"); - let mut record = read_record(&record_path)? - .unwrap_or_else(|| StreamRecord::fresh(stream, &resolved.recipient)); - anyhow::ensure!( - record.version == EVENT_VERSION - && record.stream == stream - && record.recipient == resolved.recipient, - "stream state for '{}#{stream}' is not readable at version {EVENT_VERSION}", - resolved.recipient - ); - - let rendered_sha256 = hex_digest(rendered.as_bytes()); - if let Some(entry) = record - .recent - .iter() - .find(|entry| entry.event_id == event_id) - { - anyhow::ensure!( - entry.rendered_sha256 == rendered_sha256, - "event identity `{stream}#{event_id}` reused with different content" - ); - return Ok(EventReceipt { - recipient: resolved.recipient, - stream: stream.to_owned(), - event_id: event_id.to_owned(), - filename: entry.filename.clone(), - status: EventReceiptStatus::Deduplicated, - superseded: None, - }); - } - - let (filename, resumed) = match record.pending.as_ref() { - Some(pending) if pending.event_id == event_id => { + message::with_resolved_state_dir( + root, + &canonical_recipient, + this_host, + &["resources", "streams", stream], + true, + |state_dir| { + let _lock = StreamLock::exclusive(state_dir)?; + let record_path = state_dir.join("state.json"); + let mut record = read_record(&record_path)? + .unwrap_or_else(|| StreamRecord::fresh(stream, &canonical_recipient)); anyhow::ensure!( - pending.rendered_sha256 == rendered_sha256 - && pending.key.as_deref() == key - && pending.supersede == supersede, - "event identity `{stream}#{event_id}` reused with different content" + record.version == EVENT_VERSION + && record.stream == stream + && record.recipient == canonical_recipient, + "stream state for '{}#{stream}' is not readable at version {EVENT_VERSION}", + canonical_recipient ); - (pending.filename.clone(), true) - } - Some(pending) => anyhow::bail!( - "stream '{stream}' has an interrupted event '{}'; replay it before publishing another", - pending.event_id - ), - None => (message::new_filename(), false), - }; - if !resumed { - record.pending = Some(StreamPending { - event_id: event_id.to_owned(), - filename: filename.clone(), - key: key.map(str::to_owned), - rendered_sha256: rendered_sha256.clone(), - supersede, - }); - write_record(&record_path, &record)?; - } - let inbox = message::inbox_dir(&resolved.agent_dir); - let archive = message::archive_dir(&resolved.agent_dir); - let predecessor = if supersede { - record - .recent - .iter() - .find(|entry| entry.key.as_deref() == key && entry.filename != filename) - .map(|entry| entry.filename.clone()) - } else { - None - }; - if let Some(predecessor) = &predecessor { - message::archive_msg(&inbox, &archive, predecessor)?; - } - // An archive filename is the bus's authoritative durable receipt. A crash after materializing - // and external archive, but before advancing this state, must not restore the inbox replica. - let created = if archive.join(&filename).is_file() { - false - } else { - message::materialize_message_once(&inbox, &filename, &rendered)? - }; + let rendered_sha256 = hex_digest(rendered.as_bytes()); + if let Some(entry) = record + .recent + .iter() + .find(|entry| entry.event_id == event_id) + { + anyhow::ensure!( + entry.rendered_sha256 == rendered_sha256, + "event identity `{stream}#{event_id}` reused with different content" + ); + return Ok(EventReceipt { + recipient: canonical_recipient.clone(), + stream: stream.to_owned(), + event_id: event_id.to_owned(), + filename: entry.filename.clone(), + status: EventReceiptStatus::Deduplicated, + superseded: None, + }); + } - record.pending = None; - record.recent.insert( - 0, - StreamEntry { - event_id: event_id.to_owned(), - filename: filename.clone(), - key: key.map(str::to_owned), - rendered_sha256, - }, - ); - record.recent.truncate(RING_CAPACITY); - write_record(&record_path, &record)?; + let (filename, resumed) = match record.pending.as_ref() { + Some(pending) if pending.event_id == event_id => { + anyhow::ensure!( + pending.rendered_sha256 == rendered_sha256 + && pending.key.as_deref() == key + && pending.supersede == supersede, + "event identity `{stream}#{event_id}` reused with different content" + ); + (pending.filename.clone(), true) + } + Some(pending) => anyhow::bail!( + "stream '{stream}' has an interrupted event '{}'; replay it before publishing another", + pending.event_id + ), + None => (message::new_filename(), false), + }; + if !resumed { + record.pending = Some(StreamPending { + event_id: event_id.to_owned(), + filename: filename.clone(), + key: key.map(str::to_owned), + rendered_sha256: rendered_sha256.clone(), + supersede, + }); + write_record(&record_path, &record)?; + } - Ok(EventReceipt { - recipient: resolved.recipient, - stream: stream.to_owned(), - event_id: event_id.to_owned(), - filename, - status: if created { - EventReceiptStatus::Created - } else { - EventReceiptStatus::Deduplicated + let predecessor = if supersede { + record + .recent + .iter() + .find(|entry| entry.key.as_deref() == key && entry.filename != filename) + .map(|entry| entry.filename.clone()) + } else { + None + }; + let created = message::with_resolved_message_boxes( + root, + &canonical_recipient, + this_host, + |inbox, archive| { + if let Some(predecessor) = &predecessor { + message::archive_msg(inbox, archive, predecessor)?; + } + // An archive filename is the bus's authoritative durable receipt. A crash after + // materializing an external archive, but before advancing this state, must not restore + // the inbox replica. + if archive.join(&filename).is_file() { + Ok(false) + } else { + message::materialize_message_once(inbox, &filename, &rendered) + } + }, + )?; + + record.pending = None; + record.recent.insert( + 0, + StreamEntry { + event_id: event_id.to_owned(), + filename: filename.clone(), + key: key.map(str::to_owned), + rendered_sha256, + }, + ); + record.recent.truncate(RING_CAPACITY); + write_record(&record_path, &record)?; + + Ok(EventReceipt { + recipient: canonical_recipient.clone(), + stream: stream.to_owned(), + event_id: event_id.to_owned(), + filename, + status: if created { + EventReceiptStatus::Created + } else { + EventReceiptStatus::Deduplicated + }, + superseded: predecessor, + }) }, - superseded: predecessor, - }) + ) } fn validate_component(label: &str, value: &str) -> anyhow::Result<()> { diff --git a/src/message.rs b/src/message.rs index 7fe8e051..9b90adc8 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1021,6 +1021,37 @@ pub fn with_resolved_state_dir( } } +/// Run an operation against retained, no-follow capabilities for an agent's inbox and archive. +/// +/// Both directories are opened relative to the resolved agent capability, so replacing the +/// declaration directory or either message-box ancestor with a symlink cannot redirect the +/// operation outside the catalog after recipient resolution. +pub(crate) fn with_resolved_message_boxes( + catalog_root: &Path, + identity: &str, + this_host: &str, + operation: impl FnOnce(&Path, &Path) -> anyhow::Result, +) -> anyhow::Result { + let agent = resolve_agent_handle(catalog_root, identity, this_host)?.with_context(|| { + format!( + "no agent '{identity}' found in catalog {}", + catalog_root.display() + ) + })?; + let capability = agent + .capability + .as_ref() + .context("resolved agent has no retained directory capability")?; + let inbox = open_message_box(capability, &["resources", "inbox"], true)? + .context("created inbox capability is missing")?; + let archive = open_message_box(capability, &["resources", "archive"], true)? + .context("created archive capability is missing")?; + operation( + &crate::catalog_transaction::retained_dir_path(&inbox)?, + &crate::catalog_transaction::retained_dir_path(&archive)?, + ) +} + fn resolve_agent_handle( catalog_root: &Path, recipient: &str, diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index d0fd5026..c4be2f56 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -195,6 +195,96 @@ fn ambiguous_recipient_matching_a_bus_id_and_local_identity_fails_closed() { assert!(!message::inbox_dir(&ambiguous).exists()); } +#[cfg(unix)] +#[test] +fn unobservable_declaration_entry_blocks_event_recipient_resolution() { + use std::os::unix::fs::symlink; + + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + symlink( + catalog.path().join("missing-agent.kdl"), + catalog.path().join("concealed-agent.kdl"), + ) + .unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "strict-discovery", + None, + None, + "payload", + false, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("catalog has errors"), "{error}"); + assert!(error.contains("unobservable declaration entry"), "{error}"); + assert!(!message::inbox_dir(&agent).exists()); +} + +#[cfg(unix)] +#[test] +fn symlinked_stream_state_ancestor_cannot_escape_the_agent_capability() { + use std::os::unix::fs::symlink; + + let catalog = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + fs::create_dir_all(agent.join("resources")).unwrap(); + symlink(outside.path(), agent.join("resources/streams")).unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "escape-state", + None, + None, + "payload", + false, + ) + .unwrap_err() + .to_string(); + + assert!(!error.is_empty()); + assert_eq!(fs::read_dir(outside.path()).unwrap().count(), 0); +} + +#[cfg(unix)] +#[test] +fn symlinked_inbox_cannot_escape_the_agent_capability() { + use std::os::unix::fs::symlink; + + let catalog = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + fs::create_dir_all(agent.join("resources")).unwrap(); + symlink(outside.path(), agent.join("resources/inbox")).unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "escape-inbox", + None, + None, + "payload", + false, + ) + .unwrap_err() + .to_string(); + + assert!(!error.is_empty()); + assert_eq!(fs::read_dir(outside.path()).unwrap().count(), 0); +} + #[test] fn supersede_collapses_only_the_matching_key_and_preserves_archive_receipts() { let catalog = tempfile::tempdir().unwrap(); From 3dcc7e25c0bd148a6969f480ee486cc8cbdca3b0 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 11:51:18 +0200 Subject: [PATCH 04/26] fix(stream): close remaining ingress races --- src/agent_author.rs | 32 ++++++++++++++++++++++- src/event.rs | 63 +++++++++++++++++++++++++++++++++++++++++---- tests/event_e2e.rs | 41 ++++++++++++++++++++++++++++- 3 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/agent_author.rs b/src/agent_author.rs index 5747a2b5..9b405781 100644 --- a/src/agent_author.rs +++ b/src/agent_author.rs @@ -229,7 +229,7 @@ fn author_stream( format!("acquire catalog-authoring lock: {error:#}"), ) })?; - let found = crate::discover(catalog_root); + let found = crate::discover_strict(catalog_root); if let Some(error) = found.errors.first() { return Err(AuthorError::new( "catalog-malformed", @@ -1949,4 +1949,34 @@ mod tests { "invalid-stream" ); } + + #[test] + fn stream_authoring_refuses_catalogs_with_concealed_declarations() { + use std::os::unix::fs::symlink; + + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path().join("catalog"); + let concealed = temporary.path().join("concealed"); + let declaration_path = write( + &root, + "h/worker/agent.kdl", + &declaration("worker", "h", None, "catalog"), + ); + write( + &concealed, + "agent.kdl", + &declaration("shadow", "h", None, "catalog"), + ); + symlink(&concealed, root.join("concealed-link")).unwrap(); + let original = fs::read(&declaration_path).unwrap(); + + let error = add_stream(&root, "h.worker", "h", None, "events", None).unwrap_err(); + + assert_eq!(error.code(), "catalog-malformed"); + assert!( + error.to_string().contains("unobservable declaration entry"), + "{error}" + ); + assert_eq!(fs::read(declaration_path).unwrap(), original); + } } diff --git a/src/event.rs b/src/event.rs index c064292a..7eb42487 100644 --- a/src/event.rs +++ b/src/event.rs @@ -4,6 +4,8 @@ //! own bounded, agent-local receipt ring rather than writing the agent's immutable Sent ledger. use std::fs::{self, File, OpenOptions}; +use std::io::Write as _; +use std::os::fd::{AsRawFd as _, FromRawFd as _}; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; @@ -263,7 +265,10 @@ pub fn emit( record .recent .iter() - .find(|entry| entry.key.as_deref() == key && entry.filename != filename) + .find(|entry| { + entry.filename != filename + && key.is_none_or(|key| entry.key.as_deref() == Some(key)) + }) .map(|entry| entry.filename.clone()) } else { None @@ -353,15 +358,63 @@ fn read_record(path: &Path) -> anyhow::Result> { } fn write_record(path: &Path, record: &StreamRecord) -> anyhow::Result<()> { + use std::ffi::CString; + let parent = path.parent().context("stream state has no parent")?; fs::create_dir_all(parent)?; - let temporary = parent.join(format!( + let directory = File::open(parent) + .with_context(|| format!("open stream state directory {}", parent.display()))?; + let temporary = format!( ".state.tmp-{}-{}", std::process::id(), TMP_COUNTER.fetch_add(1, Ordering::Relaxed) - )); - fs::write(&temporary, serde_json::to_vec(record)?)?; - fs::rename(&temporary, path)?; + ); + let temporary = CString::new(temporary)?; + let target = CString::new( + path.file_name() + .context("stream state has no filename")? + .as_encoded_bytes(), + )?; + let fd = unsafe { + libc::openat( + directory.as_raw_fd(), + temporary.as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0o600, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error()).with_context(|| { + format!( + "create fresh stream state temporary in {}", + parent.display() + ) + }); + } + let mut file = unsafe { File::from_raw_fd(fd) }; + let result = (|| -> anyhow::Result<()> { + file.write_all(&serde_json::to_vec(record)?)?; + file.sync_all()?; + let renamed = unsafe { + libc::renameat( + directory.as_raw_fd(), + temporary.as_ptr(), + directory.as_raw_fd(), + target.as_ptr(), + ) + }; + if renamed != 0 { + return Err(std::io::Error::last_os_error()).context("publish stream state atomically"); + } + directory.sync_all()?; + Ok(()) + })(); + if result.is_err() { + unsafe { + libc::unlinkat(directory.as_raw_fd(), temporary.as_ptr(), 0); + } + } + result?; Ok(()) } diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index c4be2f56..e417b6ed 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -256,6 +256,45 @@ fn symlinked_stream_state_ancestor_cannot_escape_the_agent_capability() { assert_eq!(fs::read_dir(outside.path()).unwrap().count(), 0); } +#[cfg(unix)] +#[test] +fn predictable_stream_state_temporary_symlink_is_never_followed() { + use std::os::unix::fs::symlink; + + let catalog = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let state_dir = agent.join("resources/streams/gh-ci"); + fs::create_dir_all(&state_dir).unwrap(); + let victim = outside.path().join("victim"); + fs::write(&victim, "must remain unchanged").unwrap(); + for counter in 0..4096 { + symlink( + &victim, + state_dir.join(format!(".state.tmp-{}-{counter}", std::process::id())), + ) + .unwrap(); + } + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "temp-symlink", + None, + None, + "payload", + false, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("fresh stream state temporary"), "{error}"); + assert_eq!(fs::read_to_string(victim).unwrap(), "must remain unchanged"); + assert!(!state_dir.join("state.json").exists()); +} + #[cfg(unix)] #[test] fn symlinked_inbox_cannot_escape_the_agent_capability() { @@ -312,7 +351,7 @@ fn supersede_collapses_only_the_matching_key_and_preserves_archive_receipts() { fn keyless_supersede_replaces_the_stream_wide_head() { let catalog = tempfile::tempdir().unwrap(); let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); - let old = emit(catalog.path(), "old", None, true); + let old = emit(catalog.path(), "old", Some("pr-1"), true); let new = emit(catalog.path(), "new", None, true); assert_eq!(new.superseded.as_deref(), Some(old.filename.as_str())); assert!(!message::inbox_dir(&agent).join(old.filename).exists()); From f73b47d253c4b48ecd1a69ab02d22d3ee161e9ee Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 12:22:03 +0200 Subject: [PATCH 05/26] docs(vrs): state stream waiter reliability contract --- .../.delta/DELTA-004-stream-dedup-horizon.md | 36 +++++++++++++++++ docs/vrs/04-stream/open-questions.md | 39 +++++++++---------- 2 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 docs/vrs/.delta/DELTA-004-stream-dedup-horizon.md diff --git a/docs/vrs/.delta/DELTA-004-stream-dedup-horizon.md b/docs/vrs/.delta/DELTA-004-stream-dedup-horizon.md new file mode 100644 index 00000000..30179cec --- /dev/null +++ b/docs/vrs/.delta/DELTA-004-stream-dedup-horizon.md @@ -0,0 +1,36 @@ +# DELTA-004: stream deduplication is bounded to the receipt ring + +## Current mismatch + +Ratified [`STREAM-R04`](../04-stream/requirements.md) promises that replaying +an event identity always returns its original filename, including after the +event is archived. [`STREAM-R05`](../04-stream/requirements.md) says +correctness never depends on the bounded ring because unread inbox copies and +archive receipts anchor replay identity. + +The shipped implementation deliberately keeps only 128 receipts per stream +and performs no inbox or archive identity scan. Within that horizon, replay is +idempotent and conflicting content fails. After eviction, the same event ID is +honestly accepted as a new event. Archive receipts remain authoritative for +their known filenames during crash recovery, but they are not an index from +`(stream, event-id)` to filename. + +## Why the implementation differs + +Searching every archive would make emit cost proportional to retained stream +history and contradict the bounded-state goal. An unread-only fallback would +make idempotency change when an agent archives an event. A bounded receipt +window gives a precise operational contract and keeps ingress work independent +of inbox/archive history. + +## Required resolution + +Requirements are protected. Maintainer approval is required to amend +STREAM-R04/R05 to make the retained receipt horizon the idempotency boundary. +Until then, the living stream spec and invariant table describe implemented +behavior, and adapters must use stable transition identities, bound their retry +and rediscovery windows, or maintain a provider-side cursor when a stronger +guarantee is required. + +Resolution must also update DQ-S3 and decision 0004, whose current wording +claims that archive receipts preserve replay identity outside the ring. diff --git a/docs/vrs/04-stream/open-questions.md b/docs/vrs/04-stream/open-questions.md index 8461fab4..ba21d938 100644 --- a/docs/vrs/04-stream/open-questions.md +++ b/docs/vrs/04-stream/open-questions.md @@ -4,26 +4,21 @@ Each entry links a spec `DQ-S*`. Questions leave this file when resolved — into [spec.md](./spec.md) as decisions or [`.experiments/`](./.experiments/) as tested hypotheses. -- **DQ-S1 Producer identity and reply routing.** What exact `from` value does a - nested stream's event carry, and what routable endpoint receives an ordinary - `message reply` to it? Candidates for producer identity are - `./` (slash keeps the bus-ID grammar unambiguous) vs - `..` (collides with the task runtime-ID grammar), but - neither currently resolves as an agent or service-principal mailbox. - Resolves by: checking every existing `from` consumer and specifying a real - reply recipient with an end-to-end proof. Typed request retirement remains - blocked until the eval-owned external requester has that working path. -- **DQ-S2 Stream state path.** Where the dedup ring lives under the owner's - resources (`resources/streams//` proposed; the ring is the only - retained-history state, accompanied by one durable in-flight publication - reservation; no supersession heads are stored). Resolves during - implementation with the state-namespace conventions of R02. -- **DQ-S3 Ring bound and identity horizon.** `K = 128` is the current +- **DQ-S1 Producer reply routing.** Events use `./` as + their producer identity, but that subordinate identity does not currently + resolve as an agent or service-principal mailbox for an ordinary + `message reply`. Resolves by: specifying a real reply recipient with an + end-to-end proof. Typed request retirement remains blocked until the + eval-owned external requester has that working path. +- **DQ-S3 Ring bound and identity horizon.** `K = 128` is the implemented deduplication and conflicting-content-detection horizon, not merely a fast - path: an evicted identity is accepted as new without scanning inbox or - archive history. Resolves by: measuring real adapter emit rates and - retry/rediscovery windows (CI transitions, builds, timer sources), then - retaining this bound or selecting another constant-size index. + path: an evicted identity is accepted as new, without scanning inbox or + archive history. This conflicts with ratified STREAM-R04/R05 and decision + 0004; [`DELTA-004`](../.delta/DELTA-004-stream-dedup-horizon.md) records the + protected-doc change required. Resolves by: measuring real adapter emit + rates and retry/rediscovery windows (CI transitions, builds, timer sources), + then approving the bounded identity contract or choosing a different + bounded index. - **DQ-S4 Request absorption staging.** The typed request/reply envelopes (`request.rs`) are absorbed by events + ordinary replies (decision 0004), but its wire types carry `deny_unknown_fields` and its invariant row names @@ -42,8 +37,10 @@ as tested hypotheses. stream died silently, the answer is yes. - **DQ-S7 Event body bounds.** Issue #238 wants bounded inbox bodies for one-inference DING handling; the DING frame carries no body, so an event's - subject is the entire wake-time signal. Resolves with #238's outcome; until - then adapters keep subjects self-sufficient. + subject is the entire wake-time signal. st2 enforces a 1,000-byte subject + ceiling but currently no body ceiling. Resolves with #238's outcome; until + then adapters keep subjects self-sufficient and impose a small explicit body + bound rather than attaching unbounded provider/build logs. - **DQ-S8 One-shot stream completion.** The task model has no run-to-completion lifecycle (`TaskLifecycle` is `Service | AdoptOnly`), so a wait-adapter that exits after its terminal event relaunches, flaps, and parks — a fault report From c79b454936bfb757f07c86a6e14dd9ceda13cd85 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 13:39:57 +0200 Subject: [PATCH 06/26] fix(stream): close review edge cases --- src/agent_author.rs | 52 +++++++++++++++++++++++++++++++++-- src/event.rs | 16 +++++++---- tests/event_e2e.rs | 22 +++++++++++++++ tests/stream_authoring_cli.rs | 30 ++++++++++++++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/src/agent_author.rs b/src/agent_author.rs index 9b405781..d54b1f05 100644 --- a/src/agent_author.rs +++ b/src/agent_author.rs @@ -241,11 +241,14 @@ fn author_stream( )); } let target = resolve_target(&found.specs, selector, this_host)?; + let actor = actor + .map(|actor| resolve_target(&found.specs, actor, this_host).map(|target| target.identity)) + .transpose()?; authorize_actor( &found.specs, &target.identity, this_host, - actor, + actor.as_deref(), "stream-not-authorized", )?; let result = edit_stream_declaration( @@ -631,6 +634,8 @@ fn edit_stream_declaration( catalog, path, &replacement, + expected_identity, + expected_host, expected_agent, name, launch, @@ -744,6 +749,8 @@ fn verify_stream_candidate( catalog: &Path, path: &Path, candidate: &str, + expected_identity: &str, + expected_host: &str, expected_agent: &str, name: &str, launch: Option<&StreamLaunch>, @@ -778,7 +785,9 @@ fn verify_stream_candidate( .map_err(|error| AuthorError::new("invalid-stream", error.to_string()))?; let spec = specs .iter() - .find(|spec| spec.identity == expected_agent) + .find(|spec| { + spec.identity == expected_agent && spec.bus_id(expected_host) == expected_identity + }) .ok_or_else(|| { AuthorError::new( "unsafe-source-edit", @@ -1891,6 +1900,45 @@ mod tests { assert_eq!(fs::read_to_string(path).unwrap(), original); } + #[test] + fn stream_candidate_verification_matches_the_exact_host_agent() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path(); + let path = write( + root, + "agents.kdl", + "agent \"worker\" { host \"alpha\"; command \"sleep 60\"; stream \"existing\" {} }\nagent \"worker\" { host \"beta\"; command \"sleep 60\"; stream \"existing\" {} }\n", + ); + + assert_eq!( + add_stream( + root, + "beta.worker", + "beta", + Some("beta.worker"), + "webhook", + None, + ) + .unwrap() + .result, + AuthorOutcome::Changed + ); + assert_eq!( + remove_stream(root, "beta.worker", "beta", Some("beta.worker"), "existing",) + .unwrap() + .result, + AuthorOutcome::Changed + ); + + let authored = fs::read_to_string(path).unwrap(); + let document = KdlDocument::parse(&authored).unwrap(); + let agents = document.nodes(); + assert!(agents[0].to_string().contains("stream \"existing\"")); + assert!(!agents[0].to_string().contains("stream \"webhook\"")); + assert!(!agents[1].to_string().contains("stream \"existing\"")); + assert!(agents[1].to_string().contains("stream \"webhook\"")); + } + #[test] fn stream_authoring_enforces_authority_nix_ownership_and_canonical_validation() { let temporary = tempfile::tempdir().unwrap(); diff --git a/src/event.rs b/src/event.rs index 7eb42487..59c73c08 100644 --- a/src/event.rs +++ b/src/event.rs @@ -261,25 +261,31 @@ pub fn emit( write_record(&record_path, &record)?; } - let predecessor = if supersede { + let predecessor_candidates = if supersede { record .recent .iter() - .find(|entry| { + .filter(|entry| { entry.filename != filename && key.is_none_or(|key| entry.key.as_deref() == Some(key)) }) .map(|entry| entry.filename.clone()) + .collect::>() } else { - None + Vec::new() }; + let mut predecessor = None; let created = message::with_resolved_message_boxes( root, &canonical_recipient, this_host, |inbox, archive| { - if let Some(predecessor) = &predecessor { - message::archive_msg(inbox, archive, predecessor)?; + if let Some(unread) = predecessor_candidates + .iter() + .find(|candidate| inbox.join(candidate).is_file()) + { + message::archive_msg(inbox, archive, unread)?; + predecessor = Some(unread.clone()); } // An archive filename is the bus's authoritative durable receipt. A crash after // materializing an external archive, but before advancing this state, must not restore diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index e417b6ed..423b8e07 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -347,6 +347,28 @@ fn supersede_collapses_only_the_matching_key_and_preserves_archive_receipts() { assert!(!inbox.join(&pr_1_old.filename).exists()); } +#[test] +fn supersede_skips_an_archived_head_and_retires_the_latest_unread_predecessor() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let older = emit(catalog.path(), "pr1-queued", Some("pr-1"), false); + let archived_head = emit(catalog.path(), "pr1-running", Some("pr-1"), false); + let inbox = message::inbox_dir(&agent); + let archive = message::archive_dir(&agent); + message::archive_msg(&inbox, &archive, &archived_head.filename).unwrap(); + + let successor = emit(catalog.path(), "pr1-pass", Some("pr-1"), true); + + assert_eq!( + successor.superseded.as_deref(), + Some(older.filename.as_str()) + ); + assert!(!inbox.join(&older.filename).exists()); + assert!(archive.join(&older.filename).exists()); + assert!(archive.join(&archived_head.filename).exists()); + assert!(inbox.join(&successor.filename).exists()); +} + #[test] fn keyless_supersede_replaces_the_stream_wide_head() { let catalog = tempfile::tempdir().unwrap(); diff --git a/tests/stream_authoring_cli.rs b/tests/stream_authoring_cli.rs index 824e01dd..c4301228 100644 --- a/tests/stream_authoring_cli.rs +++ b/tests/stream_authoring_cli.rs @@ -164,3 +164,33 @@ fn a_direct_adapter_launch_executes_the_exact_event_cli_contract() { 1 ); } + +#[test] +fn a_bare_actor_can_self_author_on_the_selected_host() { + let catalog = tempfile::tempdir().unwrap(); + write_agent(catalog.path()); + + let add = st2( + catalog.path(), + &[ + "stream", "add", "webhook", "--as", "worker", "--host", "hetz", + ], + ); + assert!( + add.status.success(), + "{}", + String::from_utf8_lossy(&add.stderr) + ); + + let remove = st2( + catalog.path(), + &[ + "stream", "rm", "webhook", "--as", "worker", "--host", "hetz", + ], + ); + assert!( + remove.status.success(), + "{}", + String::from_utf8_lossy(&remove.stderr) + ); +} From 840da94adaa9a83ea72b9abeba5d85166476be1d Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 13:51:51 +0200 Subject: [PATCH 07/26] fix(stream): publish before compacting events --- src/event.rs | 17 +++++++++-------- tests/event_e2e.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/event.rs b/src/event.rs index 59c73c08..09e6783f 100644 --- a/src/event.rs +++ b/src/event.rs @@ -280,6 +280,14 @@ pub fn emit( &canonical_recipient, this_host, |inbox, archive| { + // Publish before compacting. If predecessor archival fails or the process + // crashes between these operations, both records remain unread; replaying the + // durable pending reservation completes compaction without risking a lost wake. + let created = if archive.join(&filename).is_file() { + false + } else { + message::materialize_message_once(inbox, &filename, &rendered)? + }; if let Some(unread) = predecessor_candidates .iter() .find(|candidate| inbox.join(candidate).is_file()) @@ -287,14 +295,7 @@ pub fn emit( message::archive_msg(inbox, archive, unread)?; predecessor = Some(unread.clone()); } - // An archive filename is the bus's authoritative durable receipt. A crash after - // materializing an external archive, but before advancing this state, must not restore - // the inbox replica. - if archive.join(&filename).is_file() { - Ok(false) - } else { - message::materialize_message_once(inbox, &filename, &rendered) - } + Ok(created) }, )?; diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index 423b8e07..845d2cb0 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -369,6 +369,50 @@ fn supersede_skips_an_archived_head_and_retires_the_latest_unread_predecessor() assert!(inbox.join(&successor.filename).exists()); } +#[test] +fn failed_predecessor_archive_leaves_the_successor_unread_and_replay_completes() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let predecessor = emit(catalog.path(), "pr1-running", Some("pr-1"), false); + let inbox = message::inbox_dir(&agent); + let archive = message::archive_dir(&agent); + fs::create_dir_all(archive.join(&predecessor.filename)).unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "pr1-pass", + Some("pr-1"), + Some("CI pr1-pass"), + "{\"id\":\"pr1-pass\"}", + true, + ) + .unwrap_err(); + + assert!(error.to_string().contains("archiving"), "{error:#}"); + let unread = message::list_inbox(&inbox).unwrap(); + assert_eq!(unread.len(), 2); + assert!( + unread + .iter() + .any(|message| message.event_id.as_deref() == Some("pr1-pass")) + ); + assert!(inbox.join(&predecessor.filename).exists()); + + fs::remove_dir(archive.join(&predecessor.filename)).unwrap(); + let replay = emit(catalog.path(), "pr1-pass", Some("pr-1"), true); + assert_eq!(replay.status, EventReceiptStatus::Deduplicated); + assert_eq!( + replay.superseded.as_deref(), + Some(predecessor.filename.as_str()) + ); + assert_eq!(message::list_inbox(&inbox).unwrap().len(), 1); + assert!(!inbox.join(&predecessor.filename).exists()); + assert!(archive.join(&predecessor.filename).is_file()); +} + #[test] fn keyless_supersede_replaces_the_stream_wide_head() { let catalog = tempfile::tempdir().unwrap(); From 6421917d82f8ff5bf0f710e4deca42502d5a0a8f Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 13:53:19 +0200 Subject: [PATCH 08/26] fix(stream): require a lifecycle owner for adapters --- crates/agent-spec/src/spec.rs | 7 ++++++ crates/agent-spec/tests/discovery.rs | 35 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/crates/agent-spec/src/spec.rs b/crates/agent-spec/src/spec.rs index 6261027c..2d666fcd 100644 --- a/crates/agent-spec/src/spec.rs +++ b/crates/agent-spec/src/spec.rs @@ -1043,6 +1043,9 @@ impl RawSpec { lifecycle: TaskLifecycle::Service, }); } + let has_canonical_agent_task = tasks + .iter() + .any(|task| !task.derived && task.name == "agent"); // One derived exec companion per stream, through the exact seam the DING sidecar uses. The // marker argv carries the declared source launch; `reconcile` late-binds argv[0] to the // running st2 binary and substitutes the effective `ST_ROOT`, exactly as it does for DING @@ -1082,6 +1085,10 @@ impl RawSpec { (None, None) => None, (Some(_), Some(_)) => unreachable!("validated above"), }; + anyhow::ensure!( + launch.is_none() || has_canonical_agent_task, + "agent '{identity}' launched stream '{name}' requires a canonical `agent` task" + ); streams.push(Stream { name: name.clone(), launch, diff --git a/crates/agent-spec/tests/discovery.rs b/crates/agent-spec/tests/discovery.rs index 96f71f9e..bfc3b271 100644 --- a/crates/agent-spec/tests/discovery.rs +++ b/crates/agent-spec/tests/discovery.rs @@ -1871,6 +1871,41 @@ fn streams_are_typed_and_only_launched_streams_lower_to_derived_exec_tasks() { assert_eq!(shell.command.as_deref(), Some("watch-ci")); } +#[test] +fn launched_streams_require_a_canonical_agent_task() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "agents/h/launched/agent.kdl", + r#"agent "launched" { + host "h" + exec "worker" { command "agent" } + stream "ci" { command "watch-ci" } +}"#, + ); + write( + tmp.path(), + "agents/h/external/agent.kdl", + r#"agent "external" { + host "h" + exec "worker" { command "agent" } + stream "webhook" {} +}"#, + ); + + let found = discover(tmp.path()); + + assert_eq!(found.errors.len(), 1, "{:?}", found.errors); + assert!( + found.errors[0] + .message + .contains("launched stream 'ci' requires a canonical `agent` task"), + "{:?}", + found.errors + ); + assert_eq!(find(&found.specs, "external").streams.len(), 1); +} + #[test] fn streams_have_toml_and_json_parity_and_reject_unknown_fields() { let tmp = tempfile::tempdir().unwrap(); From b691434c6f4aa6a8ebc1e62d426e8ba8ff8662c7 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 13:56:54 +0200 Subject: [PATCH 09/26] fix(stream): reconcile interrupted ingress automatically --- src/event.rs | 68 ++++++++++++++++++++++++++++++++++++++ tests/event_e2e.rs | 82 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/src/event.rs b/src/event.rs index 09e6783f..6d9d3bd9 100644 --- a/src/event.rs +++ b/src/event.rs @@ -214,6 +214,39 @@ pub fn emit( canonical_recipient ); + if record + .pending + .as_ref() + .is_some_and(|pending| pending.event_id != event_id) + { + let pending = record + .pending + .take() + .expect("different pending event was just observed"); + let materialized = message::with_resolved_message_boxes( + root, + &canonical_recipient, + this_host, + |inbox, archive| { + Ok(message_entry_exists(inbox, &pending.filename)? + || message_entry_exists(archive, &pending.filename)?) + }, + )?; + if materialized { + record.recent.insert( + 0, + StreamEntry { + event_id: pending.event_id, + filename: pending.filename, + key: pending.key, + rendered_sha256: pending.rendered_sha256, + }, + ); + record.recent.truncate(RING_CAPACITY); + } + write_record(&record_path, &record)?; + } + let rendered_sha256 = hex_digest(rendered.as_bytes()); if let Some(entry) = record .recent @@ -259,6 +292,7 @@ pub fn emit( supersede, }); write_record(&record_path, &record)?; + test_event_checkpoint(event_id, "pending")?; } let predecessor_candidates = if supersede { @@ -288,6 +322,7 @@ pub fn emit( } else { message::materialize_message_once(inbox, &filename, &rendered)? }; + test_event_checkpoint(event_id, "materialized")?; if let Some(unread) = predecessor_candidates .iter() .find(|candidate| inbox.join(candidate).is_file()) @@ -328,6 +363,39 @@ pub fn emit( ) } +fn message_entry_exists(directory: &Path, filename: &str) -> anyhow::Result { + anyhow::ensure!( + message::is_message_filename(filename), + "invalid pending message filename {filename:?}" + ); + match fs::symlink_metadata(directory.join(filename)) { + Ok(metadata) => { + anyhow::ensure!( + metadata.is_file() && !metadata.file_type().is_symlink(), + "pending message entry {filename:?} is not a real regular file" + ); + Ok(true) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +#[cfg(debug_assertions)] +fn test_event_checkpoint(event_id: &str, point: &str) -> anyhow::Result<()> { + if std::env::var("ST2_TEST_EVENT_FAIL_AT").as_deref() + == Ok(format!("{event_id}:{point}").as_str()) + { + anyhow::bail!("injected event failure at {point}"); + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +fn test_event_checkpoint(_event_id: &str, _point: &str) -> anyhow::Result<()> { + Ok(()) +} + fn validate_component(label: &str, value: &str) -> anyhow::Result<()> { anyhow::ensure!( !value.is_empty() diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index 845d2cb0..9e88a359 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -413,6 +413,88 @@ fn failed_predecessor_archive_leaves_the_successor_unread_and_replay_completes() assert!(archive.join(&predecessor.filename).is_file()); } +#[test] +fn a_different_event_reconciles_both_pending_crash_windows() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let inbox = message::inbox_dir(&agent); + + unsafe { std::env::set_var("ST2_TEST_EVENT_FAIL_AT", "reserved-a:pending") }; + let before_materialization = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "reserved-a", + None, + Some("reserved A"), + "reserved A", + false, + ) + .unwrap_err(); + unsafe { std::env::remove_var("ST2_TEST_EVENT_FAIL_AT") }; + assert!( + before_materialization + .to_string() + .contains("injected event failure at pending") + ); + assert!(message::list_inbox(&inbox).unwrap().is_empty()); + + let after_abandoned = emit(catalog.path(), "after-abandoned", None, false); + assert_eq!(after_abandoned.status, EventReceiptStatus::Created); + + unsafe { std::env::set_var("ST2_TEST_EVENT_FAIL_AT", "materialized-a:materialized") }; + let after_materialization = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "materialized-a", + Some("pr-1"), + Some("materialized A"), + "materialized A", + false, + ) + .unwrap_err(); + unsafe { std::env::remove_var("ST2_TEST_EVENT_FAIL_AT") }; + assert!( + after_materialization + .to_string() + .contains("injected event failure at materialized") + ); + let materialized_filename = message::list_inbox(&inbox) + .unwrap() + .into_iter() + .find(|message| message.event_id.as_deref() == Some("materialized-a")) + .unwrap() + .filename; + + let after_materialized = emit(catalog.path(), "after-materialized", None, false); + assert_eq!(after_materialized.status, EventReceiptStatus::Created); + let replay = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "materialized-a", + Some("pr-1"), + Some("materialized A"), + "materialized A", + false, + ) + .unwrap(); + assert_eq!(replay.status, EventReceiptStatus::Deduplicated); + assert_eq!(replay.filename, materialized_filename); + assert_eq!( + message::list_inbox(&inbox) + .unwrap() + .into_iter() + .filter(|message| message.event_id.as_deref() == Some("materialized-a")) + .count(), + 1 + ); +} + #[test] fn keyless_supersede_replaces_the_stream_wide_head() { let catalog = tempfile::tempdir().unwrap(); From 43326401458faaaad0b87df1bdf557989a9e9992 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 14:08:32 +0200 Subject: [PATCH 10/26] fix(stream): authenticate and finish pending ingress --- src/event.rs | 132 ++++++++++++++++++++++++++++++++------------ src/message.rs | 2 +- tests/event_e2e.rs | 133 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 229 insertions(+), 38 deletions(-) diff --git a/src/event.rs b/src/event.rs index 6d9d3bd9..939ff348 100644 --- a/src/event.rs +++ b/src/event.rs @@ -4,8 +4,9 @@ //! own bounded, agent-local receipt ring rather than writing the agent's immutable Sent ledger. use std::fs::{self, File, OpenOptions}; -use std::io::Write as _; +use std::io::{Read as _, Write as _}; use std::os::fd::{AsRawFd as _, FromRawFd as _}; +use std::os::unix::fs::OpenOptionsExt as _; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; @@ -36,6 +37,8 @@ struct StreamPending { key: Option, rendered_sha256: String, supersede: bool, + #[serde(default)] + predecessor: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -228,8 +231,22 @@ pub fn emit( &canonical_recipient, this_host, |inbox, archive| { - Ok(message_entry_exists(inbox, &pending.filename)? - || message_entry_exists(archive, &pending.filename)?) + let inbox_bytes = read_message_entry(inbox, &pending.filename)?; + let archive_bytes = read_message_entry(archive, &pending.filename)?; + if let Some(bytes) = inbox_bytes.as_deref() { + validate_pending_message(stream, &pending, bytes)?; + } + if let Some(bytes) = archive_bytes.as_deref() { + validate_pending_message(stream, &pending, bytes)?; + } + if inbox_bytes.is_some() || archive_bytes.is_some() { + if let Some(predecessor) = pending.predecessor.as_deref() { + message::archive_msg(inbox, archive, predecessor)?; + } + Ok(true) + } else { + Ok(false) + } }, )?; if materialized { @@ -267,7 +284,7 @@ pub fn emit( }); } - let (filename, resumed) = match record.pending.as_ref() { + let (filename, resumed, predecessor) = match record.pending.as_ref() { Some(pending) if pending.event_id == event_id => { anyhow::ensure!( pending.rendered_sha256 == rendered_sha256 @@ -275,13 +292,34 @@ pub fn emit( && pending.supersede == supersede, "event identity `{stream}#{event_id}` reused with different content" ); - (pending.filename.clone(), true) + (pending.filename.clone(), true, pending.predecessor.clone()) } Some(pending) => anyhow::bail!( "stream '{stream}' has an interrupted event '{}'; replay it before publishing another", pending.event_id ), - None => (message::new_filename(), false), + None => { + let predecessor = if supersede { + message::with_resolved_message_boxes( + root, + &canonical_recipient, + this_host, + |inbox, _archive| { + for entry in record.recent.iter().filter(|entry| { + key.is_none_or(|key| entry.key.as_deref() == Some(key)) + }) { + if read_message_entry(inbox, &entry.filename)?.is_some() { + return Ok(Some(entry.filename.clone())); + } + } + Ok(None) + }, + )? + } else { + None + }; + (message::new_filename(), false, predecessor) + } }; if !resumed { record.pending = Some(StreamPending { @@ -290,25 +328,12 @@ pub fn emit( key: key.map(str::to_owned), rendered_sha256: rendered_sha256.clone(), supersede, + predecessor: predecessor.clone(), }); write_record(&record_path, &record)?; test_event_checkpoint(event_id, "pending")?; } - let predecessor_candidates = if supersede { - record - .recent - .iter() - .filter(|entry| { - entry.filename != filename - && key.is_none_or(|key| entry.key.as_deref() == Some(key)) - }) - .map(|entry| entry.filename.clone()) - .collect::>() - } else { - Vec::new() - }; - let mut predecessor = None; let created = message::with_resolved_message_boxes( root, &canonical_recipient, @@ -317,18 +342,21 @@ pub fn emit( // Publish before compacting. If predecessor archival fails or the process // crashes between these operations, both records remain unread; replaying the // durable pending reservation completes compaction without risking a lost wake. - let created = if archive.join(&filename).is_file() { - false - } else { - message::materialize_message_once(inbox, &filename, &rendered)? + let created = match read_message_entry(archive, &filename)? { + Some(bytes) => { + anyhow::ensure!( + bytes == rendered.as_bytes(), + "archived pending event '{}#{}' has different bytes", + stream, + event_id + ); + false + } + None => message::materialize_message_once(inbox, &filename, &rendered)?, }; test_event_checkpoint(event_id, "materialized")?; - if let Some(unread) = predecessor_candidates - .iter() - .find(|candidate| inbox.join(candidate).is_file()) - { - message::archive_msg(inbox, archive, unread)?; - predecessor = Some(unread.clone()); + if let Some(predecessor) = predecessor.as_deref() { + message::archive_msg(inbox, archive, predecessor)?; } Ok(created) }, @@ -363,24 +391,56 @@ pub fn emit( ) } -fn message_entry_exists(directory: &Path, filename: &str) -> anyhow::Result { +fn read_message_entry(directory: &Path, filename: &str) -> anyhow::Result>> { anyhow::ensure!( message::is_message_filename(filename), "invalid pending message filename {filename:?}" ); - match fs::symlink_metadata(directory.join(filename)) { - Ok(metadata) => { + let path = directory.join(filename); + match OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&path) + { + Ok(mut file) => { + let metadata = file.metadata()?; anyhow::ensure!( - metadata.is_file() && !metadata.file_type().is_symlink(), + metadata.is_file(), "pending message entry {filename:?} is not a real regular file" ); - Ok(true) + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + Ok(Some(bytes)) } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(error.into()), } } +fn validate_pending_message( + stream: &str, + pending: &StreamPending, + bytes: &[u8], +) -> anyhow::Result<()> { + anyhow::ensure!( + hex_digest(bytes) == pending.rendered_sha256, + "pending event '{}#{}' reserved file has different bytes", + stream, + pending.event_id + ); + let contents = std::str::from_utf8(bytes).context("pending event record is not UTF-8")?; + let parsed = message::parse_message(&pending.filename, contents); + anyhow::ensure!( + parsed.stream.as_deref() == Some(stream) + && parsed.event_id.as_deref() == Some(pending.event_id.as_str()) + && parsed.event_key.as_deref() == pending.key.as_deref(), + "pending event '{}#{}' reserved file has different event identity", + stream, + pending.event_id + ); + Ok(()) +} + #[cfg(debug_assertions)] fn test_event_checkpoint(event_id: &str, point: &str) -> anyhow::Result<()> { if std::env::var("ST2_TEST_EVENT_FAIL_AT").as_deref() diff --git a/src/message.rs b/src/message.rs index 9b90adc8..81e9920a 100644 --- a/src/message.rs +++ b/src/message.rs @@ -243,7 +243,7 @@ fn render_message_with_idempotency( } /// Parse a message file's contents into frontmatter fields + body. Permissive. -fn parse_message(filename: &str, contents: &str) -> Message { +pub(crate) fn parse_message(filename: &str, contents: &str) -> Message { let ts_ms = filename .split_once('-') .and_then(|(ts, _)| ts.parse::().ok()) diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index 9e88a359..a5871e83 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -1,12 +1,14 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::{Arc, Barrier}; +use std::sync::{Arc, Barrier, Mutex}; use sha2::{Digest as _, Sha256}; use st2::event::{self, EventReceiptStatus, RING_CAPACITY}; use st2::message; +static EVENT_FAIL_ENV: Mutex<()> = Mutex::new(()); + fn declare_agent(root: &Path, desired: &str, streams: &str) -> PathBuf { let directory = root.join("agents/hetz/worker"); fs::create_dir_all(&directory).unwrap(); @@ -415,6 +417,7 @@ fn failed_predecessor_archive_leaves_the_successor_unread_and_replay_completes() #[test] fn a_different_event_reconciles_both_pending_crash_windows() { + let _fail_env = EVENT_FAIL_ENV.lock().unwrap(); let catalog = tempfile::tempdir().unwrap(); let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); let inbox = message::inbox_dir(&agent); @@ -495,6 +498,134 @@ fn a_different_event_reconciles_both_pending_crash_windows() { ); } +#[test] +fn pending_reconciliation_rejects_corrupt_and_forged_reserved_files() { + let _fail_env = EVENT_FAIL_ENV.lock().unwrap(); + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + unsafe { std::env::set_var("ST2_TEST_EVENT_FAIL_AT", "reserved:pending") }; + let _ = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "reserved", + Some("pr-1"), + Some("reserved"), + "reserved", + false, + ) + .unwrap_err(); + unsafe { std::env::remove_var("ST2_TEST_EVENT_FAIL_AT") }; + + let state_path = agent.join("resources/streams/gh-ci/state.json"); + let mut state: serde_json::Value = + serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + let filename = state["pending"]["filename"].as_str().unwrap().to_owned(); + let inbox = message::inbox_dir(&agent); + fs::create_dir_all(&inbox).unwrap(); + let forged = event::render_event( + "hetz.worker/gh-ci", + Some("forged"), + "gh-ci", + "different", + Some("pr-1"), + "forged", + ); + fs::write(inbox.join(&filename), &forged).unwrap(); + + let corrupt = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "next", + None, + None, + "next", + false, + ) + .unwrap_err(); + assert!( + corrupt.to_string().contains("different bytes"), + "{corrupt:#}" + ); + + state["pending"]["renderedSha256"] = + serde_json::Value::String(format!("{:x}", Sha256::digest(forged.as_bytes()))); + fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); + let forged_identity = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "next", + None, + None, + "next", + false, + ) + .unwrap_err(); + assert!( + forged_identity + .to_string() + .contains("different event identity"), + "{forged_identity:#}" + ); + assert!( + message::list_inbox(&inbox) + .unwrap() + .iter() + .all(|message| message.event_id.as_deref() != Some("next")) + ); +} + +#[test] +fn a_different_event_completes_pending_successor_compaction() { + let _fail_env = EVENT_FAIL_ENV.lock().unwrap(); + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let predecessor = emit(catalog.path(), "running", Some("pr-1"), false); + unsafe { std::env::set_var("ST2_TEST_EVENT_FAIL_AT", "passed:materialized") }; + let _ = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "passed", + Some("pr-1"), + Some("passed"), + "passed", + true, + ) + .unwrap_err(); + unsafe { std::env::remove_var("ST2_TEST_EVENT_FAIL_AT") }; + let inbox = message::inbox_dir(&agent); + assert!(inbox.join(&predecessor.filename).is_file()); + + let next = emit(catalog.path(), "unrelated", Some("pr-2"), false); + + assert_eq!(next.status, EventReceiptStatus::Created); + assert!(!inbox.join(&predecessor.filename).exists()); + assert!( + message::archive_dir(&agent) + .join(&predecessor.filename) + .is_file() + ); + let unread = message::list_inbox(&inbox).unwrap(); + assert_eq!(unread.len(), 2); + assert!( + unread + .iter() + .any(|message| message.event_id.as_deref() == Some("passed")) + ); + assert!( + unread + .iter() + .any(|message| message.event_id.as_deref() == Some("unrelated")) + ); +} + #[test] fn keyless_supersede_replaces_the_stream_wide_head() { let catalog = tempfile::tempdir().unwrap(); From d230d80b8edb8ce16e7ddfe2e83b459b59497c25 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 14:18:08 +0200 Subject: [PATCH 11/26] fix(stream): authenticate persisted ingress files --- src/event.rs | 92 ++++++++++++++++++++++++++++++++++++++------ src/message.rs | 37 +++++++++++++++++- tests/event_e2e.rs | 96 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 211 insertions(+), 14 deletions(-) diff --git a/src/event.rs b/src/event.rs index 939ff348..614aa5f9 100644 --- a/src/event.rs +++ b/src/event.rs @@ -17,6 +17,7 @@ use sha2::{Digest as _, Sha256}; use crate::message; const EVENT_VERSION: u32 = 1; +const MAX_STATE_BYTES: u64 = 1_048_576; pub const RING_CAPACITY: usize = 128; static TMP_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -38,7 +39,7 @@ struct StreamPending { rendered_sha256: String, supersede: bool, #[serde(default)] - predecessor: Option, + predecessor: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -240,8 +241,8 @@ pub fn emit( validate_pending_message(stream, &pending, bytes)?; } if inbox_bytes.is_some() || archive_bytes.is_some() { - if let Some(predecessor) = pending.predecessor.as_deref() { - message::archive_msg(inbox, archive, predecessor)?; + if let Some(predecessor) = pending.predecessor.as_ref() { + finish_predecessor(stream, predecessor, inbox, archive)?; } Ok(true) } else { @@ -309,7 +310,7 @@ pub fn emit( key.is_none_or(|key| entry.key.as_deref() == Some(key)) }) { if read_message_entry(inbox, &entry.filename)?.is_some() { - return Ok(Some(entry.filename.clone())); + return Ok(Some(entry.clone())); } } Ok(None) @@ -355,8 +356,8 @@ pub fn emit( None => message::materialize_message_once(inbox, &filename, &rendered)?, }; test_event_checkpoint(event_id, "materialized")?; - if let Some(predecessor) = predecessor.as_deref() { - message::archive_msg(inbox, archive, predecessor)?; + if let Some(predecessor) = predecessor.as_ref() { + finish_predecessor(stream, predecessor, inbox, archive)?; } Ok(created) }, @@ -385,7 +386,7 @@ pub fn emit( } else { EventReceiptStatus::Deduplicated }, - superseded: predecessor, + superseded: predecessor.map(|entry| entry.filename), }) }, ) @@ -441,6 +442,56 @@ fn validate_pending_message( Ok(()) } +fn finish_predecessor( + stream: &str, + predecessor: &StreamEntry, + inbox: &Path, + archive: &Path, +) -> anyhow::Result<()> { + let inbox_bytes = read_message_entry(inbox, &predecessor.filename)?; + let archive_bytes = read_message_entry(archive, &predecessor.filename)?; + if let Some(bytes) = inbox_bytes.as_deref() { + validate_retained_message(stream, predecessor, bytes)?; + } + if let Some(bytes) = archive_bytes.as_deref() { + validate_retained_message(stream, predecessor, bytes)?; + } + anyhow::ensure!( + inbox_bytes.is_some() || archive_bytes.is_some(), + "supersession predecessor '{}#{}' has no inbox file or archive receipt", + stream, + predecessor.event_id + ); + if inbox_bytes.is_some() { + message::archive_msg(inbox, archive, &predecessor.filename)?; + } + Ok(()) +} + +fn validate_retained_message( + stream: &str, + entry: &StreamEntry, + bytes: &[u8], +) -> anyhow::Result<()> { + anyhow::ensure!( + hex_digest(bytes) == entry.rendered_sha256, + "supersession predecessor '{}#{}' has different bytes", + stream, + entry.event_id + ); + let contents = std::str::from_utf8(bytes).context("supersession predecessor is not UTF-8")?; + let parsed = message::parse_message(&entry.filename, contents); + anyhow::ensure!( + parsed.stream.as_deref() == Some(stream) + && parsed.event_id.as_deref() == Some(entry.event_id.as_str()) + && parsed.event_key.as_deref() == entry.key.as_deref(), + "supersession predecessor '{}#{}' has different event identity", + stream, + entry.event_id + ); + Ok(()) +} + #[cfg(debug_assertions)] fn test_event_checkpoint(event_id: &str, point: &str) -> anyhow::Result<()> { if std::env::var("ST2_TEST_EVENT_FAIL_AT").as_deref() @@ -483,10 +534,29 @@ fn hex_digest(bytes: &[u8]) -> String { } fn read_record(path: &Path) -> anyhow::Result> { - match fs::read(path) { - Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes).with_context(|| { - format!("stream state {} is malformed", path.display()) - })?)), + match OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK) + .open(path) + { + Ok(mut file) => { + let metadata = file.metadata()?; + anyhow::ensure!( + metadata.is_file(), + "stream state {} is not a real regular file", + path.display() + ); + anyhow::ensure!( + metadata.len() <= MAX_STATE_BYTES, + "stream state {} exceeds {MAX_STATE_BYTES} bytes", + path.display() + ); + let mut bytes = Vec::with_capacity(metadata.len() as usize); + file.read_to_end(&mut bytes)?; + Ok(Some(serde_json::from_slice(&bytes).with_context(|| { + format!("stream state {} is malformed", path.display()) + })?)) + } Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(error.into()), } diff --git a/src/message.rs b/src/message.rs index 81e9920a..bd2e8a40 100644 --- a/src/message.rs +++ b/src/message.rs @@ -13,7 +13,8 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs::{self, File, OpenOptions}; -use std::io::Read; +use std::io::{Read, Write as _}; +use std::os::unix::fs::OpenOptionsExt as _; use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -370,7 +371,13 @@ pub fn materialize_message_once( anyhow::bail!("message filename collision with different bytes: {filename}"); } let temporary = inbox_dir.join(tmp_name()); - fs::write(&temporary, contents)?; + let mut temporary_file = OpenOptions::new() + .write(true) + .create_new(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&temporary)?; + temporary_file.write_all(contents.as_bytes())?; + drop(temporary_file); let result = match fs::hard_link(&temporary, &destination) { Ok(()) => Ok(true), Err(_) if destination.is_file() => { @@ -2331,6 +2338,32 @@ mod tests { assert_eq!(fs::read_to_string(inbox.join(filename)).unwrap(), contents); } + #[cfg(unix)] + #[test] + fn reserved_message_temporary_symlinks_are_never_followed() { + use std::os::unix::fs::symlink; + + let tmp = tempfile::tempdir().unwrap(); + let inbox = tmp.path().join("inbox"); + fs::create_dir_all(&inbox).unwrap(); + let victim = tmp.path().join("victim"); + fs::write(&victim, "must remain unchanged").unwrap(); + let start = TMP_COUNTER.load(Ordering::Relaxed); + for counter in start..start + 4096 { + symlink( + &victim, + inbox.join(format!(".message.tmp-{}-{counter}", std::process::id())), + ) + .unwrap(); + } + + let error = materialize_message_once(&inbox, "1784649988123-symlnk.md", "must not escape") + .unwrap_err(); + + assert!(!error.to_string().is_empty()); + assert_eq!(fs::read_to_string(victim).unwrap(), "must remain unchanged"); + } + #[test] fn resolve_inbox_falls_back_to_the_flat_bus_when_catalog_less() { let tmp = tempfile::tempdir().unwrap(); diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index a5871e83..6aa39e06 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -297,6 +297,46 @@ fn predictable_stream_state_temporary_symlink_is_never_followed() { assert!(!state_dir.join("state.json").exists()); } +#[cfg(unix)] +#[test] +fn stream_state_symlink_and_fifo_fail_without_following_or_blocking() { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt as _; + use std::os::unix::fs::symlink; + + for entry in ["symlink", "fifo"] { + let catalog = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let state_dir = agent.join("resources/streams/gh-ci"); + fs::create_dir_all(&state_dir).unwrap(); + let state_path = state_dir.join("state.json"); + if entry == "symlink" { + let victim = outside.path().join("victim"); + fs::write(&victim, "must not be read").unwrap(); + symlink(&victim, &state_path).unwrap(); + } else { + let path = CString::new(state_path.as_os_str().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o600) }, 0); + } + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + entry, + None, + None, + "payload", + false, + ) + .unwrap_err(); + + assert!(!error.to_string().is_empty()); + } +} + #[cfg(unix)] #[test] fn symlinked_inbox_cannot_escape_the_agent_capability() { @@ -393,7 +433,10 @@ fn failed_predecessor_archive_leaves_the_successor_unread_and_replay_completes() ) .unwrap_err(); - assert!(error.to_string().contains("archiving"), "{error:#}"); + assert!( + error.to_string().contains("not a real regular file"), + "{error:#}" + ); let unread = message::list_inbox(&inbox).unwrap(); assert_eq!(unread.len(), 2); assert!( @@ -626,6 +669,57 @@ fn a_different_event_completes_pending_successor_compaction() { ); } +#[test] +fn pending_supersession_authenticates_its_predecessor_before_archive() { + let _fail_env = EVENT_FAIL_ENV.lock().unwrap(); + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let predecessor = emit(catalog.path(), "running", Some("pr-1"), false); + unsafe { std::env::set_var("ST2_TEST_EVENT_FAIL_AT", "passed:materialized") }; + let _ = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "passed", + Some("pr-1"), + Some("passed"), + "passed", + true, + ) + .unwrap_err(); + unsafe { std::env::remove_var("ST2_TEST_EVENT_FAIL_AT") }; + let inbox = message::inbox_dir(&agent); + fs::write(inbox.join(&predecessor.filename), "forged predecessor").unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "unrelated", + None, + None, + "unrelated", + false, + ) + .unwrap_err(); + + assert!(error.to_string().contains("different bytes"), "{error:#}"); + assert!(inbox.join(&predecessor.filename).is_file()); + assert!( + !message::archive_dir(&agent) + .join(&predecessor.filename) + .exists() + ); + assert!( + message::list_inbox(&inbox) + .unwrap() + .iter() + .all(|message| message.event_id.as_deref() != Some("unrelated")) + ); +} + #[test] fn keyless_supersede_replaces_the_stream_wide_head() { let catalog = tempfile::tempdir().unwrap(); From 55ab6a928f32fe9cf60f8049ad4d5ccdcce1e961 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 14:30:01 +0200 Subject: [PATCH 12/26] fix(stream): serialize admission with catalog edits --- src/catalog_transaction.rs | 65 ++++++++++++++- src/event.rs | 20 ++++- tests/event_e2e.rs | 159 +++++++++++++++++++++++++++++++++++-- 3 files changed, 234 insertions(+), 10 deletions(-) diff --git a/src/catalog_transaction.rs b/src/catalog_transaction.rs index e6e4f656..75d69df8 100644 --- a/src/catalog_transaction.rs +++ b/src/catalog_transaction.rs @@ -600,7 +600,7 @@ fn agent_semantic_deltas( } fn normalize_agent(spec: &agent_spec::AgentSpec) -> Result> { - use agent_spec::{Restart, RestartMode, TaskKind, TaskLifecycle}; + use agent_spec::{Restart, RestartMode, StreamLaunch, TaskKind, TaskLifecycle}; let host = spec .host @@ -682,6 +682,48 @@ fn normalize_agent(spec: &agent_spec::AgentSpec) -> Result insert_value( + &mut fields, + &format!("{stream_base}/launch"), + SemanticType::String, + "external", + ), + Some(StreamLaunch::Command(command)) => { + insert_value( + &mut fields, + &format!("{stream_base}/launch"), + SemanticType::String, + "command", + ); + insert_value( + &mut fields, + &format!("{stream_base}/command"), + SemanticType::String, + command, + ); + } + Some(StreamLaunch::Argv(argv)) => { + insert_value( + &mut fields, + &format!("{stream_base}/launch"), + SemanticType::String, + "argv", + ); + for (index, argument) in argv.iter().enumerate() { + insert_value( + &mut fields, + &format!("{stream_base}/argv/{index}"), + SemanticType::String, + argument, + ); + } + } + } + } + let restart = spec.restart_policy(); let default_restart = Restart::default(); insert_default_value( @@ -3421,6 +3463,27 @@ fn test_forced_cross_device(_point: &str) -> std::io::Result<()> { mod tests { use super::*; + #[test] + fn external_streams_participate_in_semantic_catalog_projection() { + let root = tempfile::tempdir().unwrap(); + let agent = root.path().join("agents/host/worker"); + std::fs::create_dir_all(&agent).unwrap(); + std::fs::write( + agent.join("agent.kdl"), + "agent \"worker\" {\n host \"host\"\n command \"worker\"\n stream \"webhook\" {}\n}\n", + ) + .unwrap(); + let discovered = agent_spec::discover_strict(root.path()); + assert!(discovered.errors.is_empty(), "{:?}", discovered.errors); + + let fields = normalize_agent(&discovered.specs[0]).unwrap(); + + let atom = fields + .get("/agents/host/worker/streams/webhook/launch") + .expect("external stream must affect semantic projection"); + assert_eq!(atom, &present_atom(SemanticType::String, "external")); + } + #[test] fn workspace_directory_facts_are_typed_into_the_projection_hash() { let files = BTreeMap::new(); diff --git a/src/event.rs b/src/event.rs index 614aa5f9..c519f0bf 100644 --- a/src/event.rs +++ b/src/event.rs @@ -194,7 +194,7 @@ pub fn emit( } // Serialize the eligibility observation with self-authoring and desired-state changes. Once a // suspension edit owns this lock, no later emit can publish from a stale running observation. - let _catalog_lock = crate::catalog_lock::CatalogLock::exclusive(root)?; + let _catalog_lock = crate::catalog_lock::CatalogLock::shared(root)?; let resolved = resolve_stream(root, this_host, recipient, stream)?; let canonical_recipient = resolved.recipient; let from = format!("{canonical_recipient}/{stream}"); @@ -305,11 +305,13 @@ pub fn emit( root, &canonical_recipient, this_host, - |inbox, _archive| { + |inbox, archive| { for entry in record.recent.iter().filter(|entry| { key.is_none_or(|key| entry.key.as_deref() == Some(key)) }) { - if read_message_entry(inbox, &entry.filename)?.is_some() { + if read_message_entry(inbox, &entry.filename)?.is_some() + && read_message_entry(archive, &entry.filename)?.is_none() + { return Ok(Some(entry.clone())); } } @@ -353,7 +355,17 @@ pub fn emit( ); false } - None => message::materialize_message_once(inbox, &filename, &rendered)?, + None => match read_message_entry(inbox, &filename)? { + Some(bytes) => { + validate_pending_message( + stream, + record.pending.as_ref().expect("pending reservation exists"), + &bytes, + )?; + false + } + None => message::materialize_message_once(inbox, &filename, &rendered)?, + }, }; test_event_checkpoint(event_id, "materialized")?; if let Some(predecessor) = predecessor.as_ref() { diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index 6aa39e06..1e0e2821 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -1,7 +1,8 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::{Arc, Barrier, Mutex}; +use std::sync::{Arc, Barrier, Mutex, mpsc}; +use std::time::Duration; use sha2::{Digest as _, Sha256}; use st2::event::{self, EventReceiptStatus, RING_CAPACITY}; @@ -412,7 +413,155 @@ fn supersede_skips_an_archived_head_and_retires_the_latest_unread_predecessor() } #[test] -fn failed_predecessor_archive_leaves_the_successor_unread_and_replay_completes() { +fn supersede_skips_an_archive_shadowed_inbox_candidate() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let older = emit(catalog.path(), "pr1-queued", Some("pr-1"), false); + let shadowed = emit(catalog.path(), "pr1-running", Some("pr-1"), false); + let inbox = message::inbox_dir(&agent); + let archive = message::archive_dir(&agent); + fs::create_dir_all(&archive).unwrap(); + fs::copy( + inbox.join(&shadowed.filename), + archive.join(&shadowed.filename), + ) + .unwrap(); + + let successor = emit(catalog.path(), "pr1-pass", Some("pr-1"), true); + + assert_eq!( + successor.superseded.as_deref(), + Some(older.filename.as_str()) + ); + assert!(inbox.join(&shadowed.filename).is_file()); + assert!(archive.join(&shadowed.filename).is_file()); + assert!(archive.join(&older.filename).is_file()); +} + +#[test] +fn initial_supersession_authenticates_predecessor_immediately_before_archive() { + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let predecessor = emit(catalog.path(), "running", Some("pr-1"), false); + let inbox = message::inbox_dir(&agent); + fs::write(inbox.join(&predecessor.filename), "forged predecessor").unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "passed", + Some("pr-1"), + Some("passed"), + "passed", + true, + ) + .unwrap_err(); + + assert!(error.to_string().contains("different bytes"), "{error:#}"); + assert!(inbox.join(&predecessor.filename).is_file()); + assert!( + !message::archive_dir(&agent) + .join(&predecessor.filename) + .exists() + ); + assert!( + message::list_inbox(&inbox) + .unwrap() + .iter() + .any(|message| { message.event_id.as_deref() == Some("passed") }) + ); +} + +#[test] +fn catalog_authoring_and_emit_linearize_without_deadlock() { + let catalog = tempfile::tempdir().unwrap(); + declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let admission = st2::CatalogLock::shared(catalog.path()).unwrap(); + let root = catalog.path().to_path_buf(); + let (done_tx, done_rx) = mpsc::channel(); + let author = std::thread::spawn(move || { + let result = st2::agent_author::remove_stream(&root, "hetz.worker", "hetz", None, "gh-ci"); + done_tx.send(result).unwrap(); + }); + assert!(done_rx.recv_timeout(Duration::from_millis(100)).is_err()); + + let receipt = emit(catalog.path(), "before-remove", None, false); + assert_eq!(receipt.status, EventReceiptStatus::Created); + drop(admission); + done_rx + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .unwrap(); + author.join().unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "after-remove", + None, + None, + "after", + false, + ) + .unwrap_err(); + assert!( + error.to_string().contains("does not declare stream"), + "{error:#}" + ); +} + +#[test] +fn desired_state_authoring_and_emit_linearize_without_deadlock() { + let catalog = tempfile::tempdir().unwrap(); + declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let admission = st2::CatalogLock::shared(catalog.path()).unwrap(); + let root = catalog.path().to_path_buf(); + let (done_tx, done_rx) = mpsc::channel(); + let author = std::thread::spawn(move || { + let result = st2::agent_author::set_desired_state( + &root, + "hetz.worker", + "hetz", + None, + st2::agent_author::DesiredStateValue::Suspended, + Some("maintenance"), + ); + done_tx.send(result).unwrap(); + }); + assert!(done_rx.recv_timeout(Duration::from_millis(100)).is_err()); + + assert_eq!( + emit(catalog.path(), "before-suspend", None, false).status, + EventReceiptStatus::Created + ); + drop(admission); + done_rx + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .unwrap(); + author.join().unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "after-suspend", + None, + None, + "after", + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("is suspended"), "{error:#}"); +} + +#[test] +fn invalid_archive_receipt_blocks_supersession_before_successor_publication() { let catalog = tempfile::tempdir().unwrap(); let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); let predecessor = emit(catalog.path(), "pr1-running", Some("pr-1"), false); @@ -438,17 +587,17 @@ fn failed_predecessor_archive_leaves_the_successor_unread_and_replay_completes() "{error:#}" ); let unread = message::list_inbox(&inbox).unwrap(); - assert_eq!(unread.len(), 2); + assert_eq!(unread.len(), 1); assert!( unread .iter() - .any(|message| message.event_id.as_deref() == Some("pr1-pass")) + .all(|message| message.event_id.as_deref() != Some("pr1-pass")) ); assert!(inbox.join(&predecessor.filename).exists()); fs::remove_dir(archive.join(&predecessor.filename)).unwrap(); let replay = emit(catalog.path(), "pr1-pass", Some("pr-1"), true); - assert_eq!(replay.status, EventReceiptStatus::Deduplicated); + assert_eq!(replay.status, EventReceiptStatus::Created); assert_eq!( replay.superseded.as_deref(), Some(predecessor.filename.as_str()) From 1655ea9af83ebe6267d390b1ecc8c0fec91f48c1 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 14:35:21 +0200 Subject: [PATCH 13/26] test(stream): prove recovery receipts and exact argv --- src/main.rs | 2 +- tests/event_e2e.rs | 94 +++++++++++++++++++++++++++++++++-- tests/stream_authoring_cli.rs | 77 ++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index c5c49a10..94a618fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -874,7 +874,7 @@ enum StreamCmd { /// Adapter command run under `sh -c`; omit both launch forms for external ingress. #[arg(long, conflicts_with = "adapter_argv")] command: Option, - /// Direct adapter argv. Element 0 is the program. + /// Direct adapter argv after `--`. Element 0 is the program; values are preserved exactly. #[arg(trailing_var_arg = true, allow_hyphen_values = true)] adapter_argv: Vec, #[arg(long)] diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index 1e0e2821..0a7880ae 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -451,10 +451,10 @@ fn initial_supersession_authenticates_predecessor_immediately_before_archive() { "hetz", "hetz.worker", "gh-ci", - "passed", + "initial-passed", Some("pr-1"), - Some("passed"), - "passed", + Some("initial passed"), + "initial passed", true, ) .unwrap_err(); @@ -470,7 +470,7 @@ fn initial_supersession_authenticates_predecessor_immediately_before_archive() { message::list_inbox(&inbox) .unwrap() .iter() - .any(|message| { message.event_id.as_deref() == Some("passed") }) + .any(|message| { message.event_id.as_deref() == Some("initial-passed") }) ); } @@ -869,6 +869,92 @@ fn pending_supersession_authenticates_its_predecessor_before_archive() { ); } +#[test] +fn pending_supersession_accepts_an_authenticated_archive_only_predecessor() { + let _fail_env = EVENT_FAIL_ENV.lock().unwrap(); + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let predecessor = emit(catalog.path(), "running", Some("pr-1"), false); + unsafe { std::env::set_var("ST2_TEST_EVENT_FAIL_AT", "passed:materialized") }; + let _ = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "passed", + Some("pr-1"), + Some("passed"), + "passed", + true, + ) + .unwrap_err(); + unsafe { std::env::remove_var("ST2_TEST_EVENT_FAIL_AT") }; + let inbox = message::inbox_dir(&agent); + let archive = message::archive_dir(&agent); + message::archive_msg(&inbox, &archive, &predecessor.filename).unwrap(); + + let next = emit(catalog.path(), "unrelated", Some("pr-2"), false); + + assert_eq!(next.status, EventReceiptStatus::Created); + assert!(archive.join(&predecessor.filename).is_file()); + assert!( + message::list_inbox(&inbox) + .unwrap() + .iter() + .any(|message| { message.event_id.as_deref() == Some("passed") }) + ); +} + +#[test] +fn pending_supersession_fails_closed_when_predecessor_has_no_receipt() { + let _fail_env = EVENT_FAIL_ENV.lock().unwrap(); + let catalog = tempfile::tempdir().unwrap(); + let agent = declare_agent(catalog.path(), "\"running\"", " stream \"gh-ci\" {}\n"); + let predecessor = emit(catalog.path(), "running", Some("pr-1"), false); + unsafe { std::env::set_var("ST2_TEST_EVENT_FAIL_AT", "passed:materialized") }; + let _ = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "passed", + Some("pr-1"), + Some("passed"), + "passed", + true, + ) + .unwrap_err(); + unsafe { std::env::remove_var("ST2_TEST_EVENT_FAIL_AT") }; + let inbox = message::inbox_dir(&agent); + fs::remove_file(inbox.join(&predecessor.filename)).unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "hetz.worker", + "gh-ci", + "unrelated", + None, + None, + "unrelated", + false, + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("has no inbox file or archive receipt"), + "{error:#}" + ); + assert!( + message::list_inbox(&inbox) + .unwrap() + .iter() + .all(|message| { message.event_id.as_deref() != Some("unrelated") }) + ); +} + #[test] fn keyless_supersede_replaces_the_stream_wide_head() { let catalog = tempfile::tempdir().unwrap(); diff --git a/tests/stream_authoring_cli.rs b/tests/stream_authoring_cli.rs index c4301228..7ad821eb 100644 --- a/tests/stream_authoring_cli.rs +++ b/tests/stream_authoring_cli.rs @@ -165,6 +165,83 @@ fn a_direct_adapter_launch_executes_the_exact_event_cli_contract() { ); } +#[test] +fn direct_adapter_argv_preserves_spaces_and_metacharacters_exactly() { + let catalog = tempfile::tempdir().unwrap(); + write_agent(catalog.path()); + let expected = [ + "/bin/example adapter", + "argument with spaces", + "$HOME", + "$(never-executed)", + "semi;colon", + "quote\"and\\slash", + "--looks-like-a-flag", + ]; + let mut args = vec![ + "stream", + "add", + "exact-argv", + "--agent", + "hetz.worker", + "--host", + "hetz", + "--", + ]; + args.extend(expected); + + let add = st2(catalog.path(), &args); + + assert!( + add.status.success(), + "{}", + String::from_utf8_lossy(&add.stderr) + ); + let spec = st2::discover(catalog.path()).specs.remove(0); + let stream = spec + .streams + .iter() + .find(|stream| stream.name == "exact-argv") + .unwrap(); + assert_eq!( + stream.launch, + Some(st2::spec::StreamLaunch::Argv( + expected.iter().map(|value| (*value).to_owned()).collect() + )) + ); +} + +#[test] +fn command_and_direct_argv_are_mutually_exclusive() { + let catalog = tempfile::tempdir().unwrap(); + write_agent(catalog.path()); + + let add = st2( + catalog.path(), + &[ + "stream", + "add", + "ambiguous", + "--agent", + "hetz.worker", + "--host", + "hetz", + "--command", + "echo shell", + "--", + "/bin/echo", + "direct", + ], + ); + + assert!(!add.status.success()); + assert!( + String::from_utf8_lossy(&add.stderr).contains("cannot be used with"), + "{}", + String::from_utf8_lossy(&add.stderr) + ); +} + #[test] fn a_bare_actor_can_self_author_on_the_selected_host() { let catalog = tempfile::tempdir().unwrap(); From f6de16c950b126cc68ef7452a6b6a472d8f785e8 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 14:41:02 +0200 Subject: [PATCH 14/26] fix(stream): enforce owner-host event admission --- src/event.rs | 6 ++++++ tests/event_e2e.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/event.rs b/src/event.rs index c519f0bf..567c7784 100644 --- a/src/event.rs +++ b/src/event.rs @@ -130,6 +130,12 @@ fn resolve_stream( let spec = matches .pop() .context("exactly one matching agent expected")?; + anyhow::ensure!( + spec.resolved_host(this_host) == this_host, + "agent '{}' is owned by host '{}'; event publication must run on that host", + spec.bus_id(this_host), + spec.resolved_host(this_host) + ); anyhow::ensure!( spec.streams.iter().any(|declared| declared.name == stream), "agent '{}' does not declare stream '{stream}'", diff --git a/tests/event_e2e.rs b/tests/event_e2e.rs index 0a7880ae..6ec22850 100644 --- a/tests/event_e2e.rs +++ b/tests/event_e2e.rs @@ -198,6 +198,40 @@ fn ambiguous_recipient_matching_a_bus_id_and_local_identity_fails_closed() { assert!(!message::inbox_dir(&ambiguous).exists()); } +#[test] +fn exact_remote_bus_id_cannot_bypass_the_owner_host_lock_domain() { + let catalog = tempfile::tempdir().unwrap(); + let remote = catalog.path().join("agents/berlin/worker"); + fs::create_dir_all(&remote).unwrap(); + fs::write( + remote.join("agent.kdl"), + "agent \"worker\" {\n host \"berlin\"\n desired-state \"running\"\n command \"agent\"\n stream \"gh-ci\" {}\n}\n", + ) + .unwrap(); + + let error = event::emit( + catalog.path(), + "hetz", + "berlin.worker", + "gh-ci", + "remote-attempt", + None, + None, + "payload", + false, + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("event publication must run on that host"), + "{error:#}" + ); + assert!(!message::inbox_dir(&remote).exists()); + assert!(!remote.join("resources/streams/gh-ci").exists()); +} + #[cfg(unix)] #[test] fn unobservable_declaration_entry_blocks_event_recipient_resolution() { From 34fcc4d27415510d8c1b7bbf8f80830f8151fb83 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Thu, 20 Aug 2026 14:50:08 +0200 Subject: [PATCH 15/26] fix(stream): make event receipts crash durable --- src/event.rs | 131 +++++++++++++++++++++++--- src/main.rs | 11 ++- src/message.rs | 4 + tests/event_e2e.rs | 168 ++++++++++++++++++++++++++++++++++ tests/stream_authoring_cli.rs | 7 +- 5 files changed, 307 insertions(+), 14 deletions(-) diff --git a/src/event.rs b/src/event.rs index 567c7784..0fd4a2be 100644 --- a/src/event.rs +++ b/src/event.rs @@ -4,8 +4,9 @@ //! own bounded, agent-local receipt ring rather than writing the agent's immutable Sent ledger. use std::fs::{self, File, OpenOptions}; -use std::io::{Read as _, Write as _}; +use std::io::{Read as _, Seek as _, Write as _}; use std::os::fd::{AsRawFd as _, FromRawFd as _}; +use std::os::unix::ffi::OsStrExt as _; use std::os::unix::fs::OpenOptionsExt as _; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; @@ -28,6 +29,10 @@ struct StreamEntry { filename: String, key: Option, rendered_sha256: String, + #[serde(default)] + supersede: bool, + #[serde(default)] + predecessor: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -264,6 +269,11 @@ pub fn emit( filename: pending.filename, key: pending.key, rendered_sha256: pending.rendered_sha256, + supersede: pending.supersede, + predecessor: pending + .predecessor + .as_ref() + .map(|entry| entry.filename.clone()), }, ); record.recent.truncate(RING_CAPACITY); @@ -281,13 +291,17 @@ pub fn emit( entry.rendered_sha256 == rendered_sha256, "event identity `{stream}#{event_id}` reused with different content" ); + anyhow::ensure!( + entry.supersede == supersede, + "event identity `{stream}#{event_id}` reused with different supersession intent" + ); return Ok(EventReceipt { recipient: canonical_recipient.clone(), stream: stream.to_owned(), event_id: event_id.to_owned(), filename: entry.filename.clone(), status: EventReceiptStatus::Deduplicated, - superseded: None, + superseded: entry.predecessor.clone(), }); } @@ -351,7 +365,7 @@ pub fn emit( // Publish before compacting. If predecessor archival fails or the process // crashes between these operations, both records remain unread; replaying the // durable pending reservation completes compaction without risking a lost wake. - let created = match read_message_entry(archive, &filename)? { + let (created, published_dir) = match read_message_entry(archive, &filename)? { Some(bytes) => { anyhow::ensure!( bytes == rendered.as_bytes(), @@ -359,7 +373,7 @@ pub fn emit( stream, event_id ); - false + (false, archive) } None => match read_message_entry(inbox, &filename)? { Some(bytes) => { @@ -368,11 +382,16 @@ pub fn emit( record.pending.as_ref().expect("pending reservation exists"), &bytes, )?; - false + (false, inbox) } - None => message::materialize_message_once(inbox, &filename, &rendered)?, + None => ( + message::materialize_message_once(inbox, &filename, &rendered)?, + inbox, + ), }, }; + sync_message_entry(published_dir, &filename)?; + test_event_checkpoint(event_id, "durable")?; test_event_checkpoint(event_id, "materialized")?; if let Some(predecessor) = predecessor.as_ref() { finish_predecessor(stream, predecessor, inbox, archive)?; @@ -389,6 +408,8 @@ pub fn emit( filename: filename.clone(), key: key.map(str::to_owned), rendered_sha256, + supersede, + predecessor: predecessor.as_ref().map(|entry| entry.filename.clone()), }, ); record.recent.truncate(RING_CAPACITY); @@ -411,6 +432,10 @@ pub fn emit( } fn read_message_entry(directory: &Path, filename: &str) -> anyhow::Result>> { + Ok(open_message_entry(directory, filename)?.map(|(_, bytes)| bytes)) +} + +fn open_message_entry(directory: &Path, filename: &str) -> anyhow::Result)>> { anyhow::ensure!( message::is_message_filename(filename), "invalid pending message filename {filename:?}" @@ -429,7 +454,7 @@ fn read_message_entry(directory: &Path, filename: &str) -> anyhow::Result