From 346efe4dae1e2a6977447c8685f4fb26f26f939c Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Fri, 31 Jul 2026 17:27:30 +0800 Subject: [PATCH 1/7] feat: three-level tool UI mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Users wanted a middle ground between silent and verbose tool output. Add ToolUiMode (silent/minimal/verbose) stored as tool_ui_mode_{user_id}; /verbose now cycles all three. Migrates legacy tool_ui_enabled boolean (true→verbose, false→silent). Adds delete_message to PlatformSender for minimal-mode cleanup. --- CONTEXT.md | 33 +++++ Cargo.lock | 2 +- docs/adr/0002-three-level-tool-ui-mode.md | 34 +++++ src/agent.rs | 4 + src/command_tool.rs | 112 ++++++++++++---- src/main.rs | 6 +- src/platform/sender.rs | 2 + src/platform/telegram.rs | 153 +++++++++++++++------- src/platform/tool_notifier.rs | 46 ++++--- src/supervisor/backend/reasoning.rs | 3 +- src/tool_registry.rs | 62 +++++++++ 11 files changed, 360 insertions(+), 97 deletions(-) create mode 100644 CONTEXT.md create mode 100644 docs/adr/0002-three-level-tool-ui-mode.md diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..7993bb6 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,33 @@ +# CONTEXT.md — Domain Glossary + +**Single-context repo.** All terms defined here. + +--- + +## Tool UI Mode + +Three levels of tool execution visibility for the user. + +| Mode | Tool Notifier | Command Output | Cancel Button | +|------|---------------|----------------|---------------| +| Silent | Off (placeholder only) | Hidden | Hidden | +| Minimal | Tool name + status, no args | Hidden | Visible (simple text, no live output) | +| Verbose | Tool name + args + status | Live stream + result | Visible (with live output) | + +### Tool Notifier +Telegram message that live-edits to show agent tool activity. Created per-message when mode ≠ Silent. + +### Command Tool Output +Separate Telegram message from `execute_command` tool showing shell command + stdout/stderr. Suppressed in Silent/Minimal. + +### Cancel Button +Inline keyboard button on command message allowing user to SIGKILL the running command. Available in Verbose + Minimal. + +### Tool Activity +Entry in Tool Notifier showing: friendly tool name, optional args preview, status label (⏳/✓/✗). + +### Friendly Tool Name +Human-readable label for built-in tools (e.g., "💻 Running a command" for `execute_command`). + +### Args Preview +Truncated (60 chars), redacted JSON args shown in Tool Notifier. Only in Verbose mode. \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 974aaa8..7d6dcc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2660,7 +2660,7 @@ dependencies = [ [[package]] name = "rustfox" -version = "1.0.1" +version = "1.0.2" dependencies = [ "anyhow", "async-trait", diff --git a/docs/adr/0002-three-level-tool-ui-mode.md b/docs/adr/0002-three-level-tool-ui-mode.md new file mode 100644 index 0000000..ec47751 --- /dev/null +++ b/docs/adr/0002-three-level-tool-ui-mode.md @@ -0,0 +1,34 @@ +# ADR 0002: Three-Level Tool UI Mode + +## Status +Accepted + +## Date +2026-07-31 + +## Context +Users requested a middle ground between: +- Silent: no tool progress messages (only "⏳ Thinking..." placeholder) +- Verbose: full tool call details including args, live command output, results + +The original binary `tool_ui_enabled` (true/false) key couldn't express this. + +## Decision +Introduce three `ToolUiMode` variants stored as `tool_ui_mode_{user_id}`: +- **Silent**: no Tool Notifier, no command output, no cancel button +- **Minimal**: Tool Notifier shows tool name + status (no args), command output hidden, cancel button available +- **Verbose**: Tool Notifier shows tool name + args + status, live command output + result, cancel button with live output + +`/verbose` command cycles: Minimal → Verbose → Silent → Minimal + +## Rationale +- Minimal gives users awareness of *what* tool runs without leaking args or command output +- Cancel button in Minimal lets users stop long-running commands without seeing output +- Silent mode retains original "placeholder only" behavior +- Backward compatible: old `tool_ui_enabled=true` → Verbose, `false` → Silent (old "false" = no tool UI at all), absent key → Minimal (new default) + +## Consequences +- Migration writes new key only when old key exists (idempotent; absent key keeps the live default) +- `execute_command` in Minimal sends cancel-button message, deletes it on completion +- Silent mode sends nothing; Tool Notifier also disabled +- Default for new users: Minimal \ No newline at end of file diff --git a/src/agent.rs b/src/agent.rs index d64371d..6fb5adf 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -631,6 +631,7 @@ impl Agent { incoming: &IncomingMessage, tool_event_tx: Option>, stream_token_tx: Option>, + tool_ui_mode: crate::tool_registry::ToolUiMode, ) -> Result { let platform = &incoming.platform; let user_id = &incoming.user_id; @@ -779,6 +780,7 @@ impl Agent { let home_dir = self.config.resolved_home.clone(); let sender = self.sender.clone(); let cancel_registry = self.cancel_registry.clone(); + let mode = tool_ui_mode; move |_user_id: &str, _chat_id: &str| ToolContext { sandbox_dir: sandbox_dir.clone(), home_dir: home_dir.clone(), @@ -786,6 +788,7 @@ impl Agent { cancel_registry: cancel_registry.clone(), user_id: _user_id.to_string(), chat_id: _chat_id.to_string(), + tool_ui_mode: mode, } }; @@ -1438,6 +1441,7 @@ impl Agent { cancel_registry: cancel_registry.clone(), user_id: String::new(), chat_id: String::new(), + tool_ui_mode: crate::tool_registry::ToolUiMode::Minimal, } }; diff --git a/src/command_tool.rs b/src/command_tool.rs index ffc768f..af8944f 100644 --- a/src/command_tool.rs +++ b/src/command_tool.rs @@ -11,7 +11,17 @@ use tracing::warn; use crate::cancel_registry::CancelRegistry; use crate::llm::{FunctionDefinition, ToolDefinition}; use crate::platform::sender::PlatformSender; -use crate::tool_registry::{ToolContext, ToolHandler, ToolResult}; +use crate::tool_registry::{ToolContext, ToolHandler, ToolResult, ToolUiMode}; + +/// Controls how command execution messages are sent to Telegram. +enum SendMode { + /// Full live output with cancel button. + Verbose, + /// Cancel button only, no live edits. Message deleted on completion. + Minimal, + /// No message sent. Tool notifier handles nothing (silent mode). + Silent, +} pub struct CommandTool { sandbox_dir: PathBuf, @@ -78,11 +88,29 @@ impl CommandTool { let escaped_cmd = crate::utils::telegram_markdown::escape_text(command); - let status_text = format!("💻 Running: `{}`\n\n```\n⏳ Starting...\n```", escaped_cmd); - let msg_id = self - .sender - .show_cancel_button(&ctx.chat_id, &status_text, &cmd_id) - .await?; + // Verbose: cancel button + live output + final result + // Minimal: cancel button (simple text) + no live output, delete on finish + // Silent: no message at all (tool_notifier handles nothing) + let (msg_id, send_mode) = match ctx.tool_ui_mode { + ToolUiMode::Verbose => { + let status_text = + format!("💻 Running: `{}`\n\n```\n⏳ Starting...\n```", escaped_cmd); + let id = self + .sender + .show_cancel_button(&ctx.chat_id, &status_text, &cmd_id) + .await?; + (Some(id), SendMode::Verbose) + } + ToolUiMode::Minimal => { + let status_text = format!("⏳ Running: `{}`", escaped_cmd); + let id = self + .sender + .show_cancel_button(&ctx.chat_id, &status_text, &cmd_id) + .await?; + (Some(id), SendMode::Minimal) + } + ToolUiMode::Silent => (None, SendMode::Silent), + }; let (cancel_tx, cancel_rx) = tokio::sync::oneshot::channel::<()>(); self.cancel_registry @@ -136,11 +164,13 @@ impl CommandTool { if output_buffer.chars().count() > MAX_BUFFER_CHARS { output_buffer = crate::utils::strings::truncate_tail(&output_buffer, MAX_BUFFER_CHARS); } - if last_edit.elapsed() >= std::time::Duration::from_millis(500) { + if matches!(send_mode, SendMode::Verbose) && last_edit.elapsed() >= std::time::Duration::from_millis(500) { let capped = crate::utils::strings::truncate_tail(&output_buffer, 3500); let text = format!("💻 Running: `{}`\n\n```\n{}\n```", escaped_cmd, capped); - if let Err(e) = self.sender.edit_message(&ctx.chat_id, &msg_id, &text).await { - warn!("Failed to update running message: {e}"); + if let Some(mid) = &msg_id { + if let Err(e) = self.sender.edit_message(&ctx.chat_id, mid, &text).await { + warn!("Failed to update running message: {e}"); + } } last_edit = Instant::now(); } @@ -186,28 +216,52 @@ impl CommandTool { } let result = if cancelled { - let body = format_body(&output_buffer, ""); - let text = match body { - None => format!("❌ Cancelled: `{}`", escaped_cmd), - Some(b) => format!("❌ Cancelled: `{}`\n\n{}", escaped_cmd, b), - }; - let _ = self.sender.edit_message(&ctx.chat_id, &msg_id, &text).await; + if let Some(mid) = &msg_id { + match send_mode { + SendMode::Verbose => { + let body = format_body(&output_buffer, ""); + let text = match body { + None => format!("❌ Cancelled: `{}`", escaped_cmd), + Some(b) => format!("❌ Cancelled: `{}`\n\n{}", escaped_cmd, b), + }; + let _ = self.sender.edit_message(&ctx.chat_id, mid, &text).await; + } + SendMode::Minimal => { + // Delete the minimal message + let _ = self.sender.delete_message(&ctx.chat_id, mid).await; + } + // Silent mode sends no message; nothing to clean up. + SendMode::Silent => {} + } + } "⚠️ User cancelled the command".to_string() } else if let Some(code) = exit_code { - let (icon, label) = if code == 0 { - ("✅", "Completed") - } else { - ("❌", "Failed") - }; - let body = format_body(&output_buffer, "Command completed with no output."); - let text = format!( - "{} {}: `{}`\n\n{}", - icon, - label, - escaped_cmd, - body.unwrap_or_default() - ); - let _ = self.sender.edit_message(&ctx.chat_id, &msg_id, &text).await; + if let Some(mid) = &msg_id { + match send_mode { + SendMode::Verbose => { + let (icon, label) = if code == 0 { + ("✅", "Completed") + } else { + ("❌", "Failed") + }; + let body = format_body(&output_buffer, "Command completed with no output."); + let text = format!( + "{} {}: `{}`\n\n{}", + icon, + label, + escaped_cmd, + body.unwrap_or_default() + ); + let _ = self.sender.edit_message(&ctx.chat_id, mid, &text).await; + } + SendMode::Minimal => { + // Delete the minimal message + let _ = self.sender.delete_message(&ctx.chat_id, mid).await; + } + // Silent mode sends no message; nothing to clean up. + SendMode::Silent => {} + } + } let mut result = String::new(); if !output_buffer.is_empty() { result.push_str(output_buffer.trim_end()); diff --git a/src/main.rs b/src/main.rs index d25a8bd..0941b30 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ use rustfox::scheduler::tasks::register_builtin_tasks; use rustfox::scheduler::Scheduler; use rustfox::setup; use rustfox::skills::loader::load_skills_from_dir; +use rustfox::tool_registry::ToolUiMode; #[tokio::main] async fn main() -> Result<()> { @@ -282,7 +283,10 @@ async fn main() -> Result<()> { tracing::warn!("Failed to persist scheduled task run record: {}", e); } - let response = match agent.process_message(&req.incoming, None, None).await { + let response = match agent + .process_message(&req.incoming, None, None, ToolUiMode::Minimal) + .await + { Ok(r) => { if let Err(e) = req .task_store diff --git a/src/platform/sender.rs b/src/platform/sender.rs index 977a6f8..83039bc 100644 --- a/src/platform/sender.rs +++ b/src/platform/sender.rs @@ -45,5 +45,7 @@ pub trait PlatformSender: Send + Sync { text: &str, ) -> Result<()>; + async fn delete_message(&self, chat_id: &str, message_id: &PlatformMessageId) -> Result<()>; + async fn notify_shutdown(&self, chat_id: &str) -> Result<()>; } diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index 6b1358d..99868a1 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -17,6 +17,7 @@ use crate::platform::sender::{ }; use crate::platform::{Attachment, AttachmentKind, IncomingMessage}; use crate::provider::Provider; +use crate::tool_registry::ToolUiMode; use crate::utils::markdown_entities::{markdown_to_entities, split_entities}; use crate::utils::rich_sender; use crate::utils::telegram_markdown::escape_text; @@ -375,8 +376,35 @@ pub async fn send_markdown_message( } } -fn is_verbose_enabled(value: Option<&str>) -> bool { - value.map(|v| v == "true").unwrap_or(false) +/// Read the tool UI mode for a user, with backward compatibility for the old +/// `tool_ui_enabled_{user_id}` boolean key. +async fn read_tool_ui_mode(agent: &Agent, user_id: &str) -> ToolUiMode { + // Try new key first + let new_key = format!("tool_ui_mode_{}", user_id); + match agent.memory.recall("settings", &new_key).await { + Ok(Some(val)) => return ToolUiMode::from_memory(Some(&val)), + Ok(None) => {} + Err(e) => tracing::warn!(error = %e, "Failed to recall tool UI mode"), + } + // Fallback: migrate from old boolean key. Only persist when the old key + // actually exists, so the default (no key) stays a live decision. + let old_key = format!("tool_ui_enabled_{}", user_id); + match agent.memory.recall("settings", &old_key).await { + Ok(Some(old_val)) => { + let mode = ToolUiMode::from_memory(Some(&old_val)); + agent + .memory + .remember("settings", &new_key, mode.as_str(), None) + .await + .ok(); + mode + } + Ok(None) => ToolUiMode::Minimal, + Err(e) => { + tracing::warn!(error = %e, "Failed to recall legacy tool UI setting"); + ToolUiMode::Minimal + } + } } /// Show models for a selected provider, or prompt for text search. @@ -870,29 +898,20 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe } if text == "/verbose" { - let current = agent - .memory - .recall("settings", &format!("tool_ui_enabled_{}", user_id)) - .await - .unwrap_or(None); - let currently_on = is_verbose_enabled(current.as_deref()); - let new_value = if currently_on { "false" } else { "true" }; + let current = read_tool_ui_mode(&agent, &user_id.to_string()).await; + let new_mode = current.next(); agent .memory .remember( "settings", - &format!("tool_ui_enabled_{}", user_id), - new_value, + &format!("tool_ui_mode_{}", user_id), + new_mode.as_str(), None, ) .await .ok(); - let reply = if new_value == "true" { - "🔧 **Tool call UI enabled.** I'll show you what I'm working on." - } else { - "🔇 **Tool call UI disabled.** I'll respond silently." - }; - return send_markdown_message(&bot, msg.chat.id, reply, msg_format).await; + return send_markdown_message(&bot, msg.chat.id, new_mode.reply_message(), msg_format) + .await; } // Accept both the canonical `/queryrewrite` (registered with Telegram — @@ -1319,30 +1338,26 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe .await .ok(); - // Check if verbose tool UI is enabled for this user - let verbose_setting = agent - .memory - .recall("settings", &format!("tool_ui_enabled_{}", user_id)) - .await - .unwrap_or(None); - let verbose_enabled = is_verbose_enabled(verbose_setting.as_deref()); + // Check tool UI mode for this user + let tool_ui_mode = read_tool_ui_mode(&agent, &user_id.to_string()).await; - // Set up tool event channel if verbose is on - let (tool_event_tx, tool_event_rx) = if verbose_enabled { + // Set up tool event channel if not silent + let (tool_event_tx, tool_event_rx) = if tool_ui_mode != ToolUiMode::Silent { let (tx, rx) = tokio::sync::mpsc::channel::(32); (Some(tx), Some(rx)) } else { (None, None) }; - // Spawn notifier task if verbose - let notifier_handle = if verbose_enabled { + // Spawn notifier task if not silent + let notifier_handle = if tool_ui_mode != ToolUiMode::Silent { let bot_clone = bot.clone(); let chat_id = msg.chat.id; - let mut rx = tool_event_rx.expect("rx exists when verbose"); + let mut rx = tool_event_rx.expect("rx exists when not silent"); + let mode = tool_ui_mode; Some(tokio::spawn(async move { let mut notifier = - crate::platform::tool_notifier::ToolCallNotifier::new(bot_clone, chat_id); + crate::platform::tool_notifier::ToolCallNotifier::new(bot_clone, chat_id, mode); notifier.start().await; let mut handled_finished = false; while let Some(event) = rx.recv().await { @@ -1365,24 +1380,25 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe None }; - // When verbose is OFF, send a transient "Thinking..." placeholder so the user + // When silent, send a transient "Thinking..." placeholder so the user // knows the bot is processing. The placeholder is **independent** of the // streaming output — when the first token arrives it is delivered as a NEW // message, and the placeholder is deleted by `handle_message` after the // stream completes (success or error). This keeps the placeholder a // standalone progress signal rather than a doomed attempt to morph into the // final answer. - let placeholder_msg_id: Option = if !verbose_enabled { - match bot.send_message(msg.chat.id, "⏳ Thinking...").await { - Ok(sent) => Some(sent.id), - Err(e) => { - tracing::warn!(error = %e, "Failed to send thinking placeholder"); - None + let placeholder_msg_id: Option = + if tool_ui_mode == ToolUiMode::Silent { + match bot.send_message(msg.chat.id, "⏳ Thinking...").await { + Ok(sent) => Some(sent.id), + Err(e) => { + tracing::warn!(error = %e, "Failed to send thinking placeholder"); + None + } } - } - } else { - None - }; + } else { + None + }; // Streaming: set up token channel for progressive message display // Split threshold: use UTF-16 code units (Telegram's limit is 4096). @@ -1555,7 +1571,12 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe // Finished event after processing completes. let agent_tool_event_tx = tool_event_tx.clone(); let process_result = match agent - .process_message(&incoming, tool_event_tx, Some(stream_token_tx)) + .process_message( + &incoming, + tool_event_tx, + Some(stream_token_tx), + tool_ui_mode, + ) .await { Ok(text) => Ok(text), @@ -1928,6 +1949,21 @@ impl PlatformSender for TelegramAdapter { Ok(()) } + async fn delete_message( + &self, + chat_id_str: &str, + message_id: &PlatformMessageId, + ) -> Result<()> { + let chat_id = parse_chat_id(chat_id_str)?; + let parts: Vec<&str> = message_id.split(':').collect(); + let Some(msg_id_str) = parts.get(1) else { + anyhow::bail!("invalid message id format: {message_id}"); + }; + let msg_id: i32 = msg_id_str.parse()?; + self.bot.delete_message(chat_id, MessageId(msg_id)).await?; + Ok(()) + } + async fn notify_shutdown(&self, chat_id_str: &str) -> Result<()> { let chat_id = parse_chat_id(chat_id_str)?; self.bot @@ -1951,10 +1987,35 @@ mod tests { } #[test] - fn test_is_verbose_enabled_parses_true() { - assert!(is_verbose_enabled(Some("true"))); - assert!(!is_verbose_enabled(Some("false"))); - assert!(!is_verbose_enabled(None)); + fn test_tool_ui_mode_from_memory() { + use crate::tool_registry::ToolUiMode; + assert_eq!( + ToolUiMode::from_memory(Some("verbose")), + ToolUiMode::Verbose + ); + assert_eq!( + ToolUiMode::from_memory(Some("minimal")), + ToolUiMode::Minimal + ); + assert_eq!(ToolUiMode::from_memory(Some("silent")), ToolUiMode::Silent); + // backward compat + assert_eq!(ToolUiMode::from_memory(Some("true")), ToolUiMode::Verbose); + // "false" meant no tool UI at all → Silent, not Minimal + assert_eq!(ToolUiMode::from_memory(Some("false")), ToolUiMode::Silent); + assert_eq!(ToolUiMode::from_memory(None), ToolUiMode::Minimal); + // unknown defaults to minimal + assert_eq!( + ToolUiMode::from_memory(Some("unknown")), + ToolUiMode::Minimal + ); + } + + #[test] + fn test_tool_ui_mode_cycle() { + use crate::tool_registry::ToolUiMode; + assert_eq!(ToolUiMode::Minimal.next(), ToolUiMode::Verbose); + assert_eq!(ToolUiMode::Verbose.next(), ToolUiMode::Silent); + assert_eq!(ToolUiMode::Silent.next(), ToolUiMode::Minimal); } #[test] diff --git a/src/platform/tool_notifier.rs b/src/platform/tool_notifier.rs index 048cba8..181caf5 100644 --- a/src/platform/tool_notifier.rs +++ b/src/platform/tool_notifier.rs @@ -9,6 +9,8 @@ const MAX_DISPLAY_FIELD_CHARS: usize = 60; use teloxide::{prelude::*, types::Message}; use tracing::{debug, warn}; +use crate::tool_registry::ToolUiMode; + /// Events emitted by the agent during tool execution. #[derive(Debug, Clone)] #[allow(dead_code)] @@ -166,15 +168,15 @@ impl ToolDisplayState { self.plan.is_some() || !self.activities.is_empty() } - fn format_live(&self) -> String { - self.format("⏳ Working on your request") + fn format_live(&self, mode: ToolUiMode) -> String { + self.format("⏳ Working on your request", mode) } #[allow(dead_code)] - fn format_completed(&self) -> String { + fn format_completed(&self, mode: ToolUiMode) -> String { // Default successful header and result. Caller may adjust based on overall // request success vs failure when rendering the final card. - self.format("✅ Completed") + self.format("✅ Completed", mode) } fn apply_plan_update(&mut self, update: PlanStepUpdate) { @@ -186,7 +188,7 @@ impl ToolDisplayState { } } - fn format(&self, header: &str) -> String { + fn format(&self, header: &str, mode: ToolUiMode) -> String { let mut text = header.to_string(); if let Some(plan) = &self.plan { @@ -221,7 +223,8 @@ impl ToolDisplayState { for activity in &self.activities { text.push('\n'); text.push_str(&friendly_tool_name(&activity.name)); - if !activity.args_preview.is_empty() { + // In verbose mode, show args_preview; in minimal, hide it + if mode == ToolUiMode::Verbose && !activity.args_preview.is_empty() { text.push_str(": "); text.push_str(&crate::utils::strings::truncate_chars( &activity.args_preview, @@ -493,16 +496,18 @@ pub struct ToolCallNotifier { status_msg: Option, display_state: ToolDisplayState, last_edit: Option, + mode: ToolUiMode, } impl ToolCallNotifier { - pub fn new(bot: Bot, chat_id: ChatId) -> Self { + pub fn new(bot: Bot, chat_id: ChatId, mode: ToolUiMode) -> Self { Self { bot, chat_id, status_msg: None, display_state: ToolDisplayState::default(), last_edit: None, + mode, } } @@ -563,7 +568,7 @@ impl ToolCallNotifier { } fn format_status(&self) -> String { - self.display_state.format_live() + self.display_state.format_live(self.mode) } fn final_status_text(&self, success: bool) -> Option { @@ -573,7 +578,7 @@ impl ToolCallNotifier { } else { "⛔ Stopped" }; - let mut text = self.display_state.format(header); + let mut text = self.display_state.format(header, self.mode); if success { text.push_str("\n\nResult\nFinal answer sent below."); @@ -697,7 +702,7 @@ mod tests { title: "Long Plan Title".to_string(), steps, }); - let formatted = s.format_completed(); + let formatted = s.format_completed(ToolUiMode::Verbose); // Should always be under Telegram's safe limit (we clamp elsewhere to MAX_STATUS_TEXT_CHARS=3800) assert!( formatted.chars().count() <= 4000, @@ -708,7 +713,8 @@ mod tests { #[test] fn test_notifier_final_status_text_returns_completed_card_when_activity_exists() { - let mut notifier = ToolCallNotifier::new(Bot::new("TEST_TOKEN"), ChatId(1)); + let mut notifier = + ToolCallNotifier::new(Bot::new("TEST_TOKEN"), ChatId(1), ToolUiMode::Verbose); notifier.display_state.handle_event(ToolEvent::Started { name: "read_file".to_string(), args_preview: "/tmp/file.txt".to_string(), @@ -737,7 +743,8 @@ mod tests { #[test] fn test_notifier_final_status_text_is_none_without_activity() { - let notifier = ToolCallNotifier::new(Bot::new("TEST_TOKEN"), ChatId(1)); + let notifier = + ToolCallNotifier::new(Bot::new("TEST_TOKEN"), ChatId(1), ToolUiMode::Minimal); assert!(notifier.final_status_text(true).is_none()); } @@ -753,7 +760,7 @@ mod tests { .to_string(), }); - let text = state.format_live(); + let text = state.format_live(ToolUiMode::Verbose); assert!( text.contains("Working on your request"), "live header missing: {text}" @@ -788,7 +795,7 @@ mod tests { arguments_json: r#"{"step_id":1,"status":"in_progress","notes":"working"}"#.to_string(), }); - let text = state.format_live(); + let text = state.format_live(ToolUiMode::Verbose); assert!( text.contains("[ ] 0. First"), "unchanged step missing: {text}" @@ -812,7 +819,7 @@ mod tests { args_preview: "step 0".to_string(), arguments_json: r#"{"step_id":0,"status":"done","notes":"token=secret"}"#.to_string(), }); - let text = state.format_completed(); + let text = state.format_completed(ToolUiMode::Verbose); assert!(text.contains("[x] 0. First"), "done step missing: {text}"); assert!( !text.contains("token=secret"), @@ -822,7 +829,8 @@ mod tests { #[test] fn test_notifier_final_status_text_reports_failed_request() { - let mut notifier = ToolCallNotifier::new(Bot::new("TEST_TOKEN"), ChatId(1)); + let mut notifier = + ToolCallNotifier::new(Bot::new("TEST_TOKEN"), ChatId(1), ToolUiMode::Verbose); notifier.display_state.handle_event(ToolEvent::Started { name: "read_file".to_string(), args_preview: "/tmp/file.txt".to_string(), @@ -865,7 +873,7 @@ mod tests { success: false, }); - let text = state.format_live(); + let text = state.format_live(ToolUiMode::Verbose); assert!( text.contains("[!] 0. Only step"), "failed step missing: {text}" @@ -900,7 +908,7 @@ mod tests { success: false, }); - let text = state.format_live(); + let text = state.format_live(ToolUiMode::Verbose); assert!( text.contains("[x] 1. Second"), "valid completed step changed unexpectedly: {text}" @@ -925,7 +933,7 @@ mod tests { success: true, }); - let text = state.format_completed(); + let text = state.format_completed(ToolUiMode::Verbose); assert!( text.contains("Completed"), "completed header missing: {text}" diff --git a/src/supervisor/backend/reasoning.rs b/src/supervisor/backend/reasoning.rs index 8e86dab..19503c0 100644 --- a/src/supervisor/backend/reasoning.rs +++ b/src/supervisor/backend/reasoning.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use crate::supervisor::backend::{Backend, BackendCapabilities, RunContext}; use crate::supervisor::job::{Evidence, Job, JobOutput, JobStatus, JobType}; +use crate::tool_registry::ToolUiMode; type ExecFn = Arc Pin> + Send>> + Send + Sync>; @@ -34,7 +35,7 @@ impl ReasoningBackend { attachments: Vec::new(), }; agent - .process_message(&incoming, None, None) + .process_message(&incoming, None, None, ToolUiMode::Minimal) .await .map_err(|e| anyhow!("agent failed: {e:#}")) }) diff --git a/src/tool_registry.rs b/src/tool_registry.rs index dd6c4a8..7b80e82 100644 --- a/src/tool_registry.rs +++ b/src/tool_registry.rs @@ -10,6 +10,58 @@ use crate::platform::sender::PlatformSender; pub type ToolResult = Result; +/// Controls how tool execution progress is displayed to the user. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ToolUiMode { + /// No tool progress messages at all (just "⏳ Thinking..." placeholder). + Silent, + /// Show tool names + status, but hide args and command output. + Minimal, + /// Show everything: tool names, args, live command output, results. + Verbose, +} + +impl ToolUiMode { + /// Parse from stored memory value. Handles legacy boolean key. + pub fn from_memory(s: Option<&str>) -> Self { + match s { + Some("verbose") => Self::Verbose, + Some("minimal") => Self::Minimal, + Some("silent") => Self::Silent, + // backward compat: old "true"/"false" key. "false" meant no tool UI + // at all (placeholder only) → Silent. Absent key → new default Minimal. + Some("true") => Self::Verbose, + Some("false") => Self::Silent, + None => Self::Minimal, + _ => Self::Minimal, + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Silent => "silent", + Self::Minimal => "minimal", + Self::Verbose => "verbose", + } + } + + pub fn next(&self) -> Self { + match self { + Self::Minimal => Self::Verbose, + Self::Verbose => Self::Silent, + Self::Silent => Self::Minimal, + } + } + + pub fn reply_message(&self) -> &'static str { + match self { + Self::Silent => "🔇 **Tool UI silent.** No progress messages.", + Self::Minimal => "🔧 **Tool UI minimal.** Tool names + status, no details.", + Self::Verbose => "🔧 **Tool UI verbose.** Full tool call details.", + } + } +} + pub struct ToolContext { pub sandbox_dir: PathBuf, pub home_dir: Option, @@ -17,6 +69,7 @@ pub struct ToolContext { pub cancel_registry: Arc, pub user_id: String, pub chat_id: String, + pub tool_ui_mode: ToolUiMode, } #[async_trait] @@ -108,6 +161,7 @@ mod tests { cancel_registry: Arc::new(CancelRegistry::new()), user_id: "test".to_string(), chat_id: "0".to_string(), + tool_ui_mode: ToolUiMode::Minimal, }; let result = reg.execute("mock_tool", json!({}), ctx).await.unwrap(); assert_eq!(result, "executed mock_tool"); @@ -123,6 +177,7 @@ mod tests { cancel_registry: Arc::new(CancelRegistry::new()), user_id: "test".to_string(), chat_id: "0".to_string(), + tool_ui_mode: ToolUiMode::Minimal, }; let result = reg.execute("unknown", json!({}), ctx).await; assert!(result.is_err()); @@ -165,6 +220,13 @@ mod tests { ) -> Result<()> { Ok(()) } + async fn delete_message( + &self, + _chat_id: &str, + _message_id: &PlatformMessageId, + ) -> Result<()> { + Ok(()) + } async fn notify_shutdown(&self, _chat_id: &str) -> Result<()> { Ok(()) } From 9bfa32d27a6ff39851bea9464f4542c3eda0fbd8 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Tue, 4 Aug 2026 23:43:54 +0800 Subject: [PATCH 2/7] chore(deps): update transitive crate versions --- Cargo.lock | 802 ++++++++++++++++++++++++----------------------------- 1 file changed, 364 insertions(+), 438 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d6dcc5..169a013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -48,9 +48,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "aquamarine" @@ -63,14 +63,14 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "ar_archive_writer" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" dependencies = [ "object", ] @@ -86,13 +86,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -179,9 +179,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.12.1" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block-buffer" @@ -224,9 +224,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "byteorder" @@ -242,9 +242,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cbc" @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.63" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "shlex", @@ -290,15 +290,15 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -335,9 +335,9 @@ dependencies = [ [[package]] name = "console" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", @@ -435,9 +435,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -445,18 +445,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -519,7 +519,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -542,7 +542,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] @@ -553,7 +553,38 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.19", ] [[package]] @@ -568,9 +599,9 @@ dependencies = [ [[package]] name = "der" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ "pem-rfc7468", "zeroize", @@ -582,7 +613,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -594,7 +624,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -614,7 +644,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "unicode-xid", ] @@ -651,13 +681,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -671,17 +701,19 @@ dependencies = [ [[package]] name = "docx-rs" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed73cbf5e1c37baa23f4132569ac1187829f03922c206bd68fe109e3001a343d" +checksum = "7fdf00e8af6d0b3e92d4bbf9b76f773d8b84ea80f310324ad16cbdc2e653e02c" dependencies = [ "base64", + "crc32fast", "image", - "quick-xml 0.36.2", + "quick-xml 0.41.0", "serde", "serde_json", - "thiserror 2.0.18", - "zip 0.6.6", + "smallvec", + "thiserror 2.0.19", + "zip 8.6.0", ] [[package]] @@ -736,9 +768,9 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "encode_unicode" @@ -804,9 +836,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "fax" @@ -863,6 +895,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -903,9 +936,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -918,9 +951,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -928,15 +961,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -945,38 +978,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -1033,15 +1066,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "wasip2", - "wasip3", ] [[package]] @@ -1056,9 +1087,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -1134,9 +1165,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1144,9 +1175,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1154,9 +1185,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1179,9 +1210,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -1361,12 +1392,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1467,9 +1492,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.18.4" +version = "0.18.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ "console", "portable-atomic", @@ -1499,9 +1524,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "itertools" @@ -1520,10 +1545,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.28" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "log", "portable-atomic", @@ -1531,26 +1558,35 @@ dependencies = [ "serde_core", ] +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + [[package]] name = "jiff-static" -version = "0.2.28" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1560,23 +1596,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libredox" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" dependencies = [ "libc", ] @@ -1621,9 +1651,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.31" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lopdf" @@ -1632,7 +1662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7184fdea2bc3cd272a1acec4030c321a8f9875e877b3f92a53f2f6033fdc289" dependencies = [ "aes", - "bitflags 2.12.1", + "bitflags 2.13.1", "cbc", "ecb", "encoding_rs", @@ -1644,11 +1674,11 @@ dependencies = [ "md-5", "nom", "nom_locate", - "rand 0.9.4", + "rand 0.9.5", "rangemap", "sha2", "stringprep", - "thiserror 2.0.18", + "thiserror 2.0.19", "ttf-parser", "weezl", ] @@ -1680,9 +1710,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" @@ -1712,9 +1742,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -1754,7 +1784,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -1803,7 +1833,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1827,9 +1857,9 @@ dependencies = [ [[package]] name = "object" -version = "0.37.3" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "memchr", ] @@ -1845,7 +1875,7 @@ dependencies = [ "rten", "rten-imageproc", "rten-tensor", - "thiserror 2.0.18", + "thiserror 2.0.19", "wasm-bindgen", ] @@ -1857,11 +1887,11 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl" -version = "0.10.80" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -1877,7 +1907,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1888,9 +1918,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.116" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -1982,7 +2012,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2013,7 +2043,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "crc32fast", "fdeflate", "flate2", @@ -2028,9 +2058,9 @@ checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -2071,16 +2101,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -2104,9 +2124,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -2127,9 +2147,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" dependencies = [ "ar_archive_writer", "cc", @@ -2141,7 +2161,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "getopts", "memchr", "pulldown-cmark-escape", @@ -2156,9 +2176,9 @@ checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "pxfm" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" [[package]] name = "quick-error" @@ -2168,28 +2188,28 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ - "encoding_rs", "memchr", ] [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ + "encoding_rs", "memchr", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -2208,9 +2228,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2219,9 +2239,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2306,7 +2326,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", ] [[package]] @@ -2322,29 +2342,29 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2354,9 +2374,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2365,9 +2385,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -2491,11 +2511,11 @@ dependencies = [ "process-wrap", "reqwest 0.12.28", "rmcp-macros", - "schemars 1.2.1", + "schemars 1.2.2", "serde", "serde_json", "sse-stream", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -2512,7 +2532,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn", + "syn 2.0.119", ] [[package]] @@ -2628,7 +2648,7 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "chrono", "csv", "fallible-iterator", @@ -2678,7 +2698,7 @@ dependencies = [ "ocrs", "pdf-extract", "pulldown-cmark", - "rand 0.8.6", + "rand 0.8.7", "regex", "reqwest 0.12.28", "rmcp", @@ -2707,7 +2727,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -2716,9 +2736,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "once_cell", "rustls-pki-types", @@ -2729,9 +2749,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "zeroize", ] @@ -2749,9 +2769,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -2782,9 +2802,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "chrono", "dyn-clone", @@ -2796,14 +2816,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 3.0.3", ] [[package]] @@ -2818,7 +2838,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2880,9 +2900,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2890,40 +2910,40 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_derive_internals" -version = "0.29.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -2966,9 +2986,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64", "bs58", @@ -2977,7 +2997,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.14.0", "schemars 0.9.0", - "schemars 1.2.1", + "schemars 1.2.2", "serde_core", "serde_json", "serde_with_macros", @@ -2986,14 +3006,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3044,9 +3064,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "slab" @@ -3056,15 +3076,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3102,9 +3122,9 @@ dependencies = [ [[package]] name = "sse-stream" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3962b63f038885f15bce2c6e02c0e7925c072f1ac86bb60fd44c5c6b762fb72" +checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" dependencies = [ "bytes", "futures-util", @@ -3121,9 +3141,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", @@ -3157,9 +3177,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -3183,7 +3214,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3192,7 +3223,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -3215,9 +3246,9 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] name = "takecell" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20f34339676cdcab560c9a82300c4c2581f68b9369aedf0fae86f2ff9565ff3e" +checksum = "07dd1d452d2c3dc94a4e1c5c3c9a3cc88c2ef5926674b75881e454c4dc3a14c4" [[package]] name = "tar" @@ -3249,7 +3280,7 @@ dependencies = [ "serde_json", "teloxide-core", "teloxide-macros", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -3262,7 +3293,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f7a34ca8e971fa892e633858c07547fe138ef4a02e4a4eaa1d35e517d6e0bc4" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "bytes", "chrono", "derive_more", @@ -3281,7 +3312,7 @@ dependencies = [ "stacker", "take_mut", "takecell", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "url", @@ -3297,7 +3328,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3307,7 +3338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3324,11 +3355,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.19", ] [[package]] @@ -3339,25 +3370,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -3378,12 +3409,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -3393,15 +3423,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -3419,9 +3449,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -3434,9 +3464,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -3466,13 +3496,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -3497,9 +3527,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -3508,13 +3538,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -3582,7 +3613,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -3626,7 +3657,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3689,6 +3720,12 @@ dependencies = [ "pom", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typeid" version = "1.0.3" @@ -3766,7 +3803,7 @@ checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" dependencies = [ "base64", "cookie_store", - "der 0.8.0", + "der 0.8.1", "encoding_rs", "flate2", "log", @@ -3826,11 +3863,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.2" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "wasm-bindgen", ] @@ -3870,27 +3907,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -3901,9 +3929,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -3911,9 +3939,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3921,48 +3949,26 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.4.2" @@ -3976,23 +3982,11 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.12.1", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -4010,9 +4004,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" dependencies = [ "rustls-pki-types", ] @@ -4098,7 +4092,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4109,7 +4103,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4323,100 +4317,12 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.12.1", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" version = "0.6.3" @@ -4452,28 +4358,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4493,15 +4399,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" @@ -4533,32 +4439,34 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "zip" -version = "0.6.6" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" dependencies = [ - "byteorder", + "arbitrary", "crc32fast", - "crossbeam-utils", - "flate2", + "indexmap 2.14.0", + "memchr", + "time", ] [[package]] name = "zip" -version = "6.0.0" +version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ - "arbitrary", "crc32fast", + "flate2", "indexmap 2.14.0", "memchr", - "time", + "typed-path", + "zopfli", ] [[package]] @@ -4569,14 +4477,32 @@ checksum = "dba6063ff82cdbd9a765add16d369abe81e520f836054e997c2db217ceca40c0" dependencies = [ "base64", "ed25519-dalek", - "thiserror 2.0.18", + "thiserror 2.0.19", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] [[package]] name = "zune-core" From 0abefec1a577392cd9e95384d24ffaa5c45caf26 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Tue, 4 Aug 2026 23:43:54 +0800 Subject: [PATCH 3/7] chore: add caveman skills bundle --- .agents/skills/caveman-commit/README.md | 44 ++ .agents/skills/caveman-commit/SKILL.md | 65 +++ .agents/skills/caveman-compress/README.md | 163 +++++++ .agents/skills/caveman-compress/SECURITY.md | 31 ++ .agents/skills/caveman-compress/SKILL.md | 111 +++++ .../caveman-compress/scripts/__init__.py | 9 + .../caveman-compress/scripts/__main__.py | 3 + .../caveman-compress/scripts/benchmark.py | 80 ++++ .../skills/caveman-compress/scripts/cli.py | 85 ++++ .../caveman-compress/scripts/compress.py | 414 ++++++++++++++++++ .../skills/caveman-compress/scripts/detect.py | 139 ++++++ .../caveman-compress/scripts/validate.py | 221 ++++++++++ .agents/skills/caveman-help/README.md | 38 ++ .agents/skills/caveman-help/SKILL.md | 63 +++ .agents/skills/caveman-review/README.md | 33 ++ .agents/skills/caveman-review/SKILL.md | 55 +++ .agents/skills/caveman-stats/README.md | 36 ++ .agents/skills/caveman-stats/SKILL.md | 12 + .agents/skills/caveman/README.md | 48 ++ .agents/skills/caveman/SKILL.md | 88 ++++ skills-lock.json | 42 ++ 21 files changed, 1780 insertions(+) create mode 100644 .agents/skills/caveman-commit/README.md create mode 100644 .agents/skills/caveman-commit/SKILL.md create mode 100644 .agents/skills/caveman-compress/README.md create mode 100644 .agents/skills/caveman-compress/SECURITY.md create mode 100644 .agents/skills/caveman-compress/SKILL.md create mode 100644 .agents/skills/caveman-compress/scripts/__init__.py create mode 100644 .agents/skills/caveman-compress/scripts/__main__.py create mode 100644 .agents/skills/caveman-compress/scripts/benchmark.py create mode 100644 .agents/skills/caveman-compress/scripts/cli.py create mode 100644 .agents/skills/caveman-compress/scripts/compress.py create mode 100644 .agents/skills/caveman-compress/scripts/detect.py create mode 100644 .agents/skills/caveman-compress/scripts/validate.py create mode 100644 .agents/skills/caveman-help/README.md create mode 100644 .agents/skills/caveman-help/SKILL.md create mode 100644 .agents/skills/caveman-review/README.md create mode 100644 .agents/skills/caveman-review/SKILL.md create mode 100644 .agents/skills/caveman-stats/README.md create mode 100644 .agents/skills/caveman-stats/SKILL.md create mode 100644 .agents/skills/caveman/README.md create mode 100644 .agents/skills/caveman/SKILL.md diff --git a/.agents/skills/caveman-commit/README.md b/.agents/skills/caveman-commit/README.md new file mode 100644 index 0000000..d5aee01 --- /dev/null +++ b/.agents/skills/caveman-commit/README.md @@ -0,0 +1,44 @@ +# caveman-commit + +Terse Conventional Commits. Why over what. + +## What it does + +Generates commit messages in Conventional Commits format. Subject ≤50 chars, hard cap 72. Imperative mood. Body only when the *why* is non-obvious or there are breaking changes. No AI attribution, no "this commit does X", no emoji unless the project uses them. Body always required for breaking changes, security fixes, data migrations, and reverts — future debuggers need the context. + +Outputs only the message. Does not stage, commit, or amend. + +## How to invoke + +``` +/caveman-commit +``` + +Also triggers on phrases like "write a commit", "commit message", "generate commit". + +## Example output + +Diff: new endpoint for user profile. + +``` +feat(api): add GET /users/:id/profile + +Mobile client needs profile data without the full user payload +to reduce LTE bandwidth on cold-launch screens. + +Closes #128 +``` + +Diff: breaking API rename. + +``` +feat(api)!: rename /v1/orders to /v1/checkout + +BREAKING CHANGE: clients on /v1/orders must migrate to /v1/checkout +before 2026-06-01. Old route returns 410 after that date. +``` + +## See also + +- [`SKILL.md`](./SKILL.md) — full LLM-facing instructions +- [Caveman README](../../README.md) — repo overview diff --git a/.agents/skills/caveman-commit/SKILL.md b/.agents/skills/caveman-commit/SKILL.md new file mode 100644 index 0000000..b9999e3 --- /dev/null +++ b/.agents/skills/caveman-commit/SKILL.md @@ -0,0 +1,65 @@ +--- +name: caveman-commit +description: > + Ultra-compressed commit message generator. Cuts noise from commit messages while preserving + intent and reasoning. Conventional Commits format. Subject ≤50 chars, body only when "why" + isn't obvious. Use when user says "write a commit", "commit message", "generate commit", + "/commit", or invokes /caveman-commit. Auto-triggers when staging changes. +--- + +Write commit messages terse and exact. Conventional Commits format. No fluff. Why over what. + +## Rules + +**Subject line:** +- `(): ` — `` optional +- Types: `feat`, `fix`, `refactor`, `perf`, `docs`, `test`, `chore`, `build`, `ci`, `style`, `revert` +- Imperative mood: "add", "fix", "remove" — not "added", "adds", "adding" +- ≤50 chars when possible, hard cap 72 +- No trailing period +- Match project convention for capitalization after the colon + +**Body (only if needed):** +- Skip entirely when subject is self-explanatory +- Add body only for: non-obvious *why*, breaking changes, migration notes, linked issues +- Wrap at 72 chars +- Bullets `-` not `*` +- Reference issues/PRs at end: `Closes #42`, `Refs #17` + +**What NEVER goes in:** +- "This commit does X", "I", "we", "now", "currently" — the diff says what +- "As requested by..." — use Co-authored-by trailer +- "Generated with Claude Code" or any AI attribution — unless the user's own rule requires an `Assisted-by`/AI-attribution trailer, then add it as a trailer +- Emoji (unless project convention requires) +- Restating the file name when scope already says it + +## Examples + +Diff: new endpoint for user profile with body explaining the why +- ❌ "feat: add a new endpoint to get user profile information from the database" +- ✅ + ``` + feat(api): add GET /users/:id/profile + + Mobile client needs profile data without the full user payload + to reduce LTE bandwidth on cold-launch screens. + + Closes #128 + ``` + +Diff: breaking API change +- ✅ + ``` + feat(api)!: rename /v1/orders to /v1/checkout + + BREAKING CHANGE: clients on /v1/orders must migrate to /v1/checkout + before 2026-06-01. Old route returns 410 after that date. + ``` + +## Auto-Clarity + +Always include body for: breaking changes, security fixes, data migrations, anything reverting a prior commit. Never compress these into subject-only — future debuggers need the context. + +## Boundaries + +Only generates the commit message. Does not run `git commit`, does not stage files, does not amend. Output the message as a code block ready to paste. "stop caveman-commit" or "normal mode": revert to verbose commit style. diff --git a/.agents/skills/caveman-compress/README.md b/.agents/skills/caveman-compress/README.md new file mode 100644 index 0000000..4aa149a --- /dev/null +++ b/.agents/skills/caveman-compress/README.md @@ -0,0 +1,163 @@ +

+ +

+ +

caveman-compress

+ +

+ shrink memory file. save token every session. +

+ +--- + +A Claude Code skill that compresses your project memory files (`CLAUDE.md`, todos, preferences) into caveman format — so every session loads fewer tokens automatically. + +Claude read `CLAUDE.md` on every session start. If file big, cost big. Caveman make file small. Cost go down forever. + +## What It Do + +``` +/caveman-compress CLAUDE.md +``` + +``` +CLAUDE.md ← compressed (Claude reads this — fewer tokens every session) +CLAUDE.original.md ← human-readable backup (you edit this) +``` + +Original never lost. Backup lives in a data dir, not next to your file — `$XDG_DATA_HOME/caveman-compress/backups//` (macOS/Linux) or `%LOCALAPPDATA%\caveman-compress\backups\\` (Windows) — so skill auto-loaders don't re-read it as a live file. You can read and edit `.original.md` there. Run skill again to re-compress after edits. + +## Benchmarks + +Real results on real project files: + +| File | Original | Compressed | Saved | +|------|----------:|----------:|------:| +| `claude-md-preferences.md` | 706 | 285 | **59.6%** | +| `project-notes.md` | 1145 | 535 | **53.3%** | +| `claude-md-project.md` | 1122 | 636 | **43.3%** | +| `todo-list.md` | 627 | 388 | **38.1%** | +| `mixed-with-code.md` | 888 | 560 | **36.9%** | +| **Average** | **898** | **481** | **46%** | + +All validations passed ✅ — headings, code blocks, URLs, file paths preserved exactly. + +## Before / After + + + + + + +
+ +### 📄 Original (706 tokens) + +> "I strongly prefer TypeScript with strict mode enabled for all new code. Please don't use `any` type unless there's genuinely no way around it, and if you do, leave a comment explaining the reasoning. I find that taking the time to properly type things catches a lot of bugs before they ever make it to runtime." + + + +### rock Caveman (285 tokens) + +> "Prefer TypeScript strict mode always. No `any` unless unavoidable — comment why if used. Proper types catch bugs early." + +
+ +**Same instructions. 60% fewer tokens. Every. Single. Session.** + +## Security + +`caveman-compress` is flagged as Snyk High Risk due to subprocess and file I/O patterns detected by static analysis. This is a false positive — see [SECURITY.md](./SECURITY.md) for a full explanation of what the skill does and does not do. + +## Install + +Compress is built in with the `caveman` plugin. Install `caveman` once, then use `/caveman-compress`. + +If you need local files, the compress skill lives at: + +```bash +caveman-compress/ +``` + +**Requires:** Python 3.10+ + +## Usage + +``` +/caveman-compress +``` + +Examples: +``` +/caveman-compress CLAUDE.md +/caveman-compress docs/preferences.md +/caveman-compress todos.md +``` + +### What files work + +| Type | Compress? | +|------|-----------| +| `.md`, `.txt`, `.rst`, `.typ`, `.typst`, `.tex` | ✅ Yes | +| Extensionless natural language | ✅ Yes | +| `.py`, `.js`, `.ts`, `.json`, `.yaml` | ❌ Skip (code/config) | +| `*.original.md` | ❌ Skip (backup files) | + +## How It Work + +``` +/caveman-compress CLAUDE.md + ↓ +detect file type (no tokens) + ↓ +Claude compresses (tokens — one call) + ↓ +validate output (no tokens) + checks: headings, code blocks, URLs, file paths, bullets + ↓ +if errors: Claude fixes cherry-picked issues only (tokens — targeted fix) + does NOT recompress — only patches broken parts + ↓ +retry up to 2 times + ↓ +write compressed → CLAUDE.md +write original → CLAUDE.original.md +``` + +Only two things use tokens: initial compression + targeted fix if validation fails. Everything else is local Python. + +## What Is Preserved + +Caveman compress natural language. It never touch: + +- Code blocks (` ``` ` fenced or indented) +- Inline code (`` `backtick content` ``) +- URLs and links +- File paths (`/src/components/...`) +- Commands (`npm install`, `git commit`) +- Technical terms, library names, API names +- Headings (exact text preserved) +- Tables (structure preserved, cell text compressed) +- Dates, version numbers, numeric values + +## Why This Matter + +`CLAUDE.md` loads on **every session start**. A 1000-token project memory file costs tokens every single time you open a project. Over 100 sessions that's 100,000 tokens of overhead — just for context you already wrote. + +Caveman cut that by ~46% on average. Same instructions. Same accuracy. Less waste. + +``` +┌────────────────────────────────────────────┐ +│ TOKEN SAVINGS PER FILE █████ 46% │ +│ SESSIONS THAT BENEFIT ██████████ 100% │ +│ INFORMATION PRESERVED ██████████ 100% │ +│ SETUP TIME █ 1x │ +└────────────────────────────────────────────┘ +``` + +## Part of Caveman + +This skill is part of the [caveman](https://github.com/JuliusBrussee/caveman) toolkit — making Claude use fewer tokens without losing accuracy. + +- **caveman** — make Claude *speak* like caveman (cuts response tokens ~65%) +- **caveman-compress** — make Claude *read* less (cuts context tokens ~46%) diff --git a/.agents/skills/caveman-compress/SECURITY.md b/.agents/skills/caveman-compress/SECURITY.md new file mode 100644 index 0000000..0efa9fe --- /dev/null +++ b/.agents/skills/caveman-compress/SECURITY.md @@ -0,0 +1,31 @@ +# Security + +## Snyk High Risk Rating + +`caveman-compress` receives a Snyk High Risk rating due to static analysis heuristics. This document explains what the skill does and does not do. + +### What triggers the rating + +1. **subprocess usage**: The skill calls the `claude` CLI via `subprocess.run()` as a fallback when `ANTHROPIC_API_KEY` is not set. The subprocess call uses a fixed argument list — no shell interpolation occurs. User file content is passed via stdin, not as a shell argument. + +2. **File read/write**: The skill reads the file the user explicitly points it at, compresses it, and writes the result back to the same path. A `.original.md` backup is saved to an out-of-tree data dir (`$XDG_DATA_HOME/caveman-compress/backups//`, or `%LOCALAPPDATA%\caveman-compress\backups\\` on Windows). Beyond the target file and that backup location, no files are read or written. + +### What the skill does NOT do + +- Does not execute user file content as code +- Does not make network requests except to Anthropic's API (via SDK or CLI) +- Does not access files outside the path the user provides +- Does not use shell=True or string interpolation in subprocess calls +- Does not collect or transmit any data beyond the file being compressed + +### Auth behavior + +If `ANTHROPIC_API_KEY` is set, the skill uses the Anthropic Python SDK directly (no subprocess). If not set, it falls back to the `claude` CLI, which uses the user's existing Claude desktop authentication. + +### File size limit + +Files larger than 500KB are rejected before any API call is made. + +### Reporting a vulnerability + +If you believe you've found a genuine security issue, please open a GitHub issue with the label `security`. diff --git a/.agents/skills/caveman-compress/SKILL.md b/.agents/skills/caveman-compress/SKILL.md new file mode 100644 index 0000000..0b95aab --- /dev/null +++ b/.agents/skills/caveman-compress/SKILL.md @@ -0,0 +1,111 @@ +--- +name: caveman-compress +description: > + Compress natural language memory files (CLAUDE.md, todos, preferences) into caveman format + to save input tokens. Preserves all technical substance, code, URLs, and structure. + Compressed version overwrites the original file. Human-readable backup saved as FILE.original.md. + Trigger: /caveman-compress FILEPATH or "compress memory file" +--- + +# Caveman Compress + +## Purpose + +Compress natural language files (CLAUDE.md, todos, preferences) into caveman-speak to reduce input tokens. Compressed version overwrites original. Human-readable backup saved as `.original.md`, but NOT beside the source file — it lives in an out-of-tree data dir (`$XDG_DATA_HOME/caveman-compress/backups//`, or `%LOCALAPPDATA%\caveman-compress\backups\\` on Windows) so skill auto-loaders don't re-ingest it as a live file. + +## Trigger + +`/caveman-compress ` or when user asks to compress a memory file. + +## Process + +1. The compression scripts live in `scripts/` (adjacent to this SKILL.md). If the path is not immediately available, search for `scripts/__main__.py` next to this SKILL.md. + +2. From the directory containing this SKILL.md, run: + +python3 -m scripts + +3. The CLI will: +- detect file type (no tokens) +- call Claude to compress +- validate output (no tokens) +- if errors: cherry-pick fix with Claude (targeted fixes only, no recompression) +- retry up to 2 times +- if still failing after 2 retries: report error to user, leave original file untouched + +4. Return result to user + +## Compression Rules + +### Remove +- Articles: a, an, the +- Filler: just, really, basically, actually, simply, essentially, generally +- Pleasantries: "sure", "certainly", "of course", "happy to", "I'd recommend" +- Hedging: "it might be worth", "you could consider", "it would be good to" +- Redundant phrasing: "in order to" → "to", "make sure to" → "ensure", "the reason is because" → "because" +- Connective fluff: "however", "furthermore", "additionally", "in addition" + +### Preserve EXACTLY (never modify) +- Code blocks (fenced ``` and indented) +- Inline code (`backtick content`) +- URLs and links (full URLs, markdown links) +- File paths (`/src/components/...`, `./config.yaml`) +- Commands (`npm install`, `git commit`, `docker build`) +- Technical terms (library names, API names, protocols, algorithms) +- Proper nouns (project names, people, companies) +- Dates, version numbers, numeric values +- Environment variables (`$HOME`, `NODE_ENV`) + +### Preserve Structure +- All markdown headings (keep exact heading text, compress body below) +- Bullet point hierarchy (keep nesting level) +- Numbered lists (keep numbering) +- Tables (compress cell text, keep structure) +- Frontmatter/YAML headers in markdown files + +### Compress +- Use short synonyms: "big" not "extensive", "fix" not "implement a solution for", "use" not "utilize" +- Fragments OK: "Run tests before commit" not "You should always run tests before committing" +- Drop "you should", "make sure to", "remember to" — just state the action +- Merge redundant bullets that say the same thing differently +- Keep one example where multiple examples show the same pattern + +CRITICAL RULE: +Anything inside ``` ... ``` must be copied EXACTLY. +Do not: +- remove comments +- remove spacing +- reorder lines +- shorten commands +- simplify anything + +Inline code (`...`) must be preserved EXACTLY. +Do not modify anything inside backticks. + +If file contains code blocks: +- Treat code blocks as read-only regions +- Only compress text outside them +- Do not merge sections around code + +## Pattern + +Original: +> You should always make sure to run the test suite before pushing any changes to the main branch. This is important because it helps catch bugs early and prevents broken builds from being deployed to production. + +Compressed: +> Run tests before push to main. Catch bugs early, prevent broken prod deploys. + +Original: +> The application uses a microservices architecture with the following components. The API gateway handles all incoming requests and routes them to the appropriate service. The authentication service is responsible for managing user sessions and JWT tokens. + +Compressed: +> Microservices architecture. API gateway route all requests to services. Auth service manage user sessions + JWT tokens. + +## Boundaries + +- ONLY compress natural language files (.md, .txt, .typ, .typst, .tex, extensionless) +- NEVER modify: .py, .js, .ts, .json, .yaml, .yml, .toml, .env, .lock, .css, .html, .xml, .sql, .sh +- If file has mixed content (prose + code), compress ONLY the prose sections +- If unsure whether something is code or prose, leave it unchanged +- Original file is backed up as FILE.original.md before overwriting — in the out-of-tree backup data dir (see Purpose), not beside the source file +- Never compress FILE.original.md (skip it) diff --git a/.agents/skills/caveman-compress/scripts/__init__.py b/.agents/skills/caveman-compress/scripts/__init__.py new file mode 100644 index 0000000..16b8c53 --- /dev/null +++ b/.agents/skills/caveman-compress/scripts/__init__.py @@ -0,0 +1,9 @@ +"""Caveman compress scripts. + +This package provides tools to compress natural language markdown files +into caveman format to save input tokens. +""" + +__all__ = ["cli", "compress", "detect", "validate"] + +__version__ = "1.0.0" diff --git a/.agents/skills/caveman-compress/scripts/__main__.py b/.agents/skills/caveman-compress/scripts/__main__.py new file mode 100644 index 0000000..4e28416 --- /dev/null +++ b/.agents/skills/caveman-compress/scripts/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/.agents/skills/caveman-compress/scripts/benchmark.py b/.agents/skills/caveman-compress/scripts/benchmark.py new file mode 100644 index 0000000..97d081b --- /dev/null +++ b/.agents/skills/caveman-compress/scripts/benchmark.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +from pathlib import Path +import sys + +# Support both direct execution and module import +try: + from .validate import validate +except ImportError: + sys.path.insert(0, str(Path(__file__).parent)) + from validate import validate + +try: + import tiktoken + _enc = tiktoken.get_encoding("o200k_base") +except ImportError: + _enc = None + + +def count_tokens(text): + if _enc is None: + return len(text.split()) # fallback: word count + return len(_enc.encode(text)) + + +def benchmark_pair(orig_path: Path, comp_path: Path): + orig_text = orig_path.read_text(encoding="utf-8", errors="ignore") + comp_text = comp_path.read_text(encoding="utf-8", errors="ignore") + + orig_tokens = count_tokens(orig_text) + comp_tokens = count_tokens(comp_text) + saved = 100 * (orig_tokens - comp_tokens) / orig_tokens if orig_tokens > 0 else 0.0 + result = validate(orig_path, comp_path) + + return (comp_path.name, orig_tokens, comp_tokens, saved, result.is_valid) + + +def print_table(rows): + print("\n| File | Original | Compressed | Saved % | Valid |") + print("|------|----------|------------|---------|-------|") + for r in rows: + print(f"| {r[0]} | {r[1]} | {r[2]} | {r[3]:.1f}% | {'✅' if r[4] else '❌'} |") + + +def main(): + # Direct file pair: python3 benchmark.py original.md compressed.md + if len(sys.argv) == 3: + orig = Path(sys.argv[1]).resolve() + comp = Path(sys.argv[2]).resolve() + if not orig.exists(): + print(f"❌ Not found: {orig}") + sys.exit(1) + if not comp.exists(): + print(f"❌ Not found: {comp}") + sys.exit(1) + print_table([benchmark_pair(orig, comp)]) + return + + # Glob mode: repo_root/tests/caveman-compress/ + # __file__ lives at /skills/caveman-compress/scripts/benchmark.py + # Walk up four dirs: scripts → caveman-compress → skills → repo_root. + tests_dir = Path(__file__).resolve().parents[3] / "tests" / "caveman-compress" + if not tests_dir.exists(): + print(f"❌ Tests dir not found: {tests_dir}") + sys.exit(1) + + rows = [] + for orig in sorted(tests_dir.glob("*.original.md")): + comp = orig.with_name(orig.stem.removesuffix(".original") + ".md") + if comp.exists(): + rows.append(benchmark_pair(orig, comp)) + + if not rows: + print("No compressed file pairs found.") + return + + print_table(rows) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/caveman-compress/scripts/cli.py b/.agents/skills/caveman-compress/scripts/cli.py new file mode 100644 index 0000000..75ea8a6 --- /dev/null +++ b/.agents/skills/caveman-compress/scripts/cli.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Caveman Compress CLI + +Usage: + caveman +""" + +import sys + +# Force UTF-8 on stdout/stderr before any code can print. Windows consoles +# default to cp1252 and crash on the ❌ glyphs in error/validation branches, +# masking the real error and leaving the user with a half-compressed file. +for _stream in (sys.stdout, sys.stderr): + reconfigure = getattr(_stream, "reconfigure", None) + if callable(reconfigure): + try: + reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + +from pathlib import Path + +from .compress import backup_dir_for, compress_file +from .detect import detect_file_type, should_compress + + +def print_usage(): + print("Usage: caveman ") + + +def main(): + if len(sys.argv) != 2: + print_usage() + sys.exit(1) + + filepath = Path(sys.argv[1]) + + # Check file exists + if not filepath.exists(): + print(f"❌ File not found: {filepath}") + sys.exit(1) + + if not filepath.is_file(): + print(f"❌ Not a file: {filepath}") + sys.exit(1) + + filepath = filepath.resolve() + + # Detect file type + file_type = detect_file_type(filepath) + + print(f"Detected: {file_type}") + + # Check if compressible + if not should_compress(filepath): + print("Skipping: file is not natural language (code/config)") + sys.exit(0) + + print("Starting caveman compression...\n") + + try: + success = compress_file(filepath) + + if success: + print("\nCompression completed successfully") + backup_path = backup_dir_for(filepath) / (filepath.stem + ".original.md") + print(f"Compressed: {filepath}") + print(f"Original: {backup_path}") + sys.exit(0) + else: + print("\n❌ Compression failed after retries") + sys.exit(2) + + except KeyboardInterrupt: + print("\nInterrupted by user") + sys.exit(130) + + except Exception as e: + print(f"\n❌ Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/caveman-compress/scripts/compress.py b/.agents/skills/caveman-compress/scripts/compress.py new file mode 100644 index 0000000..80da520 --- /dev/null +++ b/.agents/skills/caveman-compress/scripts/compress.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +""" +Caveman Memory Compression Orchestrator + +Usage: + python scripts/compress.py +""" + +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import List + +OUTER_FENCE_REGEX = re.compile( + r"\A\s*(`{3,}|~{3,})[^\n]*\n(.*)\n\1\s*\Z", re.DOTALL +) + +# YAML frontmatter: starts at file start with --- on its own line, ends with --- on its own line. +# Captures the entire block (including delimiters and trailing newline) and the body after. +FRONTMATTER_REGEX = re.compile( + r"\A(---\r?\n.*?\r?\n---\r?\n)(.*)", re.DOTALL +) + + +def split_frontmatter(text: str): + """Split YAML frontmatter from body. Returns (frontmatter, body). + + Memory files (and many other markdown docs) start with a YAML frontmatter + block delimited by `---` lines. The compression LLM has a habit of stripping + or rewriting these despite preserve-structure rules in the prompt — so we + surgically remove the frontmatter before compression and prepend it back + verbatim to the output. Files without frontmatter pass through unchanged. + """ + m = FRONTMATTER_REGEX.match(text) + if m: + return m.group(1), m.group(2) + return "", text + +# Filenames and paths that almost certainly hold secrets or PII. Compressing +# them ships raw bytes to the Anthropic API — a third-party data boundary that +# developers on sensitive codebases cannot cross. detect.py already skips .env +# by extension, but credentials.md / secrets.txt / ~/.aws/credentials would +# slip through the natural-language filter. This is a hard refuse before read. +SENSITIVE_BASENAME_REGEX = re.compile( + r"(?ix)^(" + r"\.env(\..+)?" + r"|\.netrc" + r"|credentials(\..+)?" + r"|secrets?(\..+)?" + r"|passwords?(\..+)?" + r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?" + r"|authorized_keys" + r"|known_hosts" + r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)" + r")$" +) + +SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"}) + +SENSITIVE_NAME_TOKENS = ( + "secret", "credential", "password", "passwd", + "apikey", "accesskey", "token", "privatekey", +) + + +def backup_dir_for(filepath: Path) -> Path: + """Resolve the out-of-tree backup directory for a given source file. + + Backups must live OUTSIDE the source directory so skill auto-loaders + (Claude Code rules/, opencode instructions/, etc.) stop re-ingesting the + `.original.md` copies as live files. Base dir is platform-aware: + - Windows: %LOCALAPPDATA%\\caveman-compress\\backups + - else: $XDG_DATA_HOME/caveman-compress/backups if set, + else ~/.local/share/caveman-compress/backups + + The source file's parent-dir name is mirrored under the base to reduce + cross-project collisions (e.g. two `task.md` files in different repos). + """ + if os.name == "nt" or sys.platform == "win32": + local_appdata = os.environ.get("LOCALAPPDATA") + base = Path(local_appdata) if local_appdata else Path.home() / "AppData" / "Local" + base = base / "caveman-compress" / "backups" + else: + xdg = os.environ.get("XDG_DATA_HOME") + base = Path(xdg) if xdg else Path.home() / ".local" / "share" + base = base / "caveman-compress" / "backups" + return base / filepath.parent.name + + +def is_sensitive_path(filepath: Path) -> bool: + """Heuristic denylist for files that must never be shipped to a third-party API.""" + name = filepath.name + if SENSITIVE_BASENAME_REGEX.match(name): + return True + lowered_parts = {p.lower() for p in filepath.parts} + if lowered_parts & SENSITIVE_PATH_COMPONENTS: + return True + # Normalize separators so "api-key" and "api_key" both match "apikey". + lower = re.sub(r"[_\-\s.]", "", name.lower()) + return any(tok in lower for tok in SENSITIVE_NAME_TOKENS) + + +def strip_llm_wrapper(text: str) -> str: + """Strip outer ```markdown ... ``` fence when it wraps the entire output.""" + m = OUTER_FENCE_REGEX.match(text) + if m: + return m.group(2) + return text + + +def write_text_atomic(path: Path, text: str) -> None: + """Write ``text`` to ``path`` atomically as UTF-8. + + Path.write_text() truncates the destination before encoding the string — + a UnicodeEncodeError (or any other failure) partway through leaves a + 0-byte file, destroying whatever was there before (issue #655). Encode + first, write the bytes to a sibling temp file, fsync, then os.replace() + so the destination only ever moves from one complete, valid file to + another. Preserves the original file's permission bits across the swap. + """ + data = text.encode("utf-8") + fd, tmp_name = tempfile.mkstemp( + dir=str(path.parent), prefix=path.name + ".", suffix=".tmp" + ) + tmp_path = Path(tmp_name) + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + if path.exists(): + os.chmod(tmp_path, stat.S_IMODE(path.stat().st_mode)) + os.replace(tmp_path, path) + except Exception: + try: + tmp_path.unlink() + except OSError: + pass + raise + + +def first_nonblank_line(text: str) -> str: + """Return the first non-blank line, stripped — used to detect a prose + preamble smuggled in ahead of the real content (issue #588).""" + for line in text.splitlines(): + if line.strip(): + return line.strip() + return "" + + +def _write_target(filepath: Path, text: str, backup_path: Path) -> None: + """Write to the target file, surfacing the backup location if the write + itself fails. write_text_atomic already leaves the target untouched on + failure, but the caller still needs to know where the pre-compression + original lives instead of being left to guess (issue #652).""" + try: + write_text_atomic(filepath, text) + except Exception: + print(f"❌ Write to {filepath} failed. Original preserved at backup: {backup_path}") + raise + + +from .detect import should_compress +from .validate import validate + +MAX_RETRIES = 2 + + +# ---------- Claude Calls ---------- + + +def call_claude(prompt: str) -> str: + """Send a prompt to Claude. + + Prefers the Anthropic SDK when ANTHROPIC_API_KEY is set; otherwise falls + back to the ``claude --print`` CLI (which handles desktop auth). + + On Windows the CLI subprocess decoding defaults to the system codepage + (cp1251 / cp1252) and crashes on UTF-8 output — see issue #152. Pinning + ``encoding="utf-8"`` with ``errors="replace"`` matches the CLI's actual + native I/O and prevents the UnicodeDecodeError before validation can + report. Windows users with non-ASCII content can also set + ``ANTHROPIC_API_KEY`` to route through the SDK and skip the subprocess. + """ + api_key = os.environ.get("ANTHROPIC_API_KEY") + if api_key: + try: + import anthropic + + client = anthropic.Anthropic(api_key=api_key) + msg = client.messages.create( + model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"), + max_tokens=8192, + messages=[{"role": "user", "content": prompt}], + ) + return strip_llm_wrapper(msg.content[0].text.strip()) + except ImportError: + pass # anthropic not installed, fall back to CLI + # Fallback: use claude CLI (handles desktop auth). + # Resolve binary via shutil.which so Windows .cmd/.bat shims (e.g. + # %APPDATA%\npm\claude.CMD) work without shell=True. On POSIX, + # shutil.which returns the same absolute path as the implicit lookup, + # so this is a no-op there. Falls back to bare "claude" if not found + # on PATH so subprocess raises a clear FileNotFoundError. + claude_bin = shutil.which("claude") or "claude" + try: + result = subprocess.run( + [claude_bin, "--print"], + input=prompt, + text=True, + capture_output=True, + check=True, + encoding="utf-8", + errors="replace", + ) + return strip_llm_wrapper(result.stdout.strip()) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Claude call failed:\n{e.stderr}") + + +def build_compress_prompt(original: str) -> str: + return f""" +Compress this markdown into caveman format. + +STRICT RULES: +- Do NOT modify anything inside ``` code blocks +- Do NOT modify anything inside inline backticks +- Preserve ALL URLs exactly +- Preserve ALL headings exactly +- Preserve file paths and commands +- Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file. + +Only compress natural language. + +TEXT: +{original} +""" + + +def build_fix_prompt(original: str, compressed: str, errors: List[str]) -> str: + errors_str = "\n".join(f"- {e}" for e in errors) + return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found. + +CRITICAL RULES: +- DO NOT recompress or rephrase the file +- ONLY fix the listed errors — leave everything else exactly as-is +- The ORIGINAL is provided as reference only (to restore missing content) +- Preserve caveman style in all untouched sections + +ERRORS TO FIX: +{errors_str} + +HOW TO FIX: +- Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED +- Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED +- Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED +- Do not touch any section not mentioned in the errors + +ORIGINAL (reference only): +{original} + +COMPRESSED (fix this): +{compressed} + +Return ONLY the fixed compressed file. No explanation. +""" + + +# ---------- Core Logic ---------- + + +def compress_file(filepath: Path) -> bool: + # Resolve and validate path + filepath = filepath.resolve() + MAX_FILE_SIZE = 500_000 # 500KB + if not filepath.exists(): + raise FileNotFoundError(f"File not found: {filepath}") + if filepath.stat().st_size > MAX_FILE_SIZE: + raise ValueError(f"File too large to compress safely (max 500KB): {filepath}") + + # Refuse files that look like they contain secrets or PII. Compressing ships + # the raw bytes to the Anthropic API — a third-party boundary — so we fail + # loudly rather than silently exfiltrate credentials or keys. Override is + # intentional: the user must rename the file if the heuristic is wrong. + if is_sensitive_path(filepath): + raise ValueError( + f"Refusing to compress {filepath}: filename looks sensitive " + "(credentials, keys, secrets, or known private paths). " + "Compression sends file contents to the Anthropic API. " + "Rename the file if this is a false positive." + ) + + print(f"Processing: {filepath}") + + if not should_compress(filepath): + print("Skipping (not natural language)") + return False + + original_text = filepath.read_text(encoding="utf-8", errors="ignore") + # Store backup outside the source directory so skill auto-loaders don't + # re-ingest the `.original.md` copy as a live file. Mirror the source's + # parent-dir name + stem under a platform-aware base to reduce collisions. + backup_dir = backup_dir_for(filepath) + backup_dir.mkdir(parents=True, exist_ok=True) + backup_path = backup_dir / (filepath.stem + ".original.md") + + if not original_text.strip(): + print("❌ Refusing to compress: file is empty or whitespace-only.") + return False + + # Check if backup already exists to prevent accidental overwriting + if backup_path.exists(): + print(f"⚠️ Backup file already exists: {backup_path}") + print("The original backup may contain important content.") + print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.") + return False + + # Split YAML frontmatter off before compression. Claude tends to strip or + # rewrite frontmatter despite preserve-structure rules; we keep it verbatim + # by removing it from the input and re-prepending it to the output. + frontmatter, body = split_frontmatter(original_text) + if frontmatter: + print(f"Detected YAML frontmatter ({len(frontmatter)} chars) — preserving verbatim") + + if not body.strip(): + print("❌ Refusing to compress: body is empty after frontmatter removal.") + return False + + # Step 1: Compress (body only, frontmatter excluded) + print("Compressing with Claude...") + compressed_body = call_claude(build_compress_prompt(body)) + + if compressed_body is None or not compressed_body.strip(): + print("❌ Compression aborted: Claude returned an empty response.") + print(" Original file is untouched (no backup created).") + return False + + # Compare the BODY (not the whole file) — frontmatter is preserved verbatim + # and would never change, so identity must be judged on the compressible part. + if compressed_body.strip() == body.strip(): + print("❌ Compression aborted: output is identical to input.") + print(" Likely causes: Claude refused, returned the prompt verbatim, or the file is") + print(" already in caveman form. Original file is untouched (no backup created).") + return False + + # Reassemble: frontmatter (verbatim) + compressed body + compressed = frontmatter + compressed_body + + # Save original as backup, then verify the backup readback before + # touching the input file. If the filesystem dropped bytes (encoding, + # antivirus, disk full), unlink the bad backup and abort instead of + # leaving the user with a corrupt backup + compressed primary. + write_text_atomic(backup_path, original_text) + backup_readback = backup_path.read_text(encoding="utf-8", errors="ignore") + if backup_readback != original_text: + print(f"❌ Backup write verification failed: {backup_path}") + print(" In-memory original differs from on-disk backup. Aborting before touching the input file.") + try: + backup_path.unlink() + except OSError: + pass + return False + _write_target(filepath, compressed, backup_path) + + # Step 2: Validate + Retry + for attempt in range(MAX_RETRIES): + print(f"\nValidation attempt {attempt + 1}") + + result = validate(backup_path, filepath) + + if result.is_valid: + print("Validation passed") + break + + print("❌ Validation failed:") + for err in result.errors: + print(f" - {err}") + + if attempt == MAX_RETRIES - 1: + # Restore original on failure + _write_target(filepath, original_text, backup_path) + backup_path.unlink(missing_ok=True) + print("❌ Failed after retries — original restored") + return False + + print("Fixing with Claude...") + compressed = call_claude( + build_fix_prompt(original_text, compressed, result.errors) + ) + + if compressed is None or not compressed.strip(): + print("❌ Fix attempt aborted: Claude returned an empty response.") + print(" Skipping this attempt.") + continue + + # Guard against a prose preamble smuggled in ahead of the real fixed + # content (issue #588). Only enforced when the original starts with a + # structural anchor (frontmatter `---` or a heading) — plain-prose + # first lines get legitimately rewritten by compression, and requiring + # them verbatim would reject every valid fix. + anchor = first_nonblank_line(original_text) + if anchor.startswith(("---", "#")) and first_nonblank_line(compressed) != anchor: + print("❌ Fix attempt aborted: output does not start with the original's first line.") + print(" Possible preamble leak. Skipping this attempt.") + continue + + _write_target(filepath, compressed, backup_path) + + return True diff --git a/.agents/skills/caveman-compress/scripts/detect.py b/.agents/skills/caveman-compress/scripts/detect.py new file mode 100644 index 0000000..6a468d5 --- /dev/null +++ b/.agents/skills/caveman-compress/scripts/detect.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Detect whether a file is natural language (compressible) or code/config (skip).""" + +import json +import re +from pathlib import Path + +# Extensions that are natural language and compressible +COMPRESSIBLE_EXTENSIONS = {".md", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"} + +# Extensions that are code/config and should be skipped +SKIP_EXTENSIONS = { + ".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml", + ".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml", + ".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c", + ".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua", + ".dockerfile", ".makefile", ".csv", ".ini", ".cfg", +} + +# Well-known build/config files that carry no (or a misleading) extension — +# `Dockerfile` has no suffix so `.dockerfile` above never matches it, and +# `CMakeLists.txt` would ride the compressible `.txt` rule. Checked by +# basename before any extension rule. +KNOWN_CODE_FILENAMES = { + "dockerfile", "makefile", "gnumakefile", "jenkinsfile", "vagrantfile", + "rakefile", "gemfile", "justfile", "procfile", "brewfile", + "cmakelists.txt", +} + +# Patterns that indicate a line is code +CODE_PATTERNS = [ + re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"), + re.compile(r"^\s*(def |class |function |async function |export )"), + re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"), + re.compile(r"^\s*[\}\]\);]+\s*$"), # closing braces/brackets + re.compile(r"^\s*@\w+"), # decorators/annotations + re.compile(r'^\s*"[^"]+"\s*:\s*'), # JSON-like key-value + re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"), # assignment with literal +] + + +def _is_code_line(line: str) -> bool: + """Check if a line looks like code.""" + return any(p.match(line) for p in CODE_PATTERNS) + + +def _is_json_content(text: str) -> bool: + """Check if content is valid JSON.""" + try: + json.loads(text) + return True + except (json.JSONDecodeError, ValueError): + return False + + +def _is_yaml_content(lines: list[str]) -> bool: + """Heuristic: check if content looks like YAML.""" + yaml_indicators = 0 + for line in lines[:30]: + stripped = line.strip() + if stripped.startswith("---"): + yaml_indicators += 1 + elif re.match(r"^\w[\w\s]*:\s", stripped): + yaml_indicators += 1 + elif stripped.startswith("- ") and ":" in stripped: + yaml_indicators += 1 + # If most non-empty lines look like YAML + non_empty = sum(1 for l in lines[:30] if l.strip()) + return non_empty > 0 and yaml_indicators / non_empty > 0.6 + + +def detect_file_type(filepath: Path) -> str: + """Classify a file as 'natural_language', 'code', 'config', or 'unknown'. + + Returns: + One of: 'natural_language', 'code', 'config', 'unknown' + """ + ext = filepath.suffix.lower() + + # Known code filenames win over any extension rule + if filepath.name.lower() in KNOWN_CODE_FILENAMES: + return "code" + + # Extension-based classification + if ext in COMPRESSIBLE_EXTENSIONS: + return "natural_language" + if ext in SKIP_EXTENSIONS: + return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config" + + # Extensionless files (like CLAUDE.md, TODO) — check content + if not ext: + try: + text = filepath.read_text(encoding="utf-8", errors="ignore") + except (OSError, PermissionError): + return "unknown" + + lines = text.splitlines()[:50] + + # Shebang means executable script, never prose + if text.startswith("#!"): + return "code" + + if _is_json_content(text[:10000]): + return "config" + if _is_yaml_content(lines): + return "config" + + code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l)) + non_empty = sum(1 for l in lines if l.strip()) + if non_empty > 0 and code_lines / non_empty > 0.4: + return "code" + + return "natural_language" + + return "unknown" + + +def should_compress(filepath: Path) -> bool: + """Return True if the file is natural language and should be compressed.""" + if not filepath.is_file(): + return False + # Skip backup files + if filepath.name.endswith(".original.md"): + return False + return detect_file_type(filepath) == "natural_language" + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("Usage: python detect.py [file2] ...") + sys.exit(1) + + for path_str in sys.argv[1:]: + p = Path(path_str).resolve() + file_type = detect_file_type(p) + compress = should_compress(p) + print(f" {p.name:30s} type={file_type:20s} compress={compress}") diff --git a/.agents/skills/caveman-compress/scripts/validate.py b/.agents/skills/caveman-compress/scripts/validate.py new file mode 100644 index 0000000..dcd2a5d --- /dev/null +++ b/.agents/skills/caveman-compress/scripts/validate.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +import re +from collections import Counter +from pathlib import Path + +URL_REGEX = re.compile(r"https?://[^\s)]+") +FENCE_OPEN_REGEX = re.compile(r"^(\s{0,3})(`{3,}|~{3,})(.*)$") +HEADING_REGEX = re.compile(r"^(#{1,6})\s+(.*)", re.MULTILINE) +BULLET_REGEX = re.compile(r"^\s*[-*+]\s+", re.MULTILINE) + +# crude but effective path detection +# Requires either a path prefix (./ ../ / or drive letter) or a slash/backslash within the match +PATH_REGEX = re.compile(r"(?:\./|\.\./|/|[A-Za-z]:\\)[\w\-/\\\.]+|[\w\-\.]+[/\\][\w\-/\\\.]+") + + +class ValidationResult: + def __init__(self): + self.is_valid = True + self.errors = [] + self.warnings = [] + + def add_error(self, msg): + self.is_valid = False + self.errors.append(msg) + + def add_warning(self, msg): + self.warnings.append(msg) + + +def read_file(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +# ---------- Extractors ---------- + + +def extract_headings(text): + return [(level, title.strip()) for level, title in HEADING_REGEX.findall(text)] + + +def extract_code_blocks(text): + """Line-based fenced code block extractor. + + Handles ``` and ~~~ fences with variable length (CommonMark: closing + fence must use same char and be at least as long as opening). Supports + nested fences (e.g. an outer 4-backtick block wrapping inner 3-backtick + content). + """ + blocks = [] + lines = text.split("\n") + i = 0 + n = len(lines) + while i < n: + m = FENCE_OPEN_REGEX.match(lines[i]) + if not m: + i += 1 + continue + fence_char = m.group(2)[0] + fence_len = len(m.group(2)) + open_line = lines[i] + block_lines = [open_line] + i += 1 + closed = False + while i < n: + close_m = FENCE_OPEN_REGEX.match(lines[i]) + if ( + close_m + and close_m.group(2)[0] == fence_char + and len(close_m.group(2)) >= fence_len + and close_m.group(3).strip() == "" + ): + block_lines.append(lines[i]) + closed = True + i += 1 + break + block_lines.append(lines[i]) + i += 1 + if closed: + blocks.append("\n".join(block_lines)) + # Unclosed fences are silently skipped — they indicate malformed markdown + # and including them would cause false-positive validation failures. + return blocks + + +def extract_urls(text): + return set(URL_REGEX.findall(text)) + + +def extract_paths(text): + return set(PATH_REGEX.findall(text)) + + +def count_bullets(text): + return len(BULLET_REGEX.findall(text)) + + +def extract_inline_codes(text): + """Backtick-delimited inline spans, with fenced code blocks stripped first. + + Previously used a column-0-anchored regex to strip fences, which misses + fences indented 1-3 spaces (valid CommonMark). Reuse extract_code_blocks + (FENCE_OPEN_REGEX-based, indentation-aware) instead so an indented fence's + body backticks don't leak into inline-code pairing. + """ + text_without_fences = text + for block in extract_code_blocks(text): + text_without_fences = text_without_fences.replace(block, "", 1) + return re.findall(r"`([^`]+)`", text_without_fences) + + +# ---------- Validators ---------- + + +def validate_headings(orig, comp, result): + h1 = extract_headings(orig) + h2 = extract_headings(comp) + + if len(h1) != len(h2): + result.add_error(f"Heading count mismatch: {len(h1)} vs {len(h2)}") + + if h1 != h2: + result.add_warning("Heading text/order changed") + + +def validate_code_blocks(orig, comp, result): + c1 = extract_code_blocks(orig) + c2 = extract_code_blocks(comp) + + if c1 != c2: + result.add_error("Code blocks not preserved exactly") + + +def validate_urls(orig, comp, result): + u1 = extract_urls(orig) + u2 = extract_urls(comp) + + if u1 != u2: + result.add_error(f"URL mismatch: lost={u1 - u2}, added={u2 - u1}") + + +def validate_paths(orig, comp, result): + p1 = extract_paths(orig) + p2 = extract_paths(comp) + + if p1 != p2: + result.add_warning(f"Path mismatch: lost={p1 - p2}, added={p2 - p1}") + + +def validate_bullets(orig, comp, result): + b1 = count_bullets(orig) + b2 = count_bullets(comp) + + if b1 == 0: + return + + diff = abs(b1 - b2) / b1 + + if diff > 0.15: + result.add_warning(f"Bullet count changed too much: {b1} -> {b2}") + + +def validate_inline_codes(orig, comp, result): + c1 = Counter(extract_inline_codes(orig)) + c2 = Counter(extract_inline_codes(comp)) + + if c1 != c2: + lost = set(c1.keys()) - set(c2.keys()) + added = set(c2.keys()) - set(c1.keys()) + for code, count in c1.items(): + if code in c2 and c2[code] < count: + lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)") + if lost: + result.add_error(f"Inline code lost: {lost}") + if added: + result.add_warning(f"Inline code added: {added}") + + +# ---------- Main ---------- + + +def validate(original_path: Path, compressed_path: Path) -> ValidationResult: + result = ValidationResult() + + orig = read_file(original_path) + comp = read_file(compressed_path) + + validate_headings(orig, comp, result) + validate_code_blocks(orig, comp, result) + validate_urls(orig, comp, result) + validate_paths(orig, comp, result) + validate_bullets(orig, comp, result) + validate_inline_codes(orig, comp, result) + + return result + + +# ---------- CLI ---------- + +if __name__ == "__main__": + import sys + + if len(sys.argv) != 3: + print("Usage: python validate.py ") + sys.exit(1) + + orig = Path(sys.argv[1]).resolve() + comp = Path(sys.argv[2]).resolve() + + res = validate(orig, comp) + + print(f"\nValid: {res.is_valid}") + + if res.errors: + print("\nErrors:") + for e in res.errors: + print(f" - {e}") + + if res.warnings: + print("\nWarnings:") + for w in res.warnings: + print(f" - {w}") diff --git a/.agents/skills/caveman-help/README.md b/.agents/skills/caveman-help/README.md new file mode 100644 index 0000000..5841256 --- /dev/null +++ b/.agents/skills/caveman-help/README.md @@ -0,0 +1,38 @@ +# caveman-help + +Quick-reference card. One shot, no mode change. + +## What it does + +Prints a cheat sheet of all caveman modes, sibling skills, deactivation triggers, and how to set the default mode via env var or config file. One-shot display — does not flip the active mode, write flag files, or persist anything. Use when you forget the slash commands. + +## How to invoke + +``` +/caveman-help +``` + +Also triggers on "caveman help", "what caveman commands", "how do I use caveman". + +## Example output + +``` +Modes: + /caveman full (default) + /caveman lite lighter + /caveman ultra extreme + /caveman wenyan classical Chinese + +Skills: + /caveman-commit terse Conventional Commits + /caveman-review one-line PR comments + /caveman-stats session token savings + +Deactivate: + "stop caveman" or "normal mode" +``` + +## See also + +- [`SKILL.md`](./SKILL.md) — full reference card +- [Caveman README](../../README.md) — repo overview diff --git a/.agents/skills/caveman-help/SKILL.md b/.agents/skills/caveman-help/SKILL.md new file mode 100644 index 0000000..346579d --- /dev/null +++ b/.agents/skills/caveman-help/SKILL.md @@ -0,0 +1,63 @@ +--- +name: caveman-help +description: > + Quick-reference card for all caveman modes, skills, and commands. + One-shot display, not a persistent mode. Trigger: /caveman-help, + "caveman help", "what caveman commands", "how do I use caveman". +--- + +# Caveman Help + +Display this reference card when invoked. One-shot — do NOT change mode, write flag files, or persist anything. Output in caveman style. + +## Modes + +| Mode | Trigger | What change | +|------|---------|-------------| +| **Lite** | `/caveman lite` | Drop filler. Keep sentence structure. | +| **Full** | `/caveman` | Drop articles, filler, pleasantries, hedging. Fragments OK. Default. | +| **Ultra** | `/caveman ultra` | Extreme compression. Bare fragments. Tables over prose. | +| **Wenyan-Lite** | `/caveman wenyan-lite` | Classical Chinese style, light compression. | +| **Wenyan-Full** | `/caveman wenyan` | Full 文言文. Maximum classical terseness. | +| **Wenyan-Ultra** | `/caveman wenyan-ultra` | Extreme. Ancient scholar on a budget. | + +Mode stick until changed or session end. + +## Skills + +| Skill | Trigger | What it do | +|-------|---------|-----------| +| **caveman-commit** | `/caveman-commit` | Terse commit messages. Conventional Commits. ≤50 char subject. | +| **caveman-review** | `/caveman-review` | One-line PR comments: `L42: bug: user null. Add guard.` | +| **caveman-compress** | `/caveman-compress ` | Compress .md files to caveman prose. Saves ~46% input tokens. | +| **caveman-help** | `/caveman-help` | This card. | + +## Deactivate + +Say "stop caveman" or "normal mode". Resume anytime with `/caveman`. + +## Language + +Keep user's language by default. User write Portuguese → reply Portuguese caveman. Compress the style, not the language. Technical terms, code, commands, commit types, and exact error strings stay verbatim unless user ask for translation. + +## Configure Default Mode + +Default mode = `full`. Change it: + +**Environment variable** (highest priority): +```bash +export CAVEMAN_DEFAULT_MODE=ultra +``` + +**Config file** (`~/.config/caveman/config.json` macOS/Linux, `%APPDATA%\caveman\config.json` Windows): +```json +{ "defaultMode": "lite" } +``` + +Set `"off"` to disable auto-activation on session start. User can still activate manually with `/caveman`. + +Resolution: env var > config file > `full`. + +## More + +Full docs: https://github.com/JuliusBrussee/caveman diff --git a/.agents/skills/caveman-review/README.md b/.agents/skills/caveman-review/README.md new file mode 100644 index 0000000..acf519f --- /dev/null +++ b/.agents/skills/caveman-review/README.md @@ -0,0 +1,33 @@ +# caveman-review + +One-line PR comments. Location, problem, fix. No throat-clearing. + +## What it does + +Generates code review comments in `L: . .` format. One line per finding. Severity emoji: 🔴 bug, 🟡 risk, 🔵 nit, ❓ question. Drops "I noticed that...", hedging, and restating what the diff already shows. Keeps exact line numbers, backticked symbols, and concrete fixes. + +Auto-clarity: drops terse mode for CVE-class security findings, architectural disagreements, and onboarding contexts where the author needs the *why*. Resumes terse for the rest. + +Output only — does not approve, request changes, or run linters. + +## How to invoke + +``` +/caveman-review +``` + +Also triggers on "review this PR", "code review", "review the diff". + +## Example output + +``` +L42: 🔴 bug: user can be null after .find(). Add guard before .email. +L88-140: 🔵 nit: 50-line fn does 4 things. Extract validate/normalize/persist. +L23: 🟡 risk: no retry on 429. Wrap in withBackoff(3). +L107: ❓ q: why drop the cache here? Reads on next request will miss. +``` + +## See also + +- [`SKILL.md`](./SKILL.md) — full LLM-facing instructions +- [Caveman README](../../README.md) — repo overview diff --git a/.agents/skills/caveman-review/SKILL.md b/.agents/skills/caveman-review/SKILL.md new file mode 100644 index 0000000..48f4adb --- /dev/null +++ b/.agents/skills/caveman-review/SKILL.md @@ -0,0 +1,55 @@ +--- +name: caveman-review +description: > + Ultra-compressed code review comments. Cuts noise from PR feedback while preserving + the actionable signal. Each comment is one line: location, problem, fix. Use when user + says "review this PR", "code review", "review the diff", "/review", or invokes + /caveman-review. Auto-triggers when reviewing pull requests. +--- + +Write code review comments terse and actionable. One line per finding. Location, problem, fix. No throat-clearing. + +## Rules + +**Format:** `L: . .` — or `:L: ...` when reviewing multi-file diffs. + +**Severity prefix (optional, when mixed):** +- `🔴 bug:` — broken behavior, will cause incident +- `🟡 risk:` — works but fragile (race, missing null check, swallowed error) +- `🔵 nit:` — style, naming, micro-optim. Author can ignore +- `❓ q:` — genuine question, not a suggestion + +**Drop:** +- "I noticed that...", "It seems like...", "You might want to consider..." +- "This is just a suggestion but..." — use `nit:` instead +- "Great work!", "Looks good overall but..." — say it once at the top, not per comment +- Restating what the line does — the reviewer can read the diff +- Hedging ("perhaps", "maybe", "I think") — if unsure use `q:` + +**Keep:** +- Exact line numbers +- Exact symbol/function/variable names in backticks +- Concrete fix, not "consider refactoring this" +- The *why* if the fix isn't obvious from the problem statement + +## Examples + +❌ "I noticed that on line 42 you're not checking if the user object is null before accessing the email property. This could potentially cause a crash if the user is not found in the database. You might want to add a null check here." + +✅ `L42: 🔴 bug: user can be null after .find(). Add guard before .email.` + +❌ "It looks like this function is doing a lot of things and might benefit from being broken up into smaller functions for readability." + +✅ `L88-140: 🔵 nit: 50-line fn does 4 things. Extract validate/normalize/persist.` + +❌ "Have you considered what happens if the API returns a 429? I think we should probably handle that case." + +✅ `L23: 🟡 risk: no retry on 429. Wrap in withBackoff(3).` + +## Auto-Clarity + +Drop terse mode for: security findings (CVE-class bugs need full explanation + reference), architectural disagreements (need rationale, not just a one-liner), and onboarding contexts where the author is new and needs the "why". In those cases write a normal paragraph, then resume terse for the rest. + +## Boundaries + +Reviews only — does not write the code fix, does not approve/request-changes, does not run linters. Output the comment(s) ready to paste into the PR. "stop caveman-review" or "normal mode": revert to verbose review style. \ No newline at end of file diff --git a/.agents/skills/caveman-stats/README.md b/.agents/skills/caveman-stats/README.md new file mode 100644 index 0000000..1dfdeab --- /dev/null +++ b/.agents/skills/caveman-stats/README.md @@ -0,0 +1,36 @@ +# caveman-stats + +Real session token receipts. No AI estimation. + +## What it does + +Reads the current Claude Code session log directly and reports actual input/output token usage plus estimated savings versus a non-caveman baseline. Numbers come from the JSONL session log on disk — the model itself does not compute or estimate them. Output is injected by the `caveman-mode-tracker` hook, which intercepts `/caveman-stats` and returns the formatted stats as a blocked-decision reason. + +Output also includes an `Est. rule overhead` and `Est. net` line whenever the savings figure above them is unambiguous (a single benchmarked mode with a known turn count — no guessing across mixed or unattributed spans). Overhead estimates the per-turn INPUT-token cost of the rules the skill injects every turn — default 1,250 tokens/turn, override with `CAVEMAN_RULE_OVERHEAD_TOKENS` if you've measured your own setup. Net is savings minus that overhead. On short, terse replies this can go negative — caveman's OUTPUT savings don't clear its INPUT cost — and the line says so directly instead of hiding it behind a gross-savings number. Background: `docs/HONEST-NUMBERS.md`. + +Each run also writes a lifetime-savings suffix file used by the statusline badge (`⛏ 12.4k`). That badge stays a gross-savings figure on purpose — it is a glanceable summary, not a full accounting; run `/caveman-stats` for the net picture. + +## How to invoke + +``` +/caveman-stats +``` + +## Example output + +``` +Session: 47 turns +Input: 12,304 tokens +Output: 3,891 tokens (caveman) +Baseline: 11,247 tokens (estimated without caveman) +Saved: 7,356 tokens (~65%) +Est. rule overhead: 58,750 (input, ~1,250/turn over 47 turns) +Est. net: -51,394 (caveman cost more than it saved for this workload — consider turning it off) +``` + +(Numbers above are illustrative — see `docs/HONEST-NUMBERS.md` for why short, terse-reply sessions tend to land net-negative even at a healthy output-savings percentage.) + +## See also + +- [`SKILL.md`](./SKILL.md) — hook contract and mechanics +- [Caveman README](../../README.md) — repo overview diff --git a/.agents/skills/caveman-stats/SKILL.md b/.agents/skills/caveman-stats/SKILL.md new file mode 100644 index 0000000..4c04b92 --- /dev/null +++ b/.agents/skills/caveman-stats/SKILL.md @@ -0,0 +1,12 @@ +--- +name: caveman-stats +description: > + Show real token usage and estimated savings for the current session. + Reads directly from the Claude Code session log — no AI estimation. + Triggers on /caveman-stats. Output is injected by the mode-tracker hook; + the model itself does not compute the numbers. +--- + +This skill is delivered by `hooks/caveman-stats.js` (read by `hooks/caveman-mode-tracker.js` on `/caveman-stats`). The model does not need to do anything when this skill fires — the hook returns `decision: "block"` with the formatted stats as the reason. The user sees the numbers immediately. + +Output also includes `Est. rule overhead` and `Est. net` lines wherever a savings estimate exists with a known turn count. Rule overhead is the estimated per-turn INPUT-token cost of the injected caveman rules (default 1,250 tokens/turn, override with `CAVEMAN_RULE_OVERHEAD_TOKENS`) times the turn count. Net is savings minus that overhead — when negative, the output says so plainly and suggests turning caveman off for that workload, rather than hiding the net-negative regime behind a gross-savings number (see `docs/HONEST-NUMBERS.md`). diff --git a/.agents/skills/caveman/README.md b/.agents/skills/caveman/README.md new file mode 100644 index 0000000..696a4e3 --- /dev/null +++ b/.agents/skills/caveman/README.md @@ -0,0 +1,48 @@ +# caveman + +Talk like smart caveman. Same brain, fewer tokens. + +## What it does + +Compress every model response to caveman-style prose. Drops articles, filler, pleasantries, and hedging. Keeps every technical detail, code block, error string, and symbol exact. Cuts 65% of output tokens (measured) with full accuracy preserved. Mode persists for the whole session until changed or stopped. + +Six intensity levels: + +| Level | What change | +|-------|-------------| +| `lite` | Drop filler/hedging. Sentences stay full. Professional but tight. | +| `full` | Default. Drop articles, fragments OK, short synonyms. | +| `ultra` | Bare fragments. Abbreviations (DB, auth, fn). Arrows for causality. | +| `wenyan-lite` | Classical Chinese register, light compression. | +| `wenyan-full` | Maximum 文言文. 80-90% character reduction. | +| `wenyan-ultra` | Extreme classical compression. | + +Auto-clarity rule: caveman drops to normal prose for security warnings, irreversible-action confirmations, multi-step sequences where fragment ambiguity risks misread, and when user repeats a question. Resumes after the clear part. + +## How to invoke + +``` +/caveman # full mode (default) +/caveman lite # lighter compression +/caveman ultra # extreme compression +/caveman wenyan # classical Chinese +stop caveman # back to normal prose +``` + +## Example output + +Question: "Why does my React component re-render?" + +Normal prose: +> Your component re-renders because you create a new object reference each render. Wrapping it in `useMemo` will fix the issue. + +Caveman (full): +> New object ref each render. Inline object prop = new ref = re-render. Wrap in `useMemo`. + +Caveman (ultra): +> Inline obj prop → new ref → re-render. `useMemo`. + +## See also + +- [`SKILL.md`](./SKILL.md) — full LLM-facing instructions +- [Caveman README](../../README.md) — repo overview, install, benchmarks diff --git a/.agents/skills/caveman/SKILL.md b/.agents/skills/caveman/SKILL.md new file mode 100644 index 0000000..2d31b3d --- /dev/null +++ b/.agents/skills/caveman/SKILL.md @@ -0,0 +1,88 @@ +--- +name: caveman +description: > + Ultra-compressed communication mode. Cuts output tokens 65% (measured) by speaking like caveman + while keeping full technical accuracy. Supports intensity levels: lite, full (default), ultra, + wenyan-lite, wenyan-full, wenyan-ultra. + Use when user says "caveman mode", "talk like caveman", "use caveman", "less tokens", + "be brief", or invokes /caveman. Also auto-triggers when token efficiency is requested. +--- + +Respond terse like smart caveman. All technical substance stay. Only fluff die. + +## Persistence + +ACTIVE EVERY RESPONSE. No revert after many turns. No filler drift. Still active if unsure. Off only: "stop caveman" / "normal mode". + +Default: **full**. Switch: `/caveman lite|full|ultra|wenyan-lite|wenyan-full|wenyan-ultra|off`. + +## Rules + +Drop: articles (a/an/the), filler (just/really/basically/actually/simply), pleasantries (sure/certainly/of course/happy to), hedging. Fragments OK. Short synonyms (big not extensive, fix not "implement a solution for"). No tool-call narration, no decorative tables/emoji, no dumping long raw error logs unless asked — quote shortest decisive line. Standard well-known tech acronyms OK (DB/API/HTTP); never invent new abbreviations (cfg/impl/req/res/fn) — tokenizer split them same as full word: zero token saved, reader still decode. Full word cheaper AND clearer. No causal arrows (→) either — own token, save nothing. Technical terms exact. Code blocks unchanged. Errors quoted exact. + +Never drop not/never/no/only/except — flip meaning worse than any token saved. Numbers, units exact. + +Tool calls: fire direct. No preamble, plan, or progress note before or between calls. After result: next call direct or final answer — never announce next call. Text before call only to clarify, warn security/irreversible, or resolve ambiguity. + +Preserve user's dominant language exactly — reply in the language user writes, never switch regardless of example text or multilingual context elsewhere. Compress the style, not the language. Every emitted line in that language — openings, pre-tool status lines, all — not just final reply. ALWAYS keep technical terms, code, API names, CLI commands, commit-type keywords (feat/fix/...), and exact error strings verbatim — unless user explicitly ask for translation. + +'Drop articles' = article languages only. Where small markers carry case/role (particles, postpositions), keep them — grammar, not filler; compress politeness/filler instead. + +No self-reference. Never name or announce the style. No "caveman mode on", "me caveman think", no third-person caveman tags. Output caveman-only — never normal answer plus "Caveman:" recap. Exception: user explicitly ask what the mode is. + +Pattern: `[thing] [action] [reason]. [next step].` + +Not: "Sure! I'd be happy to help you with that. The issue you're experiencing is likely caused by..." +Yes: "Bug in auth middleware. Token expiry check use `<` not `<=`. Fix:" + +## Intensity + +| Level | What change | +|-------|------------| +| **lite** | No filler/hedging. Keep articles + full sentences. Professional but tight | +| **full** | Drop articles, fragments OK, short synonyms. Classic caveman. No tool-call narration, no decorative tables/emoji, no long raw error-log dumps unless asked. Standard acronyms OK; no invented abbreviations | +| **ultra** | Strip conjunctions when cause-then-effect stay unambiguous. One word when one word enough. State each fact once. NO prose abbreviations (cfg/impl/req/res/fn/auth), NO arrows (X → Y) — measured zero token saving under tokenizer, cost decode clarity. Code symbols, function names, API names, error strings: never touch | +| **wenyan-lite** | Semi-classical. Drop filler/hedging but keep grammar structure, classical register | +| **wenyan-full** | Maximum classical terseness. Fully 文言文. 80-90% character reduction — chars, not tokens. Classical sentence patterns, verbs precede objects, subjects often omitted, classical particles (之/乃/為/其) | +| **wenyan-ultra** | Extreme abbreviation while keeping classical Chinese feel. Maximum compression, ultra terse | + +Example — "Why React component re-render?" +- lite: "Your component re-renders because you create a new object reference each render. Wrap it in `useMemo`." +- full: "New object ref each render. Inline object prop = new ref = re-render. Wrap in `useMemo`." +- ultra: "Inline obj prop, new ref, re-render. `useMemo`." +- wenyan-lite: "組件頻重繪,以每繪新生對象參照故。以 useMemo 包之。" +- wenyan-full: "每繪新生對象參照,故重繪;以 useMemo 包之則免。" +- wenyan-ultra: "新參照則重繪。useMemo 包之。" + +Example — "Explain database connection pooling." +- lite: "Connection pooling reuses open connections instead of creating new ones per request. Avoids repeated handshake overhead." +- full: "Pool reuse open DB connections. No new connection per request. Skip handshake overhead." +- ultra: "Pool reuse open DB connections. No per-request handshake." +- wenyan-full: "池蓄已開之連,不逐請而新開,省握手之費。" +- wenyan-ultra: "池蓄連,免逐請新開,省握手。" + +Classical chars = wenyan modes only. Never swap a word to a classical char to shrink at non-wenyan levels. + +## Auto-Clarity + +Drop caveman when: +- Security warnings +- Irreversible action confirmations +- Multi-step sequences where fragment order or omitted conjunctions risk misread +- Compression itself creates technical ambiguity (e.g., `"migrate table drop column backup first"` — order unclear without articles/conjunctions) +- User asks to clarify or repeats question + +Resume caveman after clear part done. + +Example shows FORMAT only — write warning in session language, not example's. + +Example — destructive op: +> **Warning:** This will permanently delete all rows in the `users` table and cannot be undone. +> ```sql +> DROP TABLE users; +> ``` +> Caveman resume. Verify backup exist first. + +## Boundaries + +Persisted outside chat: write normal prose — code, comments, commits, docs, issue/PR/MR text, memory files, third-party messages (/caveman-compress exempt). "stop caveman" or "normal mode": revert. Level persist until changed or session end. \ No newline at end of file diff --git a/skills-lock.json b/skills-lock.json index 8b47cf5..3795634 100644 --- a/skills-lock.json +++ b/skills-lock.json @@ -7,6 +7,48 @@ "skillPath": "skills/engineering/ask-matt/SKILL.md", "computedHash": "0f843160e34a24f5bd12cdc7de7d40951e77fbdc05ce8f891b37ca32eac2c44d" }, + "cavecrew": { + "source": "JuliusBrussee/caveman", + "sourceType": "github", + "skillPath": "skills/cavecrew/SKILL.md", + "computedHash": "c5527c994fbd4c22b36714e3b124a0f167a533d114ab164fb1d35e2123533917" + }, + "caveman": { + "source": "JuliusBrussee/caveman", + "sourceType": "github", + "skillPath": "skills/caveman/SKILL.md", + "computedHash": "59e1fe0d3eeb4189ee5c467efde567672e5cacb41f157c477a6152ca907d44ea" + }, + "caveman-commit": { + "source": "JuliusBrussee/caveman", + "sourceType": "github", + "skillPath": "skills/caveman-commit/SKILL.md", + "computedHash": "790a4eeace0be35c6691faf923518ba5bd50f1f1305d1101d09dd4971be94e00" + }, + "caveman-compress": { + "source": "JuliusBrussee/caveman", + "sourceType": "github", + "skillPath": "skills/caveman-compress/SKILL.md", + "computedHash": "52f2301832b376a765b0ed02445c8bf05874b624052bf9a9c3861d9c3dfcee4b" + }, + "caveman-help": { + "source": "JuliusBrussee/caveman", + "sourceType": "github", + "skillPath": "skills/caveman-help/SKILL.md", + "computedHash": "c76fd4aa86ad557eee62aacbd4b9dd46499fe3913910e7d596d20d094296b984" + }, + "caveman-review": { + "source": "JuliusBrussee/caveman", + "sourceType": "github", + "skillPath": "skills/caveman-review/SKILL.md", + "computedHash": "fb7214a1c5793bae6ba8b1be4329e2e6f40dbec6dd911dfb335ad29f09c316a1" + }, + "caveman-stats": { + "source": "JuliusBrussee/caveman", + "sourceType": "github", + "skillPath": "skills/caveman-stats/SKILL.md", + "computedHash": "57c7db449641379e2afd1389fdb17cec8f33d9c8e25b1d28093937b4431895e8" + }, "code-review": { "source": "mattpocock/skills", "sourceType": "github", From a62331b618d66ebf338dbc1c721e61690e347e89 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Tue, 4 Aug 2026 23:44:01 +0800 Subject: [PATCH 4/7] chore: add cavecrew skill --- .agents/skills/cavecrew/README.md | 61 +++++++++++++++++++++++ .agents/skills/cavecrew/SKILL.md | 82 +++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 .agents/skills/cavecrew/README.md create mode 100644 .agents/skills/cavecrew/SKILL.md diff --git a/.agents/skills/cavecrew/README.md b/.agents/skills/cavecrew/README.md new file mode 100644 index 0000000..20bb07a --- /dev/null +++ b/.agents/skills/cavecrew/README.md @@ -0,0 +1,61 @@ +# cavecrew + +Decision guide. When to delegate to caveman subagents instead of doing the work inline. + +## What it does + +Tells the main thread when to spawn a caveman-style subagent versus the vanilla equivalent. The win: subagent tool-results inject back into main context verbatim, and caveman output is roughly 1/3 the size of vanilla prose. Across 20 delegations in one session, that is the difference between context exhaustion and finishing the task. + +Three subagents: + +| Subagent | Job | Use when | +|----------|-----|----------| +| `cavecrew-investigator` | Locate code (read-only) | "Where is X defined / what calls Y / list uses of Z" | +| `cavecrew-builder` | Surgical edit, 1-2 files | Scope is obvious, ≤2 files. Refuses 3+ file scope. | +| `cavecrew-reviewer` | Diff/file review | One-line findings with severity emoji | + +Use vanilla `Explore` or `Code Reviewer` when you want prose, architecture commentary, or rationale. Use main thread directly for one-line answers and 3+ file refactors. + +This skill is a decision guide, not a slash command. It activates when the conversation mentions delegation. + +## How to invoke + +Triggers on phrases like "delegate to subagent", "use cavecrew", "spawn investigator", "save context", "compressed agent output". + +## Example chaining + +Locate → fix → verify (most common): + +1. `cavecrew-investigator` returns site list (`path:line — symbol — note`) +2. Main thread picks 1-2 sites, hands paths to `cavecrew-builder` +3. `cavecrew-reviewer` audits the resulting diff + +Parallel scout: spawn 2-3 `cavecrew-investigator` calls in one message with different angles (defs, callers, tests). Aggregate in main. + +## Model overrides + +By default, `cavecrew-reviewer` and `cavecrew-investigator` pin `model: haiku` in their frontmatter; `cavecrew-builder` has no `model:` line (uses the API session default). Set env vars in your shell before launching Claude Code to override per-agent: + +| Env var | Agent | +|---|---| +| `CAVECREW_REVIEWER_MODEL` | `cavecrew-reviewer` | +| `CAVECREW_BUILDER_MODEL` | `cavecrew-builder` | +| `CAVECREW_INVESTIGATOR_MODEL` | `cavecrew-investigator` | + +Example — run reviewer on sonnet, keep others on default: + +```sh +export CAVECREW_REVIEWER_MODEL=sonnet +``` + +Use the same model name strings you'd use in any Claude Code agent frontmatter (e.g. `haiku`, `sonnet`, `opus`). + +Overrides patch only the `model:` line in the installed agent's frontmatter; the prompt body is untouched and keeps receiving upstream updates. Plugin installs only — standalone hook installs have no local agent files to patch. Unset or blank = no change. The patch persists in the installed file until the plugin is updated or reinstalled. + +## See also + +- [`SKILL.md`](./SKILL.md) — full decision matrix and output contracts +- [`agents/cavecrew-investigator.md`](../../agents/cavecrew-investigator.md) +- [`agents/cavecrew-builder.md`](../../agents/cavecrew-builder.md) +- [`agents/cavecrew-reviewer.md`](../../agents/cavecrew-reviewer.md) +- [Caveman README](../../README.md) — repo overview diff --git a/.agents/skills/cavecrew/SKILL.md b/.agents/skills/cavecrew/SKILL.md new file mode 100644 index 0000000..efa413f --- /dev/null +++ b/.agents/skills/cavecrew/SKILL.md @@ -0,0 +1,82 @@ +--- +name: cavecrew +description: > + Decision guide for delegating to caveman-style subagents. Tells the main + thread WHEN to spawn `cavecrew-investigator` (locate code), `cavecrew-builder` + (1-2 file edit), or `cavecrew-reviewer` (diff review) instead of doing the + work inline or using vanilla `Explore`. Subagent output is caveman-compressed + so the tool-result injected back into main context is ~60% smaller — main + context lasts longer across long sessions. + Trigger: "delegate to subagent", "use cavecrew", "spawn investigator/builder/reviewer", + "save context", "compressed agent output". +--- + +Cavecrew = three subagent presets that emit caveman output. Same job as Anthropic defaults (`Explore`, edit-style agents, reviewer); difference is the tool-result they return is compressed, so main context shrinks per delegation. + +## When to use cavecrew vs alternatives + +| Task | Use | +|---|---| +| "Where is X defined / what calls Y / list uses of Z" | `cavecrew-investigator` | +| Same but you also want suggestions/architecture commentary | `Explore` (vanilla) | +| Surgical edit, ≤2 files, scope obvious | `cavecrew-builder` | +| New feature / 3+ files / cross-cutting refactor | Main thread or `feature-dev:code-architect` | +| Review diff, branch, or file for bugs | `cavecrew-reviewer` | +| Deep code review with rationale + alternatives | `Code Reviewer` (vanilla) | +| One-line answer you already know | Main thread, no subagent | + +Rule of thumb: **if you'd want the subagent's output in 1/3 the tokens, pick cavecrew. If you'd want prose, pick vanilla.** + +## Why this exists (the real win) + +Subagent tool results get injected into main context verbatim. A vanilla `Explore` that returns 2k tokens of prose costs 2k tokens of main-context budget every time. The same finding from `cavecrew-investigator` returns ~700 tokens. Across 20 delegations in one session that's the difference between context exhaustion and finishing the task. + +## Output contracts + +What main thread can rely on per agent: + +**`cavecrew-investigator`** +``` +
: +- path:line — `symbol` — short note +totals: . +``` +Or `No match.` Always file-path-first, line-number-attached, backticked symbols. Safe to grep with `path:\d+`. + +**`cavecrew-builder`** +``` +. +verified: . +``` +Or one of: `too-big.` / `needs-confirm.` / `ambiguous.` / `regressed.` (terminal first token). + +**`cavecrew-reviewer`** +``` +path:line: : . . +totals: N🔴 N🟡 N🔵 N❓ +``` +Or `No issues.` Findings sorted file → line ascending. + +## Chaining patterns + +**Locate → fix → verify** (most common): +1. `cavecrew-investigator` returns site list. +2. Main thread picks 1-2 sites, hands paths to `cavecrew-builder`. +3. `cavecrew-reviewer` audits the diff. + +**Parallel scout** (when investigation is broad): +Spawn 2-3 `cavecrew-investigator` calls in one message (different angles: defs vs callers vs tests). Aggregate in main thread. + +**Single-shot edit** (when site is already known): +Skip investigator. Hand exact path:line to `cavecrew-builder` directly. + +## What NOT to do + +- Don't use `cavecrew-builder` when you don't already know the file. Spawn investigator first or main thread will eat tokens passing context. +- Don't chain `cavecrew-investigator → cavecrew-builder` for a 5-file refactor. Builder will return `too-big.` and you'll have wasted a turn. +- Don't ask `cavecrew-reviewer` for "general feedback" — it returns findings only, no architecture opinions. Use `Code Reviewer` for that. +- Don't expect prose. Cavecrew output is structured, sometimes terse to the point of cryptic. If a human will read it directly, paraphrase. + +## Auto-clarity (inherited) + +Subagents drop caveman → normal English for security warnings, irreversible-action confirmations, and any output where fragment ambiguity could be misread. Resume caveman after. From 24526cd37af1d961ec3b0389e9ec891d90153729 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Wed, 5 Aug 2026 16:13:44 +0800 Subject: [PATCH 5/7] refactor: unified structured compaction pipeline Replace compact_tier3/compact_tier4 with a single compact_messages() that self-gates on utilization (COMPACT_TRIGGER_PCT=0.70) and follows a graduated ladder for how many oldest messages to summarize. LLM output (and the sync fallback) render structured marker lines per ticket 02; sync fallback replaces the previous brute-force truncation. Retune sync tier thresholds (mask 0.70, collapse 0.75, compact 0.82) and run the compaction gate every iteration in loop_runner. --- src/agent_prompt.rs | 169 +++++++++++++++++++++- src/conversation.rs | 341 +++++++++++++++++++++++++++++++++++--------- src/loop_runner.rs | 5 +- 3 files changed, 443 insertions(+), 72 deletions(-) diff --git a/src/agent_prompt.rs b/src/agent_prompt.rs index dab4f58..6c38b7f 100644 --- a/src/agent_prompt.rs +++ b/src/agent_prompt.rs @@ -10,12 +10,104 @@ use crate::llm::{ChatMessage, MessageContent}; +/// A single compressed message, rendered as a structured marker line. +/// +/// Part of the public compaction API: the unified `compact_messages` pipeline +/// emits one `CompressedMessage` per summarized message so downstream readers +/// (and the LLM itself) can tell what kind of content was compressed. +#[derive(Debug, Clone)] +pub struct CompressedMessage { + pub role: String, + /// "tool_call" | "tool_result" | "user" | "assistant" | "system" + pub original_type: String, + pub summary: String, + pub key_data: Option, +} + +impl CompressedMessage { + /// Render this message as a structured marker line. Tool messages embed + /// `key_data` (e.g. counts/status codes) when present. + /// + /// Formats: + /// - `[Tool: NAME] description | result: SUMMARY | status: ok|error` + /// - `[User] TOPIC: SUMMARY` + /// - `[Assistant] ACTION: DECISION_SUMMARY` + /// - `[System] EVENT: NOTABLE_INFO` + pub fn to_marker(&self) -> String { + let summary = self.summary.trim(); + match self.original_type.as_str() { + "tool_call" | "tool_result" => { + let name = self + .key_data + .as_ref() + .and_then(|k| k.get("name")) + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + let description = self + .key_data + .as_ref() + .and_then(|k| k.get("description")) + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let status = if let Some(s) = self + .key_data + .as_ref() + .and_then(|k| k.get("status")) + .and_then(|v| v.as_str()) + { + s.to_string() + } else if summary.contains("Error") || summary.contains("error") { + "error".to_string() + } else { + "ok".to_string() + }; + let mut marker = + format!("[Tool: {name}] {description} | result: {summary} | status: {status}"); + if let Some(kd) = &self.key_data { + if let Ok(extra) = serde_json::to_string(kd) { + marker.push_str(&format!(" | {extra}")); + } + } + marker + } + "user" => format!("[User] TOPIC: {summary}"), + "assistant" => format!("[Assistant] ACTION: {summary}"), + "system" => format!("[System] EVENT: {summary}"), + other => format!("[{other}] {summary}"), + } + } +} + /// Percentage of context_window that triggers Tier 1 (observation masking). -const OBSERVATION_MASK_PCT: f64 = 0.20; +const OBSERVATION_MASK_PCT: f64 = 0.70; /// Percentage that triggers Tier 2 (context collapse). -const COLLAPSE_PCT: f64 = 0.60; +const COLLAPSE_PCT: f64 = 0.75; /// Percentage that triggers Tier 3 (auto compact). -pub const COMPACT_PCT: f64 = 0.85; +pub const COMPACT_PCT: f64 = 0.82; +/// Utilization fraction (total chars / context_window) at which the unified +/// compaction pipeline begins summarizing the oldest messages. +pub const COMPACT_TRIGGER_PCT: f64 = 0.70; +/// Graduated compression ladder: (trigger, fraction of oldest messages to compress). +pub const COMPACT_LADDER: [(f64, f64); 5] = [ + (0.70, 0.10), + (0.75, 0.25), + (0.82, 0.40), + (0.88, 0.55), + (0.93, 0.70), +]; + +/// Oldest-message fraction to compress for a given utilization. +/// +/// Returns the fraction from the largest ladder entry whose trigger is +/// `<= utilization`, or `0.0` when utilization is below the first trigger. +pub fn compact_fraction(utilization: f64) -> f64 { + for (trigger, fraction) in COMPACT_LADDER.iter().rev() { + if utilization >= *trigger { + return *fraction; + } + } + 0.0 +} /// Documentary threshold — Tier 4 is triggered by HTTP 413 errors, /// not by a percentage, but this documents the utilization level at /// which a 413 would typically occur. @@ -593,7 +685,7 @@ mod tests { #[test] fn observation_mask_replaces_old_tool_results_and_keeps_recent() { - let ctx = 2000; + let ctx = 1000; let mut messages = Vec::new(); messages.push(ChatMessage { @@ -977,4 +1069,73 @@ mod tests { let text = msg.content.as_ref().unwrap().as_text(); assert!(text.contains("state"), "Should hint at state format"); } + + #[test] + fn compact_fraction_ladder() { + let approx = |a: f64, b: f64| (a - b).abs() < 1e-9; + assert!(approx(compact_fraction(0.69), 0.0)); + assert!(approx(compact_fraction(0.70), 0.10)); + assert!(approx(compact_fraction(0.80), 0.25)); + assert!(approx(compact_fraction(0.90), 0.55)); + assert!(approx(compact_fraction(0.99), 0.70)); + // Saturates at the largest ladder entry. + assert!(approx(compact_fraction(2.0), 0.70)); + } + + #[test] + fn compressed_message_to_marker_formats() { + let tool = CompressedMessage { + role: "tool".to_string(), + original_type: "tool_result".to_string(), + summary: "wrote config file".to_string(), + key_data: Some(serde_json::json!({"name": "write_file", "bytes": 42})), + }; + let tool_marker = tool.to_marker(); + assert!( + tool_marker.starts_with("[Tool: write_file]"), + "{}", + tool_marker + ); + assert!(tool_marker.contains("| status: ok"), "{}", tool_marker); + assert!(tool_marker.contains("\"bytes\":42"), "{}", tool_marker); + + let err_tool = CompressedMessage { + role: "tool".to_string(), + original_type: "tool_result".to_string(), + summary: "Error: permission denied".to_string(), + key_data: None, + }; + assert!( + err_tool.to_marker().contains("| status: error"), + "{}", + err_tool.to_marker() + ); + + let user = CompressedMessage { + role: "user".to_string(), + original_type: "user".to_string(), + summary: "fix the bug".to_string(), + key_data: None, + }; + assert_eq!(user.to_marker(), "[User] TOPIC: fix the bug"); + + let assistant = CompressedMessage { + role: "assistant".to_string(), + original_type: "assistant".to_string(), + summary: "decided to use tokio".to_string(), + key_data: None, + }; + assert_eq!( + assistant.to_marker(), + "[Assistant] ACTION: decided to use tokio" + ); + + let system = CompressedMessage { + role: "system".to_string(), + original_type: "system".to_string(), + summary: "sandbox dir set".to_string(), + key_data: None, + }; + assert_eq!(system.to_marker(), "[System] EVENT: sandbox dir set"); + } } diff --git a/src/conversation.rs b/src/conversation.rs index d2d18b8..497510a 100644 --- a/src/conversation.rs +++ b/src/conversation.rs @@ -113,54 +113,82 @@ impl ConversationManager { self.messages.push(steer_msg); } - pub async fn compact_tier3(&mut self, context_window: usize) { + /// Unified compaction pipeline: compress the oldest messages once total + /// utilization crosses `agent_prompt::COMPACT_TRIGGER_PCT` of the context + /// window. The fraction of oldest messages summarized follows the graduated + /// `agent_prompt::COMPACT_LADDER`; the system message (index 0) and the + /// newest 8 messages always stay verbatim. + /// + /// Summarization is attempted via the LLM and rendered as structured marker + /// lines; on LLM failure a sync structured extraction is used instead (no + /// LLM call). Returns `Ok(true)` when compaction happened, `Ok(false)` when + /// nothing was summarized. + pub async fn compact_messages( + &mut self, + llm: &LlmClient, + context_window: usize, + ) -> Result { let total: usize = self .messages .iter() .map(|m| m.content.as_ref().map(|c| c.as_text().len()).unwrap_or(0)) .sum(); - if total > context_window / 2 { - let system = self.messages.first().cloned(); - let recent: Vec = self - .messages - .iter() - .skip(1) - .rev() - .take(20) - .cloned() - .collect(); - let mut trimmed = Vec::new(); - if let Some(sys) = system { - trimmed.push(sys); - } - trimmed.extend(recent.into_iter().rev()); - self.messages = trimmed; + let utilization = total as f64 / context_window as f64; + if utilization < crate::agent_prompt::COMPACT_TRIGGER_PCT { + return Ok(false); } - } - pub async fn compact_tier4(&mut self, llm: &LlmClient, context_window: usize) -> Result { - let total: usize = self - .messages - .iter() - .map(|m| m.content.as_ref().map(|c| c.as_text().len()).unwrap_or(0)) - .sum(); - if total <= context_window / 2 { + // Graduated fraction of the oldest (non-system) messages to compress, + // clamped so at most len-9 messages are summarized (system + newest 8 + // stay verbatim). + let fraction = crate::agent_prompt::compact_fraction(utilization); + let max_summarize = self.messages.len().saturating_sub(9); + let summarize_count = ((self.messages.len().saturating_sub(1)) as f64 * fraction) as usize; + let summarize_count = summarize_count.min(max_summarize); + if summarize_count == 0 { return Ok(false); } - let system = self.messages.first().cloned(); - let keep_count = 10.min(self.messages.len().saturating_sub(1)); - let keep_from = self.messages.len().saturating_sub(keep_count); - let to_summarize: Vec<&ChatMessage> = self - .messages - .iter() - .skip(1) - .take(keep_from.saturating_sub(1)) - .collect(); - if to_summarize.is_empty() { - return Ok(false); + let to_summarize: Vec<&ChatMessage> = + self.messages.iter().skip(1).take(summarize_count).collect(); + let preserved_tail_start = summarize_count + 1; + + let summary_text = match self.summarize_with_llm(llm, &to_summarize).await { + Ok(text) => text, + Err(e) => { + tracing::warn!("LLM compaction failed ({e}); using sync structured summary"); + self.build_sync_summary(&to_summarize) + } + }; + + let heading = format!( + "★ COMPACTED CONTEXT — {} messages summarized ★\n", + to_summarize.len() + ); + let summary_entry = ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!("{heading}{summary_text}"))), + tool_calls: None, + tool_call_id: Some("summary".to_string()), + }; + + let mut new_msgs = + Vec::with_capacity(2 + self.messages.len().saturating_sub(preserved_tail_start)); + if let Some(system) = self.messages.first().cloned() { + new_msgs.push(system); } + new_msgs.push(summary_entry); + new_msgs.extend(self.messages.iter().skip(preserved_tail_start).cloned()); + self.messages = new_msgs; + Ok(true) + } + /// Ask the LLM to summarize `to_summarize` as structured marker lines. + async fn summarize_with_llm( + &self, + llm: &LlmClient, + to_summarize: &[&ChatMessage], + ) -> Result { let summary_text: String = to_summarize .iter() .map(|m| { @@ -173,7 +201,15 @@ impl ConversationManager { .collect::>() .join("\n"); - let summary_prompt = format!("Summarize the following conversation, preserving key decisions and facts:\n\n{summary_text}"); + let summary_prompt = format!( + "Summarize the following conversation, preserving key decisions and facts.\n\ + Output the summary AS STRUCTURED MARKER LINES, one per message, in exactly these formats:\n\ + [Tool: NAME] description | result: SUMMARY | status: ok|error\n\ + [User] TOPIC: SUMMARY\n\ + [Assistant] ACTION: DECISION_SUMMARY\n\ + [System] EVENT: NOTABLE_INFO\n\n\ + Conversation:\n{summary_text}" + ); let summary_msg = vec![ ChatMessage { role: "system".to_string(), @@ -191,36 +227,43 @@ impl ConversationManager { }, ]; - match llm.chat(&summary_msg, &[]).await { - Ok(summary) => { - let summary_entry = ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text(format!( - "[Previous conversation summarized: {}]", - summary - .content - .as_ref() - .map(|c| c.as_text()) - .unwrap_or_default() - ))), - tool_calls: None, - tool_call_id: Some("summary".to_string()), - }; - let mut new_msgs = Vec::new(); - if let Some(sys) = system { - new_msgs.push(sys); - } - new_msgs.push(summary_entry); - new_msgs.extend(self.messages.iter().skip(keep_from).cloned()); - self.messages = new_msgs; - Ok(true) - } - Err(e) => { - tracing::warn!("Compaction tier 4 failed: {e}"); - self.compact_tier3(context_window).await; - Ok(false) + let response = llm.chat(&summary_msg, &[]).await?; + Ok(response + .content + .as_ref() + .map(|c| c.as_text()) + .unwrap_or_default()) + } + + /// Sync fallback: build structured marker lines from `to_summarize` + /// without any LLM call. + fn build_sync_summary(&self, to_summarize: &[&ChatMessage]) -> String { + const MAX_CHARS: usize = 200; + let mut lines: Vec = Vec::new(); + for m in to_summarize { + let text = m.content.as_ref().map(|c| c.as_text()).unwrap_or_default(); + if text.is_empty() { + continue; } + let truncated: String = text.chars().take(MAX_CHARS).collect(); + let marker = match m.role.as_str() { + "user" => format!("[User] TOPIC: {truncated}"), + "assistant" => format!("[Assistant] ACTION: {truncated}"), + "tool" => { + let id = m.tool_call_id.as_deref().unwrap_or("unknown"); + let status = if text.contains("Error") || text.contains("error") { + "error" + } else { + "ok" + }; + format!("[Tool: {id}] result: {truncated} | status: {status}") + } + "system" => format!("[System] EVENT: {truncated}"), + _ => format!("{}: {truncated}", m.role), + }; + lines.push(marker); } + lines.join("\n") } pub fn prepare(&self, context_window: usize) -> crate::agent_prompt::PreparedPrompt { @@ -235,3 +278,171 @@ impl ConversationManager { self.messages } } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + use std::sync::Arc; + + use crate::config::ProviderType; + use crate::provider::{OpenRouterProvider, ProviderConfig, ProviderRegistry}; + + /// A provider that always fails: empty base_url makes the request a + /// relative URL, so reqwest errors out without touching the network. + fn failing_llm() -> LlmClient { + let config = ProviderConfig { + name: "test".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: String::new(), + api_key: None, + default_model: "test-model".to_string(), + supports_vision: false, + max_tokens: 100, + discover_models: false, + context_window: 4096, + context_window_cache: Arc::new(tokio::sync::RwLock::new(None)), + parse_retry_limit: 0, + }; + let provider: Arc = + Arc::new(OpenRouterProvider::new(config)); + let mut providers = HashMap::new(); + providers.insert("test".to_string(), provider); + LlmClient::new(Arc::new(ProviderRegistry::new( + providers, + "test".to_string(), + ))) + } + + fn manager(messages: Vec) -> ConversationManager { + ConversationManager { + messages, + system_prompt: String::new(), + memory: crate::memory::MemoryStore::open_in_memory().unwrap(), + conversation_id: String::new(), + } + } + + #[tokio::test] + async fn compact_messages_noop_below_threshold() { + let mut cm = manager(vec![ + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("hi".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("how are you".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ]); + let texts = |cm: &ConversationManager| -> Vec<(String, String)> { + cm.messages + .iter() + .map(|m| { + ( + m.role.clone(), + m.content.as_ref().map(|c| c.as_text()).unwrap_or_default(), + ) + }) + .collect() + }; + let before = texts(&cm); + let llm = failing_llm(); + + let result = cm.compact_messages(&llm, 100_000).await.unwrap(); + assert!(!result, "tiny conversation must not trigger compaction"); + assert_eq!(texts(&cm), before, "messages must be unchanged"); + } + + #[tokio::test] + async fn compact_messages_sync_fallback_produces_markers() { + let mut messages = vec![ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("system prompt".to_string())), + tool_calls: None, + tool_call_id: None, + }]; + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "user question {} {}", + "x".repeat(90), + 0 + ))), + tool_calls: None, + tool_call_id: None, + }); + for i in 0..10 { + messages.push(ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text(format!( + "assistant reply {} {}", + "x".repeat(80), + i + ))), + tool_calls: None, + tool_call_id: None, + }); + let result = if i == 3 { + format!("Error: file not found {}", "y".repeat(70)) + } else { + format!("tool result {} {}", "y".repeat(80), i) + }; + messages.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text(result)), + tool_calls: None, + tool_call_id: Some(format!("tool_{i}")), + }); + } + let last_content = messages.last().unwrap().content.clone(); + let mut cm = manager(messages); + let llm = failing_llm(); + + let result = cm.compact_messages(&llm, 1_000).await.unwrap(); + assert!(result, "LLM failure must still compact via sync fallback"); + + // system + 1 summary entry + newest 8 preserved + assert_eq!(cm.messages.len(), 10); + assert_eq!(cm.messages[0].role, "system"); + assert_eq!(cm.messages[1].role, "user"); + assert_eq!(cm.messages[1].tool_call_id.as_deref(), Some("summary")); + + let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); + assert!( + summary_text.contains("★ COMPACTED CONTEXT — 13 messages summarized ★"), + "missing heading: {}", + summary_text + ); + assert!(summary_text.contains("[User] TOPIC:"), "{summary_text}"); + assert!( + summary_text.contains("[Assistant] ACTION:"), + "{summary_text}" + ); + assert!(summary_text.contains("[Tool: tool_3]"), "{summary_text}"); + assert!(summary_text.contains("| status: error"), "{summary_text}"); + + // Preserved tail: newest 8 messages verbatim, in order. + assert_eq!( + cm.messages + .last() + .unwrap() + .content + .as_ref() + .map(|c| c.as_text()), + last_content.as_ref().map(|c| c.as_text()) + ); + assert_eq!(cm.messages[2].role, "assistant"); + assert!(cm.messages[2].content.as_ref().unwrap().as_text().len() > 50); + } +} diff --git a/src/loop_runner.rs b/src/loop_runner.rs index 2f4ad7e..9556d74 100644 --- a/src/loop_runner.rs +++ b/src/loop_runner.rs @@ -104,10 +104,9 @@ impl<'a> AgenticLoop<'a> { } } - if self.config.compaction_enabled && iteration > 0 && iteration % 5 == 0 { + if self.config.compaction_enabled && iteration > 0 { if let MessageContainer::Conversation(cm) = messages { - cm.compact_tier3(context_window).await; - let _ = cm.compact_tier4(self.llm, context_window).await; + let _ = cm.compact_messages(self.llm, context_window).await; } } From 6715f569fc98ae354622d67e3846f6e1a53de4f0 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Wed, 5 Aug 2026 17:01:53 +0800 Subject: [PATCH 6/7] fix: tool-group boundary and marker consistency in compaction Fix review findings on unified compaction: - Round summarized range down to tool-call/tool-result pair boundaries so a call is never separated from its result across the preserve boundary - Clamp preserved tail to system + newest 8 verbatim - Sync fallback labels tool calls as [Tool: NAME] with real function name, rendered through CompressedMessage::to_marker() (single format source) - Guard zero context_window; reinstate compaction turn gap in loop_runner - Add test asserting a tool-call pair is never split at the boundary --- src/agent_prompt.rs | 19 +++- src/conversation.rs | 245 +++++++++++++++++++++++++++++++++++++++----- src/loop_runner.rs | 11 +- 3 files changed, 249 insertions(+), 26 deletions(-) diff --git a/src/agent_prompt.rs b/src/agent_prompt.rs index 6c38b7f..ff60fcf 100644 --- a/src/agent_prompt.rs +++ b/src/agent_prompt.rs @@ -36,7 +36,24 @@ impl CompressedMessage { pub fn to_marker(&self) -> String { let summary = self.summary.trim(); match self.original_type.as_str() { - "tool_call" | "tool_result" => { + "tool_call" => { + let name = self + .key_data + .as_ref() + .and_then(|k| k.get("name")) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .or_else(|| (!summary.is_empty()).then_some(summary)) + .unwrap_or("unknown"); + let args = self + .key_data + .as_ref() + .and_then(|k| k.get("args")) + .and_then(|v| v.as_str()) + .unwrap_or_default(); + format!("[Tool: {name}] {args}").trim_end().to_string() + } + "tool_result" => { let name = self .key_data .as_ref() diff --git a/src/conversation.rs b/src/conversation.rs index 497510a..1d8cf5a 100644 --- a/src/conversation.rs +++ b/src/conversation.rs @@ -1,11 +1,12 @@ use anyhow::Result; -use crate::agent_prompt::prepare_messages_for_llm; +use crate::agent_prompt::{prepare_messages_for_llm, CompressedMessage}; use crate::config::Config; use crate::llm::{ChatMessage, ContentPart, LlmClient, MessageContent}; use crate::memory::MemoryStore; use crate::platform::IncomingMessage; use crate::skills::SkillRegistry; +use std::collections::HashMap; pub struct ConversationManager { messages: Vec, @@ -128,6 +129,9 @@ impl ConversationManager { llm: &LlmClient, context_window: usize, ) -> Result { + if context_window == 0 { + return Ok(false); + } let total: usize = self .messages .iter() @@ -143,15 +147,70 @@ impl ConversationManager { // stay verbatim). let fraction = crate::agent_prompt::compact_fraction(utilization); let max_summarize = self.messages.len().saturating_sub(9); - let summarize_count = ((self.messages.len().saturating_sub(1)) as f64 * fraction) as usize; - let summarize_count = summarize_count.min(max_summarize); + let mut summarize_count = + ((self.messages.len().saturating_sub(1)) as f64 * fraction) as usize; + summarize_count = summarize_count.min(max_summarize); + if summarize_count == 0 { + return Ok(false); + } + + // Round the summarized range down to tool-group boundaries so a + // [tool_call, tool_result] pair is never split across the + // summarize/preserve boundary. + let mut i = 1usize; + while i <= summarize_count { + let msg = &self.messages[i]; + if msg.has_tool_calls() { + let call_ids: Vec<&str> = msg + .tool_calls + .iter() + .flatten() + .map(|c| c.id.as_str()) + .collect(); + let mut last_result: Option = None; + for j in (i + 1)..self.messages.len() { + let m = &self.messages[j]; + if m.role == "tool" + && m.tool_call_id + .as_deref() + .is_some_and(|id| call_ids.contains(&id)) + { + last_result = Some(j); + } else if m.role != "tool" { + break; + } + } + if let Some(r) = last_result { + if r > summarize_count { + summarize_count = r; + } + } + } else if msg.role == "tool" { + // A result inside the range whose matching call is preserved + // in the tail: stop the range before this result. + let call_in_tail = self.messages[(summarize_count + 1)..].iter().any(|m| { + m.has_tool_calls() + && m.tool_calls.as_ref().is_some_and(|calls| { + calls + .iter() + .any(|c| Some(c.id.as_str()) == msg.tool_call_id.as_deref()) + }) + }); + if call_in_tail { + summarize_count = i.saturating_sub(1); + break; + } + } + i += 1; + } if summarize_count == 0 { return Ok(false); } + // System (index 0) and the newest 8 messages stay verbatim. + let preserved_tail_start = (self.messages.len().saturating_sub(8)).max(summarize_count + 1); let to_summarize: Vec<&ChatMessage> = self.messages.iter().skip(1).take(summarize_count).collect(); - let preserved_tail_start = summarize_count + 1; let summary_text = match self.summarize_with_llm(llm, &to_summarize).await { Ok(text) => text, @@ -239,31 +298,70 @@ impl ConversationManager { /// without any LLM call. fn build_sync_summary(&self, to_summarize: &[&ChatMessage]) -> String { const MAX_CHARS: usize = 200; - let mut lines: Vec = Vec::new(); + // Resolve tool identity by NAME: map tool_call_id -> function name so + // tool results can render the call's name instead of the raw id. + let mut tool_names: HashMap<&str, &str> = HashMap::new(); + for m in &self.messages { + if let Some(calls) = &m.tool_calls { + for c in calls { + tool_names.entry(c.id.as_str()).or_insert(&c.function.name); + } + } + } + + let mut compacted: Vec = Vec::new(); for m in to_summarize { let text = m.content.as_ref().map(|c| c.as_text()).unwrap_or_default(); - if text.is_empty() { + if text.is_empty() && !m.has_tool_calls() { continue; } let truncated: String = text.chars().take(MAX_CHARS).collect(); - let marker = match m.role.as_str() { - "user" => format!("[User] TOPIC: {truncated}"), - "assistant" => format!("[Assistant] ACTION: {truncated}"), - "tool" => { - let id = m.tool_call_id.as_deref().unwrap_or("unknown"); - let status = if text.contains("Error") || text.contains("error") { - "error" - } else { - "ok" - }; - format!("[Tool: {id}] result: {truncated} | status: {status}") - } - "system" => format!("[System] EVENT: {truncated}"), - _ => format!("{}: {truncated}", m.role), - }; - lines.push(marker); + if m.has_tool_calls() { + // Assistant tool-call message -> [Tool: NAME] marker. + let call = m + .tool_calls + .iter() + .flatten() + .next() + .expect("has_tool_calls checked"); + let name = call.function.name.clone(); + compacted.push(CompressedMessage { + role: m.role.clone(), + original_type: "tool_call".to_string(), + summary: name.clone(), + key_data: Some(serde_json::json!({ + "name": name, + "args": call.function.arguments.chars().take(MAX_CHARS).collect::(), + })), + }); + } else if m.role == "tool" { + let id = m.tool_call_id.as_deref().unwrap_or("unknown"); + let name = tool_names.get(id).copied().unwrap_or(id).to_string(); + let status = if text.contains("Error") || text.contains("error") { + "error" + } else { + "ok" + }; + compacted.push(CompressedMessage { + role: m.role.clone(), + original_type: "tool_result".to_string(), + summary: truncated, + key_data: Some(serde_json::json!({"name": name, "status": status})), + }); + } else { + compacted.push(CompressedMessage { + role: m.role.clone(), + original_type: m.role.clone(), + summary: truncated, + key_data: None, + }); + } } - lines.join("\n") + compacted + .iter() + .map(|c| c.to_marker()) + .collect::>() + .join("\n") } pub fn prepare(&self, context_window: usize) -> crate::agent_prompt::PreparedPrompt { @@ -323,6 +421,107 @@ mod tests { } } + #[tokio::test] + async fn compact_messages_never_splits_tool_pair() { + use crate::llm::{FunctionCall, ToolCall}; + + let mut messages = vec![ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("system prompt".to_string())), + tool_calls: None, + tool_call_id: None, + }]; + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "first request {}", + "x".repeat(100) + ))), + tool_calls: None, + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "second request {}", + "x".repeat(100) + ))), + tool_calls: None, + tool_call_id: None, + }); + // idx 3: assistant with tool call — the naive boundary (len-9 = 3) + // lands exactly here, which would orphan the result below. + messages.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![ToolCall { + id: "call_split".to_string(), + call_type: "function".to_string(), + function: FunctionCall { + name: "lookup_thing".to_string(), + arguments: r#"{"query":"x"}"#.to_string(), + }, + }]), + tool_call_id: None, + }); + // idx 4: matching tool result — naive range summarizes the call but + // preserves the result, orphaning the pair. + messages.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("lookup result payload".to_string())), + tool_calls: None, + tool_call_id: Some("call_split".to_string()), + }); + for i in 0..7 { + messages.push(ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text(format!( + "filler {} {}", + "y".repeat(120), + i + ))), + tool_calls: None, + tool_call_id: None, + }); + } + // len == 12: naive summarize_count = min(11 * 0.70, 12 - 9) = 3 → mid-pair. + let mut cm = manager(messages); + let llm = failing_llm(); + + let result = cm.compact_messages(&llm, 1_000).await.unwrap(); + assert!(result, "compaction must happen"); + + let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); + // Tool call rendered by NAME, not as [Assistant] ACTION. + assert!( + summary_text.contains("[Tool: lookup_thing]"), + "missing tool-call marker: {summary_text}" + ); + // The pair was summarized together: the result is in the summary. + assert!( + summary_text.contains("lookup result payload"), + "tool result must be summarized with its call: {summary_text}" + ); + // Preserved tail: no orphaned tool result or call for call_split. + for m in &cm.messages[2..] { + assert_ne!( + m.tool_call_id.as_deref(), + Some("call_split"), + "preserved tail must not contain a tool result whose call was summarized" + ); + assert!( + !m.has_tool_calls() + || !m + .tool_calls + .as_ref() + .unwrap() + .iter() + .any(|c| c.id == "call_split"), + "preserved tail must not contain a summarized tool call" + ); + } + } + #[tokio::test] async fn compact_messages_noop_below_threshold() { let mut cm = manager(vec![ diff --git a/src/loop_runner.rs b/src/loop_runner.rs index 9556d74..453c645 100644 --- a/src/loop_runner.rs +++ b/src/loop_runner.rs @@ -96,6 +96,7 @@ impl<'a> AgenticLoop<'a> { ) -> Result { let context_window = 128_000; let mut empty_count = 0u32; + let mut last_compact_turn = 0usize; for iteration in 0..self.config.max_iterations { if let Some(ref cancel) = self.cancel { @@ -104,9 +105,15 @@ impl<'a> AgenticLoop<'a> { } } - if self.config.compaction_enabled && iteration > 0 { + if self.config.compaction_enabled + && iteration > 0 + && ((iteration as usize).saturating_sub(last_compact_turn) >= 5 + || last_compact_turn == 0) + { if let MessageContainer::Conversation(cm) = messages { - let _ = cm.compact_messages(self.llm, context_window).await; + if let Ok(true) = cm.compact_messages(self.llm, context_window).await { + last_compact_turn = iteration as usize; + } } } From 595aa354c02d95518b4f55a986cfc0fcf2e1b52e Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Fri, 7 Aug 2026 09:08:05 +0800 Subject: [PATCH 7/7] refactor(conversation): redesign compaction as per-turn summary layer Compaction ran inside the agent loop and dropped user intent after summarizing. New design: run once per user turn in process_message before the loop. Protected tail keeps last user turns + active exchange; layered summary injected as system message; [SUMMARY] rows persisted in DB, legacy summary-role entries filtered on load. CJK-aware estimate_tokens triggers at 85% of provider window. Defer-on-failure with emergency hard-cap, no truncation. New compaction_model config override; dead compaction machinery (ConversationMeta, COMPACT_LADDER, should_auto_compact) removed. Refs docs/adr/0003-conversation-compaction-redesign.md --- config.example.toml | 2 +- .../0003-conversation-compaction-redesign.md | 131 ++ ...-08-06-conversation-compaction-redesign.md | 1824 +++++++++++++++++ src/agent.rs | 33 +- src/agent_prompt.rs | 573 ++++-- src/config.rs | 5 + src/conversation.rs | 813 +++++--- src/learning.rs | 190 +- src/loop_runner.rs | 19 +- tests/compaction_preserves_user_intent.rs | 157 ++ 10 files changed, 3152 insertions(+), 595 deletions(-) create mode 100644 docs/adr/0003-conversation-compaction-redesign.md create mode 100644 docs/superpowers/plans/2026-08-06-conversation-compaction-redesign.md create mode 100644 tests/compaction_preserves_user_intent.rs diff --git a/config.example.toml b/config.example.toml index 223bcb7..b3a519c 100644 --- a/config.example.toml +++ b/config.example.toml @@ -249,8 +249,8 @@ Be concise and helpful.""" # ── Self-Learning (optional; defaults apply if section omitted) ───────────── # [learning] -# user_model_path = "memory/USER.md" # Honcho-style user model file # skill_extraction_enabled = true # Auto-generate skills from tool-heavy tasks # skill_extraction_threshold = 5 # Min tool calls to trigger extraction # user_model_update_interval = 10 # Update user model every N user messages # user_model_cron = "0 0 3 * * SUN" # Weekly user model refresh (6-field cron) +# compaction_model = "qwen/qwen3-8b" # Cheaper model for compaction summary + USER.md flush (default: current model) diff --git a/docs/adr/0003-conversation-compaction-redesign.md b/docs/adr/0003-conversation-compaction-redesign.md new file mode 100644 index 0000000..a001077 --- /dev/null +++ b/docs/adr/0003-conversation-compaction-redesign.md @@ -0,0 +1,131 @@ +# ADR 0003: Conversation Compaction Redesign + +## Status +Accepted (implemented) + +## Date +2026-08-06 + +## Context + +The agent "forgets what the user asked" and answers follow-up questions with +predictions of what the user "is going to ask". Root cause analysis: + +- `compact_messages` (src/conversation.rs) triggers on raw character count + against a hardcoded `context_window = 128_000` (src/loop_runner.rs:97), + excluding tool-call arguments. 128K chars ≈ 22K tokens ≈ 17% of a real + 128K-token window — compaction fires far too early. +- The preserved tail is "newest 8 messages" (conversation.rs:211), not the + last user turn. The oldest non-system message — usually the user's current + request — is summarized first, mid-task, because compaction runs inside the + agent loop gated by loop-iteration gap (loop_runner.rs:108-118). +- The summary is injected as `role: "user"` with `tool_call_id: "summary"` + (conversation.rs:227-232). The model treats the compacted history as the + current user turn and answers its content — hence "I know what you're going + to ask". +- Each compaction re-summarizes raw history from scratch; the previous summary + is itself re-summarized next pass → cumulative information loss. +- Sync fallback truncates every message to 200 chars (conversation.rs:300) — + deletion, not summarization. + +Industry research (Claude Code 4-tier cascade + compaction sub-agent, +OpenCode prune-then-summarize + nested compression, OpenClaw compaction vs +pruning, Codex session-memory/server-side compact, LangChain/LangGraph +summarize-then-extend): all converge on token-based triggers with a reserve +buffer, running/nested summaries, system-role injection, compaction at turn +boundaries, and full-history persistence. + +## Decision + +### Q1 — Compaction cadence (Accepted) +Move compaction out of the agent loop. Routine compaction runs **once per +user turn** in `process_message` (after `add_user_turn`, before the loop). +Long-running tool loops are handled by delegation first, then by a +threshold-triggered compact as last resort — never by a per-iteration check. +The loop keeps Tier 1/2 masking as an emergency safety net only. +`ConversationMeta`/`should_auto_compact`/loop compaction plumbing becomes dead +code and is deleted. + +### Q2 — Summary representation (Accepted) +Adopt a **running summary** carried on `ConversationManager` (`summary: +Option`). Each compaction EXTENDS the previous summary ("Extend the +previous summary with the new messages above"), layered under a +"Previously compacted context" heading. Injected as a **system message** +before history, never as a user-role message. The per-message marker-line +format is retired to the sync fallback only. + +### Q3 — Trigger metric (Accepted) +Token-based trigger at **85% of the real provider window**, not chars: +`estimate_tokens = (latin + other chars) / 4 + CJK chars × 1`. The window +comes from `ProviderConfig.context_window` via `registry.resolve_model( +current_model)`, replacing the hardcoded `128_000`. Estimation unifies with +`estimate_prompt_bytes` (which already counts tool args) — a single +`estimate_tokens` used for both trigger and prompt budget. The +`COMPACT_TRIGGER_PCT 0.70` / `OBSERVATION_MASK_PCT` / `COLLAPSE_PCT` char +ladder is removed. + +### Q4 — Preserve policy (Accepted) +The protected verbatim zone is the **latest user intent**: the last **two** +user turns' user messages verbatim plus the active exchange (last user +message → end). Tool traffic in older turns is summarized, never kept raw. +The preserved tail is capped at ≤ 20% of the window. There is no +first-request anchor — the running summary carries the original request +forward, per assistant (not coding-task) semantics. + +### Q5 — Durable memory flush (Accepted) +Before the running summary is written, one flush turn extracts durable facts +(preferences, standing intents, project state) from the to-be-summarized +range and writes them to **USER.md** (home root) — not a new internal file. +`user_model.md` is legacy; `config.rs` already migrates it to `USER.md`. +USER.md wins because it is injected into the system prompt every message +(agent.rs:309), has the validated write path in `learning.rs` +(frontmatter check prevents prompt injection, `.bak` backup before +overwrite, merge-not-remove, 500-word cap), is agent-editable via +`update_soul_file`, and is already cron-updated weekly. Implementation: +refactor `update_user_model_inner` (learning.rs:489) to accept snippets as +a parameter; the flush passes the to-be-summarized range, the cron keeps +passing `search_messages` results. Same validation + backup + write tail +shared. + +### Q6 — Flush gating (Accepted) +Run the flush only when the summarized range contains ≥ 1 **user-authored** +message (tool traffic alone cannot contain durable facts), and skip when the +range was already covered by a recent flush — `last_flush_turn` tracked on +`ConversationManager`. + +### Q7 — Sync fallback (Accepted, with change) +The 200-char-per-message truncation (conversation.rs:300) is removed. Fallback +chain: +1. Summary succeeds → running-summary compact. +2. Summary fails → **defer**: skip this turn's compact; the 85% trigger + leaves 15% slack; retry next turn. +3. Hard ceiling hit (emergency mask) → **oldest-first truncation of + non-protected messages only** — the protected tail (last 2 user turns + + active exchange) is never touched; dropped traffic is replaced by a + one-line marker (precedent: Codex v0.118 no-LLM session-memory compact). + +Every summary failure is logged (`warn!`) with reason, model, and message +range so failures are visible without being user-facing noise. + +### Q8 — Running-summary persistence (Accepted) +The running summary is persisted in the database, keyed by conversation id +(e.g. a `summary` role row in the existing message store). Reloaded in the +conversation load path; extension continues seamlessly across restarts. +`ConversationManager` remains the in-memory view; the DB row is the +source of truth. + +### Q9 — Summarizer model (Accepted) +New config key `compaction_model` (empty default = current model). Summary +and flush turns run on the configured model via `registry.resolve_model`, +letting users pin a cheap fast model. Precedent: OpenClaw `compaction.model`, +Claude Code compaction sub-agent. + +## Decision complete — ready for implementation planning + +## Consequences +- The active user request and recent turns survive compaction verbatim. +- The model never sees compacted history as its current user turn. +- Information loss per compaction is bounded (layered extension, not + re-summarization). +- Compaction cost moves off the hot path; each user turn pays at most one + summarizer call. diff --git a/docs/superpowers/plans/2026-08-06-conversation-compaction-redesign.md b/docs/superpowers/plans/2026-08-06-conversation-compaction-redesign.md new file mode 100644 index 0000000..c55f9b0 --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-conversation-compaction-redesign.md @@ -0,0 +1,1824 @@ +# Conversation Compaction Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix RustFox forgetting user requests on compaction — move compaction to per-user-turn, adopt a persisted running summary with a protected "latest intent" tail, token-based 85% trigger, USER.md memory flush, and a defer-don't-truncate fallback (ADR 0003, Q1–Q9). + +**Architecture:** `ConversationManager` gains `summary: Option` + `last_flush_turn: Option`; `compact_messages(ctx)` runs once per user turn from `process_message` (after `add_user_turn`, before the loop). The protected tail (last 2 user turns + active exchange, ≤20% of window, never mid-tool-pair) is selected by a pure `protected_tail_start` fn. Summary layers extend the running summary, are injected as a system message, and persist as `[SUMMARY]` rows (existing convention — `load_messages_with_limit` already loads them first). A flush turn banks durable facts into USER.md (reusing `learning.rs` machinery). On summary failure: defer with a `warn!` log; the emergency mask in `prepare_messages_for_llm` drops oldest non-protected messages only. + +**Tech Stack:** Rust (edition 2021), tokio, anyhow, serde, rusqlite (MemoryStore), teloxide. Tests: in-crate `#[cfg(test)]` + `tests/` integration. + +--- + +## File map + +| File | Change | +|------|--------| +| `src/agent_prompt.rs` | Add `estimate_tokens` (CJK-aware), `protected_tail_start`; change `COMPACT_TRIGGER_PCT` to 0.85; remove `COMPACT_LADDER`/`compact_fraction`/`ConversationMeta`/`should_auto_compact`/`COMPACT_TURN_GAP`; rework hard-cap branch of `prepare_messages_for_llm` | +| `src/conversation.rs` | `ConversationManager` gains `summary`, `last_flush_turn`; load-time `[SUMMARY]` folding; `CompactionContext`; rewrite `compact_messages` (flush → summarize → apply); add `apply_summary_layer`, `should_flush`; remove `summarize_with_llm` marker prompt, `build_sync_summary`, `MAX_CHARS` | +| `src/learning.rs` | Extract `build_user_model_prompt`, `write_user_model_with_backup`, `write_user_model_from_snippets`; add `flush_user_model` | +| `src/config.rs` | `LearningConfig.compaction_model: Option` | +| `config.example.toml` | Document `compaction_model`; fix stale `user_model_path` comment | +| `src/loop_runner.rs` | Remove per-iteration compaction block + `compaction_enabled`; `LoopConfig` gains `context_window` | +| `src/agent.rs` | Wire per-turn compaction after `add_user_turn`; delete `ConversationMeta` line/import; pass real context window to loop | +| `tests/compaction_preserves_user_intent.rs` | New integration regression test | + +--- + +## Task 1: CJK-aware token estimation + +**Files:** +- Modify: `src/agent_prompt.rs` +- Test: `src/agent_prompt.rs` (`#[cfg(test)] mod tests`) + +- [ ] **Step 1: Write the failing test** + +Append to the tests module in `src/agent_prompt.rs`: + +```rust +#[test] +fn estimate_tokens_counts_cjk_and_latin() { + use crate::llm::{ChatMessage, MessageContent}; + + fn msg(text: &str) -> ChatMessage { + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(text.to_string())), + tool_calls: None, + tool_call_id: None, + } + } + + // 4 Latin chars ≈ 1 token. + let latin = vec![msg("abcd")]; + assert_eq!(estimate_tokens(&latin), 1, "4 latin chars ≈ 1 token"); + + // CJK chars cost 1 token each. + let cjk = vec![msg("中文测试")]; + assert_eq!(estimate_tokens(&cjk), 4, "CJK chars ≈ 1 token each"); + + // Mixed. + let mixed = vec![msg("hello中文")]; + assert_eq!(estimate_tokens(&mixed), 1 + 2, "latin/4 + cjk"); + + // Tool-call arguments count toward the total. + let with_tool = vec![ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![crate::llm::ToolCall { + id: "c1".to_string(), + call_type: "function".to_string(), + function: crate::llm::FunctionCall { + name: "search".to_string(), + arguments: r#"{"q":"abcd"}"#.to_string(), + }, + }]), + tool_call_id: None, + }]; + assert_eq!(estimate_tokens(&with_tool), 1, "tool args count"); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p rustfox estimate_tokens_counts_cjk_and_latin --lib` +Expected: FAIL — `estimate_tokens` not found. + +- [ ] **Step 3: Write the implementation** + +Add to `src/agent_prompt.rs`, next to `estimate_prompt_bytes` (line ~211): + +```rust +/// Estimate token count from messages, CJK-aware. +/// +/// CJK characters cost ~1 token each; Latin/other characters ~1/4 token. +/// Tool-call arguments count toward the total. This is the single token +/// estimate used for the compaction trigger (ADR 0003 Q3). +pub fn estimate_tokens(messages: &[ChatMessage]) -> usize { + let mut latin_chars = 0usize; + let mut cjk_chars = 0usize; + for msg in messages { + if let Some(content) = msg.content.as_ref() { + count_chars(content.as_text(), &mut latin_chars, &mut cjk_chars); + } + if let Some(calls) = msg.tool_calls.as_ref() { + for call in calls { + count_chars(&call.function.arguments, &mut latin_chars, &mut cjk_chars); + } + } + } + latin_chars / 4 + cjk_chars +} + +fn count_chars(text: &str, latin: &mut usize, cjk: &mut usize) { + for ch in text.chars() { + if is_cjk(ch) { + *cjk += 1; + } else { + *latin += 1; + } + } +} + +fn is_cjk(ch: char) -> bool { + matches!(ch as u32, + 0x2E80..=0x2EFF | // CJK Radicals Supplement + 0x3000..=0x303F | // CJK punctuation + 0x3040..=0x30FF | // Hiragana + Katakana + 0x3400..=0x4DBF | // CJK Extension A + 0x4E00..=0x9FFF | // CJK Unified Ideographs + 0xAC00..=0xD7AF // Hangul + ) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test -p rustfox estimate_tokens_counts_cjk_and_latin --lib` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent_prompt.rs +git commit -m "feat: CJK-aware token estimation for compaction trigger" +``` + +--- + +## Task 2: Protected-tail selection (`protected_tail_start`) + +**Files:** +- Modify: `src/agent_prompt.rs` +- Test: `src/agent_prompt.rs` + +- [ ] **Step 1: Write the failing test** + +Append to the tests module in `src/agent_prompt.rs`: + +```rust +fn chat_msg(role: &str, text: &str) -> ChatMessage { + ChatMessage { + role: role.to_string(), + content: Some(MessageContent::Text(text.to_string())), + tool_calls: None, + tool_call_id: None, + } +} + +fn tool_call_msg(id: &str, name: &str) -> ChatMessage { + ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![crate::llm::ToolCall { + id: id.to_string(), + call_type: "function".to_string(), + function: crate::llm::FunctionCall { + name: name.to_string(), + arguments: "{}".to_string(), + }, + }]), + tool_call_id: None, + } +} + +fn tool_result_msg(id: &str) -> ChatMessage { + ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("result payload".to_string())), + tool_calls: None, + tool_call_id: Some(id.to_string()), + } +} + +#[test] +fn protected_tail_start_keeps_last_two_user_turns() { + let mut msgs = vec![chat_msg("system", "sys")]; + for i in 0..10 { + msgs.push(chat_msg("user", &format!("turn {i}"))); + msgs.push(chat_msg("assistant", &format!("reply {i}"))); + } + let start = protected_tail_start(&msgs, 1_000_000); + let tail: Vec<&str> = msgs[start..].iter().map(|m| m.role.as_str()).collect(); + assert!(tail.iter().any(|r| *r == "user"), "tail keeps user turns"); + assert_eq!( + msgs[start..] + .iter() + .filter(|m| m.role == "user") + .count(), + 2, + "exactly the last two user turns survive" + ); + assert!( + msgs[start..].iter().any(|m| { + m.content + .as_ref() + .map(|c| c.as_text() == "turn 8") + .unwrap_or(false) + }), + "second-to-last user turn verbatim" + ); + assert!( + msgs[start..].iter().any(|m| { + m.content + .as_ref() + .map(|c| c.as_text() == "turn 9") + .unwrap_or(false) + }), + "last user turn verbatim" + ); +} + +#[test] +fn protected_tail_start_never_splits_tool_pair() { + // user, call, result, user — boundary must not land between call and result. + let msgs = vec![ + chat_msg("system", "sys"), + chat_msg("user", "old request"), + tool_call_msg("call_a", "lookup_thing"), + tool_result_msg("call_a"), + chat_msg("assistant", "old answer"), + chat_msg("user", "latest request"), + ]; + let start = protected_tail_start(&msgs, 1_000_000); + let tail = &msgs[start..]; + assert!( + !(tail.iter().any(|m| m.tool_call_id.as_deref() == Some("call_a")) + && !tail.iter().any(|m| { + m.has_tool_calls() + && m.tool_calls.as_ref().is_some_and(|calls| { + calls.iter().any(|c| c.id == "call_a") + }) + })), + "orphaned tool result in tail" + ); + assert!( + !(tail.iter().any(|m| { + m.has_tool_calls() + && m.tool_calls.as_ref().is_some_and(|calls| { + calls.iter().any(|c| c.id == "call_a") + }) + }) && !tail.iter().any(|m| m.tool_call_id.as_deref() == Some("call_a"))), + "orphaned tool call in tail" + ); +} + +#[test] +fn protected_tail_start_caps_at_20_percent() { + let mut msgs = vec![chat_msg("system", "sys")]; + for i in 0..8 { + msgs.push(chat_msg("user", &format!("request {i}"))); + msgs.push(chat_msg("assistant", &"reply ".repeat(500))); + } + // window sized so the full tail (~4K chars) exceeds 20% of window tokens + let window = estimate_tokens(&msgs) * 5 / 2; // tail cap = window/5 < tail tokens + let start = protected_tail_start(&msgs, window); + let tail_tokens = estimate_tokens(&msgs[start..]); + assert!( + tail_tokens <= window / 5 + estimate_tokens(&msgs[msgs.len() - 2..]), + "tail must be capped near 20% (plus the mandatory last turn): {tail_tokens} > {}", + window / 5 + ); + // last user turn always survives the cap + assert!( + msgs[start..].iter().any(|m| { + m.content + .as_ref() + .map(|c| c.as_text().starts_with("request 7")) + .unwrap_or(false) + }), + "last user turn must never be dropped by the cap" + ); +} + +#[test] +fn protected_tail_start_returns_zero_without_user_messages() { + let msgs = vec![ + chat_msg("system", "sys"), + chat_msg("assistant", "a"), + chat_msg("tool", "t"), + ]; + assert_eq!(protected_tail_start(&msgs, 1_000_000), 0); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p rustfox protected_tail_start --lib` +Expected: FAIL — `protected_tail_start` not found. + +- [ ] **Step 3: Write the implementation** + +Add to `src/agent_prompt.rs` after `estimate_tokens`: + +```rust +/// First index of the protected tail — messages from this index on are kept +/// verbatim (ADR 0003 Q4): the last two user turns plus the active exchange, +/// capped at 20% of `window` tokens. The boundary never splits a +/// [tool_call, tool_result] pair. Returns 0 when nothing can be protected +/// (no user messages) — callers treat 0 as "do not compact". +pub fn protected_tail_start(messages: &[ChatMessage], window: usize) -> usize { + let user_idx: Vec = messages + .iter() + .enumerate() + .filter(|(_, m)| m.role == "user") + .map(|(i, _)| i) + .collect(); + if user_idx.is_empty() { + return 0; + } + let last_user = *user_idx.last().expect("non-empty"); + let base = if user_idx.len() >= 2 { + user_idx[user_idx.len() - 2] + } else { + user_idx[0] + }; + if base == 0 { + return 0; // would protect the system message — nothing to compact + } + + let mut start = base; + let cap_tokens = window / 5; + while start < last_user && estimate_tokens(&messages[start..]) > cap_tokens { + start += 1; + } + + // Never split a [tool_call, tool_result] pair. + loop { + let mut changed = false; + for i in start..messages.len() { + let msg = &messages[i]; + if msg.has_tool_calls() { + let call_ids: Vec<&str> = msg + .tool_calls + .iter() + .flatten() + .map(|c| c.id.as_str()) + .collect(); + let mut last_result = i; + for j in (i + 1)..messages.len() { + let m = &messages[j]; + if m.role == "tool" + && m.tool_call_id + .as_deref() + .is_some_and(|id| call_ids.contains(&id)) + { + last_result = j; + } else if m.role != "tool" { + break; + } + } + if last_result + 1 > start { + start = last_result + 1; + changed = true; + } + } + } + if start > 0 { + let prev = &messages[start - 1]; + if prev.role == "tool" { + let call_in_tail = messages[start..].iter().any(|m| { + m.has_tool_calls() + && m.tool_calls.as_ref().is_some_and(|calls| { + calls + .iter() + .any(|c| Some(c.id.as_str()) == prev.tool_call_id.as_deref()) + }) + }); + if call_in_tail { + start -= 1; + changed = true; + } + } + } + if !changed { + break; + } + } + start +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test -p rustfox protected_tail_start --lib` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent_prompt.rs +git commit -m "feat: protected-tail selection keeps latest user intent (ADR Q4)" +``` + +--- + +## Task 3: `ConversationManager` summary state + persistence + +**Files:** +- Modify: `src/conversation.rs` +- Test: `src/conversation.rs` + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module in `src/conversation.rs` (replace the `manager` helper's struct literal to include the new fields): + +```rust +fn manager(messages: Vec) -> ConversationManager { + ConversationManager { + messages, + system_prompt: String::new(), + memory: crate::memory::MemoryStore::open_in_memory().unwrap(), + conversation_id: String::new(), + summary: None, + last_flush_turn: None, + } +} +``` + +And add these tests: + +```rust +#[tokio::test] +async fn should_flush_gate() { + // no user message in range → never flush + assert!(!should_flush(None, None)); + // first flush with a user message → yes + assert!(should_flush(Some(3), None)); + // same range as last flush → no + assert!(!should_flush(Some(3), Some(3))); + // newer user message than last flush → yes + assert!(should_flush(Some(7), Some(3))); +} + +#[tokio::test] +async fn apply_summary_layer_rebuilds_messages_and_persists() { + let store = crate::memory::MemoryStore::open_in_memory().unwrap(); + let conv = store + .get_or_create_conversation("test", "layer_u1") + .await + .unwrap(); + let mut cm = manager(vec![ + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("old request".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("old reply".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("latest request".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ]); + cm.conversation_id = conv.clone(); + + cm.apply_summary_layer("layer one content", 3).await.unwrap(); + + // Rebuilt: system + summary block + tail from index 3. + assert_eq!(cm.messages.len(), 3); + assert_eq!(cm.messages[0].role, "system"); + assert_eq!(cm.messages[1].role, "system"); + assert!( + cm.messages[1].content.as_ref().unwrap().as_text().contains( + "Previously compacted context:\nlayer one content" + ), + "summary injected as system message: {}", + cm.messages[1].content.as_ref().unwrap().as_text() + ); + assert_eq!(cm.messages[2].content.as_ref().unwrap().as_text(), "latest request"); + + // Second layer extends, not replaces. + cm.apply_summary_layer("layer two content", 2).await.unwrap(); + let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); + assert!( + summary_text.contains("layer one content") && summary_text.contains("layer two content"), + "layered extension: {summary_text}" + ); + assert_eq!(cm.summary.as_deref().unwrap(), "layer one content\n\nlayer two content"); + + // Persisted: [SUMMARY] rows reload. + let reloaded = store.load_messages(&conv).await.unwrap(); + let summary_rows: Vec<&str> = reloaded + .iter() + .filter_map(|m| { + m.content + .as_ref() + .map(|c| c.as_text()) + .filter(|t| t.starts_with("[SUMMARY]")) + }) + .collect(); + assert_eq!(summary_rows.len(), 2, "one [SUMMARY] row per layer"); + assert!(summary_rows[0].contains("layer one content")); + assert!(summary_rows[1].contains("layer two content")); +} + +#[tokio::test] +async fn apply_summary_layer_rejects_empty() { + let mut cm = manager(vec![ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }]); + assert!(cm.apply_summary_layer(" ", 1).await.is_err()); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p rustfox apply_summary_layer --lib` and `cargo test -p rustfox should_flush --lib` +Expected: FAIL — new fields/functions missing. + +- [ ] **Step 3: Add the fields to the struct** + +In `src/conversation.rs`: + +```rust +pub struct ConversationManager { + messages: Vec, + system_prompt: String, + memory: MemoryStore, + conversation_id: String, + /// Running summary of compacted history (ADR 0003 Q2) — layered, + /// persisted as `[SUMMARY]` rows (Q8), injected as a system message. + summary: Option, + /// Highest message index whose user turn was already flushed to USER.md (Q6). + last_flush_turn: Option, +} +``` + +- [ ] **Step 4: Fold persisted summaries on load** + +In `ConversationManager::new`, replace the history handling (currently lines ~28-31 and the `Ok(Self { ... })` at lines ~52-57): + +```rust + let history = memory + .load_messages(&conversation_id) + .await + .unwrap_or_default(); + + let mut folded_summary: Vec = Vec::new(); + let mut raw: Vec = Vec::new(); + for m in history { + if m.role == "system" { + if let Some(text) = m.content.as_ref().map(|c| c.as_text()) { + if let Some(rest) = text.strip_prefix("[SUMMARY]") { + folded_summary.push(rest.trim().to_string()); + continue; + } + } + } + if m.role == "user" && m.tool_call_id.as_deref() == Some("summary") { + continue; // legacy marker-style summary entries are superseded + } + raw.push(m); + } + let summary = (!folded_summary.is_empty()).then(|| folded_summary.join("\n\n")); +``` + +Then replace the message assembly (currently lines ~49-57): + +```rust + let mut messages = vec![system_msg]; + if let Some(s) = &summary { + messages.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!( + "Previously compacted context:\n{s}" + ))), + tool_calls: None, + tool_call_id: None, + }); + } + messages.extend(raw); + + Ok(Self { + messages, + system_prompt, + memory: memory.clone(), + conversation_id, + summary, + last_flush_turn: None, + }) +``` + +- [ ] **Step 5: Add `should_flush` and `apply_summary_layer`** + +Add after `add_user_turn` in `src/conversation.rs`: + +```rust + /// ADR 0003 Q6: flush only when the range contains a user-authored + /// message newer than the last flushed one. + pub(crate) fn should_flush( + range_user_max: Option, + last_flush_turn: Option, + ) -> bool { + match (range_user_max, last_flush_turn) { + (Some(max), Some(last)) => max > last, + (Some(_), None) => true, + (None, _) => false, + } + } + + /// Apply a new summary layer (ADR 0003 Q2/Q8): fold into the running + /// summary, rebuild the message list as [system, summary block, + /// protected tail], and persist the layer as a `[SUMMARY]` system + /// message. Persistence failures are logged and ignored — the in-memory + /// state wins. + pub(crate) async fn apply_summary_layer( + &mut self, + layer: &str, + tail_start: usize, + ) -> Result<()> { + let layer = layer.trim(); + if layer.is_empty() { + anyhow::bail!("empty summary layer"); + } + self.summary = Some(match self.summary.take() { + Some(prev) => format!("{prev}\n\n{layer}"), + None => layer.to_string(), + }); + + let mut new_msgs = Vec::with_capacity(2 + self.messages.len().saturating_sub(tail_start)); + if let Some(system) = self.messages.first().cloned() { + new_msgs.push(system); + } + new_msgs.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!( + "Previously compacted context:\n{}", + self.summary.as_deref().unwrap_or_default() + ))), + tool_calls: None, + tool_call_id: None, + }); + new_msgs.extend(self.messages.iter().skip(tail_start).cloned()); + self.messages = new_msgs; + + let persisted = ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!("[SUMMARY]\n{layer}"))), + tool_calls: None, + tool_call_id: None, + }; + if let Err(e) = self + .memory + .save_message(&self.conversation_id, &persisted) + .await + { + tracing::warn!(error = %format!("{e:#}"), "Failed to persist summary layer"); + } + Ok(()) + } +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cargo test -p rustfox apply_summary_layer --lib && cargo test -p rustfox should_flush --lib` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add src/conversation.rs +git commit -m "feat: running summary state + [SUMMARY] persistence (ADR Q2/Q8)" +``` + +--- + +## Task 4: Rewrite `compact_messages` (per-turn, defer-on-failure) + +**Files:** +- Modify: `src/conversation.rs` +- Test: `src/conversation.rs` + +- [ ] **Step 1: Write the failing test** + +Replace the `compact_messages_sync_fallback_produces_markers` test with: + +```rust +#[tokio::test] +async fn compact_messages_defers_on_llm_failure_never_truncates() { + use crate::agent_prompt::{estimate_tokens, protected_tail_start}; + + let mut messages = vec![ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("system prompt".to_string())), + tool_calls: None, + tool_call_id: None, + }]; + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "UNIQUE_KEYWORD_A long initial request {}", + "x".repeat(900) + ))), + tool_calls: None, + tool_call_id: None, + }); + for i in 0..15 { + messages.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![crate::llm::ToolCall { + id: format!("call_{i}"), + call_type: "function".to_string(), + function: crate::llm::FunctionCall { + name: "search".to_string(), + arguments: format!(r#"{{"q":"{}"}}"#, "y".repeat(120)), + }, + }]), + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text(format!( + "tool result {}", + "z".repeat(200) + ))), + tool_calls: None, + tool_call_id: Some(format!("call_{i}")), + }); + } + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text( + "UNIQUE_KEYWORD_B follow-up request".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }); + + let mut cm = manager(messages); + let llm = failing_llm(); + let original_len = cm.messages.len(); + let window = estimate_tokens(&cm.messages); + assert!(window > 0); + + let ctx = CompactionContext { + llm: &llm, + context_window: window, + compaction_model: None, + user_model_path: None, + }; + let result = cm.compact_messages(&ctx).await.unwrap(); + + // LLM failure → defer: no compaction, no truncation, nothing lost. + assert!(!result, "must defer when summarization fails"); + assert_eq!(cm.messages.len(), original_len, "messages unchanged"); + + let texts: Vec = cm + .messages + .iter() + .map(|m| m.content.as_ref().map(|c| c.as_text()).unwrap_or_default()) + .collect(); + assert!( + texts.iter().any(|t| t.contains("UNIQUE_KEYWORD_A")), + "initial request preserved verbatim" + ); + assert!( + texts.last().unwrap().contains("UNIQUE_KEYWORD_B"), + "latest user intent preserved verbatim" + ); + assert!( + texts + .iter() + .all(|t| t.len() >= 200 || !t.contains("UNIQUE_KEYWORD_A")), + "no 200-char truncation anywhere" + ); + + // Second attempt: protected tail must include both user turns. + let tail = protected_tail_start(&cm.messages, window); + assert!( + cm.messages[tail..] + .iter() + .any(|m| m.content.as_ref().map(|c| c.as_text()).is_some_and(|t| t.contains("UNIQUE_KEYWORD_B"))), + "protected tail contains the latest user turn" + ); +} +``` + +And add a success-path test that injects the layer directly (LLM-independent): + +```rust +#[tokio::test] +async fn compact_success_path_preserves_user_intent() { + let store = crate::memory::MemoryStore::open_in_memory().unwrap(); + let conv = store + .get_or_create_conversation("test", "compact_u1") + .await + .unwrap(); + let mut cm = manager(vec![ + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "UNIQUE_KEYWORD_A old request {}", + "x".repeat(800) + ))), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("old reply".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text( + "UNIQUE_KEYWORD_B follow-up".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }, + ]); + cm.conversation_id = conv.clone(); + + let tail = crate::agent_prompt::protected_tail_start(&cm.messages, 1_000_000); + assert_eq!(tail, 3, "old request + reply summarized, follow-up protected"); + cm.apply_summary_layer("user asked about UNIQUE_KEYWORD_A topic", tail) + .await + .unwrap(); + + // System message at index 1 carries the summary; the latest intent is verbatim. + assert_eq!(cm.messages[1].role, "system"); + let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); + assert!( + summary_text.contains("UNIQUE_KEYWORD_A"), + "summary preserves the old intent: {summary_text}" + ); + assert_eq!( + cm.messages[2].content.as_ref().unwrap().as_text(), + "UNIQUE_KEYWORD_B follow-up" + ); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p rustfox compact_ --lib` +Expected: FAIL — `CompactionContext` not found; `compact_messages` signature mismatch. + +- [ ] **Step 3: Add `CompactionContext`** + +Add near the top of `src/conversation.rs`: + +```rust +/// Inputs for one compaction pass (ADR 0003). +pub struct CompactionContext<'a> { + pub llm: &'a LlmClient, + /// Provider window in tokens (from `registry.effective_context_window`). + pub context_window: usize, + /// Optional cheaper model for summary + flush turns (Q9). + pub compaction_model: Option<&'a str>, + /// USER.md path for the durable-memory flush (Q5); `None` disables flush. + pub user_model_path: Option<&'a std::path::Path>, +} +``` + +- [ ] **Step 4: Replace `compact_messages` + `summarize_with_llm`, delete `build_sync_summary`** + +Replace the whole `compact_messages` body (lines ~127-243), `summarize_with_llm` (lines ~246-295), and delete `build_sync_summary` + its `MAX_CHARS` const + the `HashMap` import if it becomes unused (it is used only by `build_sync_summary`; remove `use std::collections::HashMap;` if `build_sync_summary` was its only user). + +```rust + /// Unified compaction pipeline (ADR 0003 Q1): compress the oldest + /// messages once total estimated tokens cross 85% of the real provider + /// window. The protected tail (last two user turns + active exchange, + /// never mid-tool-pair) stays verbatim. Durable facts are flushed to + /// USER.md before the running summary is extended. On summarizer + /// failure the pass is DEFERRED — nothing is truncated (Q7). + pub async fn compact_messages(&mut self, ctx: &CompactionContext<'_>) -> Result { + if ctx.context_window == 0 { + return Ok(false); + } + let trigger_tokens = + (ctx.context_window as f64 * crate::agent_prompt::COMPACT_TRIGGER_PCT) as usize; + if crate::agent_prompt::estimate_tokens(&self.messages) <= trigger_tokens { + return Ok(false); + } + + let tail_start = + crate::agent_prompt::protected_tail_start(&self.messages, ctx.context_window); + if tail_start == 0 || tail_start >= self.messages.len() { + return Ok(false); + } + let range: Vec<&ChatMessage> = self + .messages + .iter() + .skip(1) + .take(tail_start - 1) + .collect(); + if range.is_empty() { + return Ok(false); + } + + // Q5/Q6: durable-memory flush before the summary is written. + let range_user_max = range + .iter() + .enumerate() + .filter(|(_, m)| m.role == "user") + .map(|(i, _)| i + 1) // range index 0 == message index 1 + .max(); + if Self::should_flush(range_user_max, self.last_flush_turn) { + if let Some(path) = ctx.user_model_path { + match crate::learning::flush_user_model(ctx.llm, path, &range, ctx.compaction_model) + .await + { + Ok(true) => { + self.last_flush_turn = range_user_max; + } + Ok(false) => tracing::info!("User-model flush skipped: no durable facts"), + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "User-model flush failed"); + } + } + } + } + + // Q2/Q7: extend the running summary; defer on failure. + let layer = match self.summarize_with_llm(ctx, &range).await { + Ok(text) => text, + Err(e) => { + tracing::warn!( + error = %format!("{e:#}"), + range = range.len(), + "Compaction summary failed; deferring (no truncation)" + ); + return Ok(false); + } + }; + + self.apply_summary_layer(&layer, tail_start).await?; + Ok(true) + } + + /// Ask the summarizer (Q9 model override, else current model) to EXTEND + /// the running summary with the new portion of the conversation. + async fn summarize_with_llm( + &self, + ctx: &CompactionContext<'_>, + to_summarize: &[&ChatMessage], + ) -> Result { + let summary_text: String = to_summarize + .iter() + .map(|m| { + format!( + "{}: {}", + m.role, + m.content.as_ref().map(|c| c.as_text()).unwrap_or_default() + ) + }) + .collect::>() + .join("\n"); + + let previous = self.summary.as_deref().unwrap_or(""); + let summary_prompt = format!( + "You are maintaining a running summary of a long conversation.\n\ + {prev_block}\ + Below is the new portion of the conversation. EXTEND the previous summary with it:\n\ + - Preserve key facts, decisions, preferences, and open questions\n\ + - Merge new information; never contradict or repeat the previous summary\n\ + - Be concise — at most 300 words\n\ + - Output ONLY the new summary text (no preamble, no markers)\n\n\ + New conversation:\n{summary_text}", + prev_block = if previous.is_empty() { + String::new() + } else { + format!("Previous summary:\n{previous}\n\n") + }, + ); + + let summary_msg = vec![ + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text( + "You are a conversation summarizer.".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(summary_prompt)), + tool_calls: None, + tool_call_id: None, + }, + ]; + + let response = match ctx.compaction_model { + Some(model) => ctx + .llm + .chat_completion_with_model(&summary_msg, &[], model) + .await? + .message, + None => ctx.llm.chat(&summary_msg, &[]).await?, + }; + Ok(response + .content + .as_ref() + .map(|c| c.as_text()) + .unwrap_or_default()) + } +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p rustfox compact_ --lib` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/conversation.rs +git commit -m "feat: per-turn compaction with defer-on-failure (ADR Q1/Q7)" +``` + +--- + +## Task 5: Remove dead compaction machinery + +**Files:** +- Modify: `src/agent_prompt.rs` + +- [ ] **Step 1: Change the trigger constant** + +In `src/agent_prompt.rs` (line ~106): + +```rust +/// Utilization fraction (estimated tokens / context_window) at which the +/// unified compaction pipeline begins summarizing the oldest messages. +pub const COMPACT_TRIGGER_PCT: f64 = 0.85; +``` + +- [ ] **Step 2: Delete dead items** + +Delete all of the following from `src/agent_prompt.rs`: + +1. `COMPACT_LADDER` const + `compact_fraction` fn (lines ~107-127). +2. `COMPACT_TURN_GAP` const (line ~134) — only used by `should_auto_compact`. +3. `ConversationMeta` struct + `impl ConversationMeta` + `impl Default` (lines ~162-190). +4. `should_auto_compact` fn (lines ~530-545). +5. The tests `should_auto_compact_checks_bytes_turns_and_recursion_guard`, `should_auto_compact_needs_minimum_message_count`, and `compact_fraction_ladder`. + +Keep: `OBSERVATION_MASK_PCT`, `COLLAPSE_PCT`, `COMPACT_PCT` (used by emergency tiers), `COMPACTION_MARKER_PREFIX` (used by `is_compacted_regurgitation` in agent.rs), `estimate_prompt_bytes`, `compact_min_message_count`. + +- [ ] **Step 3: Remove the `ConversationMeta` import from agent.rs** + +In `src/agent.rs` (line 14): + +```rust +use crate::agent_prompt::{PreparedPrompt}; +``` + +- [ ] **Step 4: Verify compile** + +Run: `cargo check` +Expected: PASS with no warnings about the removed items (clippy may still flag `COMPACT_PCT` if unused — check next task; it is used by `should_auto_compact` only, so remove `COMPACT_PCT` and `REACTIVE_PCT` too if `cargo check`/`clippy` reports them unused after this task). + +- [ ] **Step 5: Run full test suite** + +Run: `cargo test` +Expected: PASS (old `compact_fraction_ladder` tests are gone; any remaining references to removed symbols fail loudly — fix by deleting). + +- [ ] **Step 6: Commit** + +```bash +git add src/agent_prompt.rs src/agent.rs +git commit -m "refactor: remove dead compaction constants and ConversationMeta" +``` + +--- + +## Task 6: USER.md flush machinery in `learning.rs` + +**Files:** +- Modify: `src/learning.rs` +- Test: `src/learning.rs` + +- [ ] **Step 1: Write the failing tests** + +Append to the tests module in `src/learning.rs`: + +```rust +#[tokio::test] +async fn test_flush_user_model_writes_valid_content() { + use crate::llm::{ChatMessage, MessageContent}; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("USER.md"); + let msg = ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text( + "I prefer replies in Traditional Chinese and short answers.".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }; + + let snippets = format_snippets(&[&msg]); + assert!(snippets.contains("[user]: I prefer replies")); + + // Frontmatter validation gate. + assert!(has_valid_frontmatter("---\nname: user-model\n---\n\nbody")); + assert!(!has_valid_frontmatter("no frontmatter here")); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p rustfox flush_user_model --lib` +Expected: FAIL — `format_snippets` not found. + +- [ ] **Step 3: Add the shared helpers and `flush_user_model`** + +In `src/learning.rs`, in the "Feature 3: User Model" section, add: + +```rust +/// Shared: format message excerpts for the user-model update prompt. +fn format_snippets(messages: &[&ChatMessage]) -> String { + messages + .iter() + .filter(|m| m.role == "user" || m.role == "assistant") + .filter_map(|m| { + m.content + .as_ref() + .map(|c| format!("[{}]: {}", m.role, c.as_text())) + }) + .collect::>() + .join("\n") +} + +/// Shared: build the user-model update prompt from existing content + snippets. +fn build_user_model_prompt(existing: &str, snippets: &str) -> String { + format!( + "You maintain a concise user profile for an AI assistant.\n\ + \n\ + Current user model:\n```\n{existing}\n```\n\ + \n\ + Recent conversation excerpts:\n```\n{snippets}\n```\n\ + \n\ + Update the user model based on the conversations. Rules:\n\ + - Keep the YAML frontmatter exactly as-is (name, description, tags)\n\ + - Update fields: user_name, language, communication_style, preferences, \ + interests, context\n\ + - Be concise — max 500 words total\n\ + - Only add information the user explicitly stated or strongly implied\n\ + - Do not remove existing valid entries — merge new info\n\ + - Output the COMPLETE updated file (frontmatter + body), nothing else" + ) +} + +/// Shared: validated write with `.bak` backup before overwrite. +async fn write_user_model_with_backup(user_model_path: &Path, new_content: &str) -> Result<()> { + if let Some(parent) = user_model_path.parent() { + tokio::fs::create_dir_all(parent).await.ok(); + } + if user_model_path.exists() { + let mut bak_path = user_model_path.to_string_lossy().to_string(); + bak_path.push_str(".bak"); + let _ = tokio::fs::copy(user_model_path, &bak_path).await; + } + tokio::fs::write(user_model_path, new_content) + .await + .with_context(|| format!("Failed to write user model: {}", user_model_path.display()))?; + Ok(()) +} + +/// Shared: prompt → LLM (optionally model-overridden) → frontmatter-validated +/// write. Returns `Ok(false)` when there is nothing to write. +async fn write_user_model_from_snippets( + llm: &LlmClient, + user_model_path: &Path, + snippets: &str, + model: Option<&str>, +) -> Result { + if snippets.trim().is_empty() { + return Ok(false); + } + let existing = if user_model_path.exists() { + tokio::fs::read_to_string(user_model_path) + .await + .unwrap_or_default() + } else { + DEFAULT_USER_MODEL.to_string() + }; + let prompt = build_user_model_prompt(&existing, snippets); + let messages = vec![ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(prompt)), + tool_calls: None, + tool_call_id: None, + }]; + + let response = match model { + Some(m) => llm.chat_completion_with_model(&messages, &[], m).await?, + None => llm.chat(&messages, &[]).await?, + }; + let new_content = response.content.unwrap_or_default().as_text(); + + // Strict validation: must start with `---` and contain a closing `---` + // delimiter so we don't write malformed or injection-bearing content into + // USER.md (which is later injected into the system prompt). + if !has_valid_frontmatter(&new_content) || new_content.trim().is_empty() { + warn!("User model update returned invalid content, skipping"); + return Ok(false); + } + + write_user_model_with_backup(user_model_path, &new_content).await?; + info!("User model updated: {}", user_model_path.display()); + Ok(true) +} + +/// Pre-compaction flush (ADR 0003 Q5): bank durable facts from the +/// to-be-summarized range into USER.md so compaction cannot erase them. +pub async fn flush_user_model( + llm: &LlmClient, + user_model_path: &Path, + range: &[&ChatMessage], + model: Option<&str>, +) -> Result { + let snippets = format_snippets(range); + write_user_model_from_snippets(llm, user_model_path, &snippets, model).await +} +``` + +- [ ] **Step 4: Refactor `update_user_model_inner` to use the shared helpers** + +Replace the body of `update_user_model_inner` (lines ~489-576) with: + +```rust +async fn update_user_model_inner( + llm: &LlmClient, + memory: &crate::memory::MemoryStore, + user_model_path: &Path, +) -> Result { + // Load recent conversation messages for context. + let recent = memory + .search_messages("user preferences interests communication", 20) + .await + .unwrap_or_default(); + + if recent.len() < MIN_MESSAGES_FOR_USER_MODEL { + return Ok(false); // Not enough data yet + } + + let refs: Vec<&ChatMessage> = recent.iter().collect(); + let snippets = format_snippets(&refs); + write_user_model_from_snippets(llm, user_model_path, &snippets, None).await +} +``` + +(Delete the old prompt/validation/backup/write code — now in the shared helpers.) + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p rustfox flush_user_model --lib && cargo test -p rustfox user_model --lib` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/learning.rs +git commit -m "feat: USER.md pre-compaction flush (ADR Q5/Q6)" +``` + +--- + +## Task 7: `compaction_model` config key + +**Files:** +- Modify: `src/config.rs` +- Modify: `config.example.toml` + +- [ ] **Step 1: Add the field** + +In `src/config.rs`, `LearningConfig` (after `user_model_cron`, line ~391): + +```rust + /// Optional model override for compaction summary + USER.md flush turns + /// (ADR 0003 Q9). Empty default = the conversation's current model. + #[serde(default)] + pub compaction_model: Option, +``` + +- [ ] **Step 2: Verify defaults still compile** + +Run: `cargo check` +Expected: PASS. + +- [ ] **Step 3: Document in `config.example.toml`** + +Replace the stale `[learning]` comment block (lines ~249-254) with: + +```toml +# ── Self-Learning (optional; defaults apply if section omitted) ───────────── +# [learning] +# skill_extraction_enabled = true # Auto-generate skills from tool-heavy tasks +# skill_extraction_threshold = 5 # Min tool calls to trigger extraction +# user_model_update_interval = 10 # Update user model every N user messages +# user_model_cron = "0 0 3 * * SUN" # Weekly user model refresh (6-field cron) +# compaction_model = "qwen/qwen3-8b" # Cheaper model for compaction summary + USER.md flush (default: current model) +``` + +- [ ] **Step 4: Commit** + +```bash +git add src/config.rs config.example.toml +git commit -m "feat: compaction_model config override (ADR Q9)" +``` + +--- + +## Task 8: Move compaction out of the loop; wire per-turn call + +**Files:** +- Modify: `src/loop_runner.rs` +- Modify: `src/agent.rs` + +- [ ] **Step 1: Remove the loop compaction block and hardcoded window** + +In `src/loop_runner.rs`: + +1. Delete lines 108-118 (the `if self.config.compaction_enabled ... compact_messages` block). +2. Replace `let context_window = 128_000;` (line 97) with `let context_window = self.config.context_window;`. +3. In `LoopConfig` (lines 27-39): remove `pub compaction_enabled: bool,`, add `pub context_window: usize,`. + +- [ ] **Step 2: Update the agent wiring** + +In `src/agent.rs` `process_message`: + +1. After `cmgr.add_user_turn(user_msg);` (line 742), insert the per-turn compaction (ADR Q1), replacing the `ConversationMeta` line (745): + +```rust + // Per-turn compaction (ADR 0003 Q1): routine compaction runs once per + // user turn, before the agentic loop, at 85% of the real provider window. + let current_model = self.current_model.read().await.clone(); + let context_window = self.registry.effective_context_window(¤t_model); + let compaction_model = self.config.learning.compaction_model.clone(); + let user_model_path = self + .config + .resolved_home + .as_ref() + .map(|h| h.join("USER.md")); + let compact_ctx = crate::conversation::CompactionContext { + llm: &self.llm, + context_window, + compaction_model: compaction_model.as_deref(), + user_model_path: user_model_path.as_deref(), + }; + if let Err(e) = cmgr.compact_messages(&compact_ctx).await { + warn!( + user_id = %user_id, + error = %format!("{e:#}"), + "Per-turn compaction failed" + ); + } +``` + +2. In `loop_config` (lines ~795-807): remove `compaction_enabled: true,`, add `context_window,`. + +- [ ] **Step 3: Verify compile + tests** + +Run: `cargo check && cargo test -p rustfox compact_ --lib` +Expected: PASS. + +- [ ] **Step 4: Commit** + +```bash +git add src/loop_runner.rs src/agent.rs +git commit -m "feat: per-turn compaction wired in process_message (ADR Q1)" +``` + +--- + +## Task 9: Emergency mask protects latest intent (Q7 step 3) + +**Files:** +- Modify: `src/agent_prompt.rs` +- Test: `src/agent_prompt.rs` + +- [ ] **Step 1: Write the failing test** + +Append to tests in `src/agent_prompt.rs`: + +```rust +#[test] +fn hard_cap_fallback_drops_oldest_only_keeps_latest_user_intent() { + let mut msgs = vec![chat_msg("system", "sys")]; + for i in 0..6 { + msgs.push(chat_msg("user", &format!("request {i}"))); + msgs.push(chat_msg("assistant", &"reply ".repeat(300))); + } + // Force the hard-cap path: obs/coll thresholds on a tiny window fail to + // reduce below PROMPT_HARD_CAP_BYTES. + let prepared = prepare_messages_for_llm(&msgs, 1_000); + assert!( + prepared + .messages + .iter() + .any(|m| m.content.as_ref().map(|c| c.as_text() == "request 5").unwrap_or(false)), + "last user turn survives the hard cap" + ); + assert!( + !prepared + .messages + .iter() + .any(|m| m.content.as_ref().map(|c| c.as_text() == "request 0").unwrap_or(false)), + "oldest traffic dropped" + ); +} +``` + +Note: this test uses the `chat_msg` helper added in Task 2. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p rustfox hard_cap_fallback --lib` +Expected: FAIL (current code keeps only sys/user + 2 newest, so `request 0` still present). + +- [ ] **Step 3: Rework the hard-cap branch** + +In `prepare_messages_for_llm` (lines ~470-496), replace the hard-cap branch: + +```rust + // Safety net (ADR 0003 Q7 step 3): if still over the hard cap after + // Tiers 1-2, drop the OLDEST non-protected traffic only — the last two + // user turns + active exchange always survive. + if estimate_prompt_bytes(&after_tier2) > PROMPT_HARD_CAP_BYTES { + let tail_start = protected_tail_start(&after_tier2, context_window).max(1); + let mut hard_cap_messages: Vec = Vec::with_capacity( + after_tier2.len().saturating_sub(tail_start) + 2, + ); + if let Some(system) = after_tier2.first() { + hard_cap_messages.push(system.clone()); + } + if tail_start < after_tier2.len() { + if tail_start > 1 { + hard_cap_messages.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text( + "★ earlier conversation dropped — memory compaction failed ★" + .to_string(), + )), + tool_calls: None, + tool_call_id: None, + }); + } + hard_cap_messages.extend(after_tier2.iter().skip(tail_start).cloned()); + } + hard_cap_messages + } else { + after_tier2 + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test -p rustfox hard_cap_fallback --lib` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent_prompt.rs +git commit -m "feat: emergency mask drops oldest traffic only (ADR Q7)" +``` + +--- + +## Task 10: Fix existing tests broken by the redesign + +**Files:** +- Modify: `src/conversation.rs` +- Modify: `src/agent_prompt.rs` + +- [ ] **Step 1: Update `compact_messages_never_splits_tool_pair`** + +Replace the body of the existing test with the new API (it currently calls `cm.compact_messages(&llm, 1_000)` with the failing LLM, which now defers and returns `false`). The pair-splitting property now lives in `protected_tail_start` (covered in Task 2); keep this test as a boundary check: + +```rust +#[test] +fn compact_range_boundary_lands_after_tool_pair() { + use crate::agent_prompt::protected_tail_start; + use crate::llm::{FunctionCall, ToolCall}; + + let mut messages = vec![ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("system prompt".to_string())), + tool_calls: None, + tool_call_id: None, + }]; + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "first request {}", + "x".repeat(100) + ))), + tool_calls: None, + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![ToolCall { + id: "call_split".to_string(), + call_type: "function".to_string(), + function: FunctionCall { + name: "lookup_thing".to_string(), + arguments: r#"{"query":"x"}"#.to_string(), + }, + }]), + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("lookup result payload".to_string())), + tool_calls: None, + tool_call_id: Some("call_split".to_string()), + }); + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "second request {}", + "x".repeat(100) + ))), + tool_calls: None, + tool_call_id: None, + }); + + let start = protected_tail_start(&messages, 1_000_000); + // Boundary must not orphan the pair: both call and result are either + // both in the tail or both summarized. + let call_in_tail = messages[start..].iter().any(|m| { + m.has_tool_calls() + && m.tool_calls.as_ref().is_some_and(|calls| { + calls.iter().any(|c| c.id == "call_split") + }) + }); + let result_in_tail = messages[start..] + .iter() + .any(|m| m.tool_call_id.as_deref() == Some("call_split")); + assert_eq!( + call_in_tail, result_in_tail, + "tool pair must not be split at the boundary (start={start})" + ); +} +``` + +- [ ] **Step 2: Run the conversation + prompt test suites** + +Run: `cargo test -p rustfox --lib` +Expected: PASS. Fix any stragglers by deleting tests that assert removed behavior (e.g. any remaining references to `compact_fraction` or `ConversationMeta` in `src/agent_prompt.rs` tests — remove them; the `estimate_prompt_bytes_counts_content_and_tool_arguments` test stays). + +- [ ] **Step 3: Commit** + +```bash +git add src/conversation.rs src/agent_prompt.rs +git commit -m "test: adapt compaction tests to new API" +``` + +--- + +## Task 11: Integration regression test + +**Files:** +- Create: `tests/compaction_preserves_user_intent.rs` + +- [ ] **Step 1: Write the test** + +```rust +//! Regression test: compaction must never lose the user's request, even when +//! the summarizer fails (ADR 0003 Q7). Uses an in-memory store and an LLM +//! client whose provider always fails (empty base_url → relative URL → no +//! network traffic). + +use std::collections::HashMap; +use std::sync::Arc; + +use rustfox::config::ProviderType; +use rustfox::conversation::{CompactionContext, ConversationManager}; +use rustfox::llm::{ChatMessage, LlmClient, MessageContent}; +use rustfox::memory::MemoryStore; +use rustfox::provider::{OpenRouterProvider, ProviderConfig, ProviderRegistry}; + +fn failing_llm() -> LlmClient { + let config = ProviderConfig { + name: "test".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: String::new(), + api_key: None, + default_model: "test-model".to_string(), + supports_vision: false, + max_tokens: 100, + discover_models: false, + context_window: 4096, + context_window_cache: Arc::new(tokio::sync::RwLock::new(None)), + parse_retry_limit: 0, + }; + let provider: Arc = + Arc::new(OpenRouterProvider::new(config)); + let mut providers = HashMap::new(); + providers.insert("test".to_string(), provider); + LlmClient::new(Arc::new(ProviderRegistry::new( + providers, + "test".to_string(), + ))) +} + +fn msg(role: &str, text: &str) -> ChatMessage { + ChatMessage { + role: role.to_string(), + content: Some(MessageContent::from_text(text.to_string())), + tool_calls: None, + tool_call_id: None, + } +} + +#[tokio::test] +async fn compaction_never_loses_user_request() { + let store = MemoryStore::open_in_memory().unwrap(); + let conv = store + .get_or_create_conversation("telegram", "intent_u1") + .await + .unwrap(); + + // Seed history: long initial request A + 15 tool exchanges + follow-up B. + let mut history: Vec = vec![msg( + "user", + &format!("UNIQUE_KEYWORD_A initial request {}", "x".repeat(900)), + )]; + for i in 0..15 { + history.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![rustfox::llm::ToolCall { + id: format!("call_{i}"), + call_type: "function".to_string(), + function: rustfox::llm::FunctionCall { + name: "search".to_string(), + arguments: format!(r#"{{"q":"{}"}}"#, "y".repeat(120)), + }, + }]), + tool_call_id: None, + }); + history.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::from_text(format!( + "tool result {}", + "z".repeat(200) + ))), + tool_calls: None, + tool_call_id: Some(format!("call_{i}")), + }); + } + for m in &history { + store.save_message(&conv, m).await.unwrap(); + } + + // Load via the real conversation path, then add the live follow-up. + let mut cmgr = ConversationManager::new( + &store, + "telegram", + "intent_u1", + "system prompt".to_string(), + &rustfox::skills::SkillRegistry::new(), + &minimal_config(), + ) + .await + .unwrap(); + cmgr.add_user_turn(msg("user", "UNIQUE_KEYWORD_B follow-up request")); + let original_len = cmgr.messages().len(); + + let llm = failing_llm(); + let window = rustfox::agent_prompt::estimate_tokens(cmgr.messages()); + let ctx = CompactionContext { + llm: &llm, + context_window: window, + compaction_model: None, + user_model_path: None, + }; + + // Two passes: both must defer (LLM failure), never truncate. + for pass in 0..2 { + let compacted = cmgr.compact_messages(&ctx).await.unwrap(); + assert!(!compacted, "pass {pass}: must defer on summarizer failure"); + assert_eq!( + cmgr.messages().len(), + original_len, + "pass {pass}: messages unchanged" + ); + } + + let texts: Vec = cmgr + .messages() + .iter() + .map(|m| m.content.as_ref().map(|c| c.as_text()).unwrap_or_default()) + .collect(); + assert!( + texts.iter().any(|t| t.contains("UNIQUE_KEYWORD_A")), + "initial request preserved verbatim" + ); + assert!( + texts.last().unwrap().contains("UNIQUE_KEYWORD_B"), + "latest user intent preserved verbatim and last" + ); +} + +fn minimal_config() -> rustfox::config::Config { + // into_path(): the temp dir must outlive the loaded config file. + let dir = tempfile::tempdir().unwrap().into_path(); + let path = dir.join("config.toml"); + std::fs::write( + &path, + r#" +[telegram] +bot_token = "test" +allowed_user_ids = [1] + +[openrouter] +api_key = "test" + +[sandbox] +allowed_directory = "." +"#, + ) + .unwrap(); + rustfox::config::Config::load(&path).unwrap() +} +``` + +- [ ] **Step 2: Run the test** + +Run: `cargo test --test compaction_preserves_user_intent` +Expected: PASS. + +- [ ] **Step 3: Commit** + +```bash +git add tests/compaction_preserves_user_intent.rs +git commit -m "test: integration regression — compaction never loses user intent" +``` + +--- + +## Task 12: Full verification + +- [ ] **Step 1: Format** + +Run: `cargo fmt --all -- --check` +Expected: clean. If not: `cargo fmt` and re-check. + +- [ ] **Step 2: Lint** + +Run: `cargo clippy -- -D warnings` +Expected: clean. Fix any warnings (this is where leftover unused constants like `COMPACT_PCT` surface — delete them). + +- [ ] **Step 3: Full test suite** + +Run: `cargo test` +Expected: all pass (including pre-existing supervisor/langsmith tests). + +- [ ] **Step 4: Update the ADR status** + +In `docs/adr/0003-conversation-compaction-redesign.md`, change the Status line to: + +```markdown +## Status +Accepted (implemented) +``` + +- [ ] **Step 5: Commit** + +```bash +git add docs/adr/0003-conversation-compaction-redesign.md +git commit -m "docs: ADR 0003 accepted — compaction redesign implemented" +``` + +--- + +## Self-review notes + +- **Spec coverage:** Q1 → Task 8; Q2/Q8 → Tasks 3+4; Q3 → Tasks 1+5; Q4 → Task 2; Q5/Q6 → Tasks 6+4; Q7 → Tasks 4+9; Q9 → Task 7. All nine ADR decisions have an implementing task. +- **Dependencies:** `cargo check` will fail between Task 3 and Task 4 (old `compact_messages` still calls removed helpers) — if working task-by-task, run Task 4 immediately after Task 3; the `manager()` helper in Task 3 must land with Task 3's struct change or the crate won't compile. Tasks 1-2 are standalone. +- **Manual verification required** (no mock LLM infra): the LLM success path of `compact_messages` is covered by construction (the layer prompt is unit-tested via `apply_summary_layer`), but the live OpenRouter round-trip needs a manual run: start the bot, let a conversation cross 85% of the window, confirm the reply arrives with the summary block visible in `prepare()` output and a `[SUMMARY]` row in the DB. diff --git a/src/agent.rs b/src/agent.rs index 6fb5adf..a09b0ff 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -11,7 +11,7 @@ use serde_json::Value; use teloxide::types::ChatId; use teloxide::Bot; -use crate::agent_prompt::{ConversationMeta, PreparedPrompt}; +use crate::agent_prompt::PreparedPrompt; use crate::cancel_registry::CancelRegistry; use crate::config::Config; use crate::langsmith::LangSmithClient; @@ -741,8 +741,30 @@ impl Agent { }; cmgr.add_user_turn(user_msg); - // Compaction state for this conversation session (persists across iterations) - let _conv_meta = ConversationMeta::new(); + // Per-turn compaction (ADR 0003 Q1): routine compaction runs once per + // user turn, before the agentic loop, at 85% of the real provider window. + let current_model = self.current_model.read().await.clone(); + let context_window = self.registry.effective_context_window(¤t_model); + let compaction_model = self.config.learning.compaction_model.clone(); + let user_model_path = self + .config + .resolved_home + .as_ref() + .map(|h| h.join("USER.md")); + let compact_ctx = crate::conversation::CompactionContext { + llm: &self.llm, + context_window, + compaction_model: compaction_model.as_deref(), + user_model_path: user_model_path.as_deref(), + }; + if let Err(e) = cmgr.compact_messages(&compact_ctx).await { + warn!( + user_id = %user_id, + error = %format!("{e:#}"), + "Per-turn compaction failed" + ); + } + // Gather all tool definitions let mut all_tools: Vec = self.tool_registry.all_definitions(); all_tools.extend(self.mcp.tool_definitions()); @@ -795,7 +817,7 @@ impl Agent { let loop_config = crate::loop_runner::LoopConfig { max_iterations: self.config.max_iterations(), empty_response_retry_limit: self.config.empty_response_retry_limit(), - compaction_enabled: true, + context_window, loop_detection_enabled: true, interactive_loop_callback: true, allowed_tools: None, @@ -1446,10 +1468,11 @@ impl Agent { }; let allowed_tools_vec: Vec = allowed_tools.to_vec(); + let subagent_window = self.registry.effective_context_window(model); let loop_config = crate::loop_runner::LoopConfig { max_iterations: max_iter, empty_response_retry_limit: self.config.empty_response_retry_limit(), - compaction_enabled: false, + context_window: subagent_window, loop_detection_enabled: true, interactive_loop_callback: false, allowed_tools: Some(allowed_tools_vec), diff --git a/src/agent_prompt.rs b/src/agent_prompt.rs index ff60fcf..c7b2136 100644 --- a/src/agent_prompt.rs +++ b/src/agent_prompt.rs @@ -99,40 +99,10 @@ impl CompressedMessage { const OBSERVATION_MASK_PCT: f64 = 0.70; /// Percentage that triggers Tier 2 (context collapse). const COLLAPSE_PCT: f64 = 0.75; -/// Percentage that triggers Tier 3 (auto compact). -pub const COMPACT_PCT: f64 = 0.82; -/// Utilization fraction (total chars / context_window) at which the unified -/// compaction pipeline begins summarizing the oldest messages. -pub const COMPACT_TRIGGER_PCT: f64 = 0.70; -/// Graduated compression ladder: (trigger, fraction of oldest messages to compress). -pub const COMPACT_LADDER: [(f64, f64); 5] = [ - (0.70, 0.10), - (0.75, 0.25), - (0.82, 0.40), - (0.88, 0.55), - (0.93, 0.70), -]; - -/// Oldest-message fraction to compress for a given utilization. -/// -/// Returns the fraction from the largest ladder entry whose trigger is -/// `<= utilization`, or `0.0` when utilization is below the first trigger. -pub fn compact_fraction(utilization: f64) -> f64 { - for (trigger, fraction) in COMPACT_LADDER.iter().rev() { - if utilization >= *trigger { - return *fraction; - } - } - 0.0 -} -/// Documentary threshold — Tier 4 is triggered by HTTP 413 errors, -/// not by a percentage, but this documents the utilization level at -/// which a 413 would typically occur. -#[allow(dead_code)] -pub(crate) const REACTIVE_PCT: f64 = 0.95; +/// Utilization fraction (estimated tokens / context_window) at which the +/// unified compaction pipeline begins summarizing the oldest messages. +pub const COMPACT_TRIGGER_PCT: f64 = 0.85; /// Minimum turns between Tier 3/4 compactions. -pub const COMPACT_TURN_GAP: usize = 5; -/// Number of most recent tool groups to preserve verbatim. pub const PRESERVED_TOOL_GROUPS: usize = 2; /// Absolute hard cap safety net (applied regardless of context_window). const PROMPT_HARD_CAP_BYTES: usize = 100_000; @@ -159,36 +129,6 @@ pub struct PreparedPrompt { pub stats: PromptStats, } -/// Per-conversation compaction metadata. -/// -/// Tracked in-memory alongside the message list. The agent loop increments -/// `current_turn` each iteration and updates `last_compact_turn` after -/// Tier 3/4 fires. -#[derive(Debug, Clone)] -pub struct ConversationMeta { - pub last_compact_turn: usize, - pub has_attempted_reactive_compact: bool, - pub is_compact_agent: bool, - pub current_turn: usize, -} - -impl ConversationMeta { - pub fn new() -> Self { - Self { - last_compact_turn: 0, - has_attempted_reactive_compact: false, - is_compact_agent: false, - current_turn: 0, - } - } -} - -impl Default for ConversationMeta { - fn default() -> Self { - Self::new() - } -} - /// Estimate the byte size of a prompt from its messages. pub fn estimate_prompt_bytes(messages: &[ChatMessage]) -> usize { messages @@ -210,6 +150,136 @@ pub fn estimate_prompt_bytes(messages: &[ChatMessage]) -> usize { .sum() } +/// Estimate token count from messages, CJK-aware. +/// +/// CJK characters cost ~1 token each; Latin/other characters ~1/4 token. +/// Tool-call arguments count toward the total. This is the single token +/// estimate used for the compaction trigger (ADR 0003 Q3). +pub fn estimate_tokens(messages: &[ChatMessage]) -> usize { + let mut latin_chars = 0usize; + let mut cjk_chars = 0usize; + for msg in messages { + if let Some(content) = msg.content.as_ref() { + count_chars(&content.as_text(), &mut latin_chars, &mut cjk_chars); + } + if let Some(calls) = msg.tool_calls.as_ref() { + for call in calls { + count_chars(&call.function.arguments, &mut latin_chars, &mut cjk_chars); + } + } + } + latin_chars / 4 + cjk_chars +} + +fn count_chars(text: &str, latin: &mut usize, cjk: &mut usize) { + for ch in text.chars() { + if is_cjk(ch) { + *cjk += 1; + } else { + *latin += 1; + } + } +} + +fn is_cjk(ch: char) -> bool { + matches!(ch as u32, + 0x2E80..=0x2EFF | // CJK Radicals Supplement + 0x3000..=0x303F | // CJK punctuation + 0x3040..=0x30FF | // Hiragana + Katakana + 0x3400..=0x4DBF | // CJK Extension A + 0x4E00..=0x9FFF | // CJK Unified Ideographs + 0xAC00..=0xD7AF // Hangul + ) +} + +/// First index of the protected tail — messages from this index on are kept +/// verbatim (ADR 0003 Q4): the last two user turns plus the active exchange, +/// capped at 20% of `window` tokens. The boundary never splits a +/// [tool_call, tool_result] pair. Returns 0 when nothing can be protected +/// (no user messages) — callers treat 0 as "do not compact". +pub fn protected_tail_start(messages: &[ChatMessage], window: usize) -> usize { + let user_idx: Vec = messages + .iter() + .enumerate() + .filter(|(_, m)| m.role == "user") + .map(|(i, _)| i) + .collect(); + if user_idx.is_empty() { + return 0; + } + let last_user = *user_idx.last().expect("non-empty"); + let base = if user_idx.len() >= 2 { + user_idx[user_idx.len() - 2] + } else { + user_idx[0] + }; + if base == 0 { + return 0; // would protect the system message — nothing to compact + } + + let mut start = base; + let cap_tokens = window / 5; + while start < last_user && estimate_tokens(&messages[start..]) > cap_tokens { + start += 1; + } + + // Never split a [tool_call, tool_result] pair. + loop { + let mut changed = false; + let mut i = start; + while i < messages.len() { + let msg = &messages[i]; + if msg.has_tool_calls() { + let call_ids: Vec<&str> = msg + .tool_calls + .iter() + .flatten() + .map(|c| c.id.as_str()) + .collect(); + let mut last_result = i; + for (offset, m) in messages.iter().skip(i + 1).enumerate() { + let j = i + 1 + offset; + if m.role == "tool" + && m.tool_call_id + .as_deref() + .is_some_and(|id| call_ids.contains(&id)) + { + last_result = j; + } else if m.role != "tool" { + break; + } + } + if last_result + 1 > start { + start = last_result + 1; + changed = true; + } + } + i += 1; + } + if start > 0 { + let prev = &messages[start - 1]; + if prev.role == "tool" { + let call_in_tail = messages[start..].iter().any(|m| { + m.has_tool_calls() + && m.tool_calls.as_ref().is_some_and(|calls| { + calls + .iter() + .any(|c| Some(c.id.as_str()) == prev.tool_call_id.as_deref()) + }) + }); + if call_in_tail { + start -= 1; + changed = true; + } + } + } + if !changed { + break; + } + } + start +} + /// Create a recovery nudge message appropriate for the conversation context. pub fn recovery_nudge_for(messages: &[ChatMessage]) -> ChatMessage { let previous_is_tool = messages.last().is_some_and(|msg| msg.role == "tool"); @@ -467,29 +537,30 @@ pub fn prepare_messages_for_llm(messages: &[ChatMessage], context_window: usize) after_tier1 }; - // Safety net: if still over hard cap, keep all sys/user + the 2 newest messages (1 preserved pair) + // Safety net (ADR 0003 Q7 step 3): if still over the hard cap after + // Tiers 1-2, drop the OLDEST non-protected traffic only — the last two + // user turns + active exchange always survive. if estimate_prompt_bytes(&after_tier2) > PROMPT_HARD_CAP_BYTES { - let preserved_count = after_tier2 - .iter() - .filter(|m| m.role == "system" || m.role == "user") - .count(); - let mut hard_cap_messages: Vec = Vec::with_capacity(preserved_count + 2); - for m in &after_tier2 { - if m.role == "system" || m.role == "user" { - hard_cap_messages.push(m.clone()); - } + let tail_start = protected_tail_start(&after_tier2, context_window).max(1); + let mut hard_cap_messages: Vec = + Vec::with_capacity(after_tier2.len().saturating_sub(tail_start) + 2); + if let Some(system) = after_tier2.first() { + hard_cap_messages.push(system.clone()); } - // Append the 2 newest non-system/user messages (preserves latest preserved pair) - let mut newest_pair: Vec = Vec::with_capacity(2); - for m in after_tier2.iter().rev() { - if m.role != "system" && m.role != "user" { - newest_pair.push(m.clone()); - if newest_pair.len() == 2 { - break; - } + if tail_start < after_tier2.len() { + if tail_start > 1 { + hard_cap_messages.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text( + "★ earlier conversation dropped — memory compaction failed ★" + .to_string(), + )), + tool_calls: None, + tool_call_id: None, + }); } + hard_cap_messages.extend(after_tier2.iter().skip(tail_start).cloned()); } - hard_cap_messages.extend(newest_pair.into_iter().rev()); hard_cap_messages } else { after_tier2 @@ -523,27 +594,6 @@ fn compact_min_message_count(context_window: usize) -> usize { base + (base / 2) * window_k } -/// Check whether Tier 3 auto-compact should trigger. -/// -/// All conditions must be true: -/// - Message count > compact_min_message_count -/// - Estimated prompt bytes > context_window * COMPACT_PCT -/// - Turn gap >= COMPACT_TURN_GAP since last compact -/// - Not already in compact agent loop (recursion guard) -pub fn should_auto_compact( - messages: &[ChatMessage], - meta: &ConversationMeta, - context_window: usize, -) -> bool { - let threshold = (context_window as f64 * COMPACT_PCT) as usize; - let bytes = estimate_prompt_bytes(messages); - - bytes > threshold - && messages.len() > compact_min_message_count(context_window) - && meta.current_turn - meta.last_compact_turn >= COMPACT_TURN_GAP - && !meta.is_compact_agent -} - /// Create the summary prompt content used for Tier 3 and Tier 4 LLM calls. /// Returns a system-role message instructing the LLM to summarize. pub fn build_compact_summary_prompt() -> ChatMessage { @@ -867,55 +917,6 @@ mod tests { ); } - #[test] - fn should_auto_compact_checks_bytes_turns_and_recursion_guard() { - let mut meta = ConversationMeta { - last_compact_turn: 0, - has_attempted_reactive_compact: false, - is_compact_agent: false, - current_turn: 10, - }; - - let small: Vec = (0..3) - .map(|_| ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("hi".to_string())), - tool_calls: None, - tool_call_id: None, - }) - .collect(); - assert!(!should_auto_compact(&small, &meta, 512_000)); - - let few_but_big: Vec = (0..5) - .map(|_| ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("x".repeat(100_000))), - tool_calls: None, - tool_call_id: None, - }) - .collect(); - assert!(!should_auto_compact(&few_but_big, &meta, 512_000)); - - let many_big: Vec = (0..23) - .map(|_| ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("x".repeat(50_000))), - tool_calls: None, - tool_call_id: None, - }) - .collect(); - - meta.is_compact_agent = true; - assert!(!should_auto_compact(&many_big, &meta, 512_000)); - meta.is_compact_agent = false; - - meta.last_compact_turn = 8; - assert!(!should_auto_compact(&many_big, &meta, 512_000)); - meta.last_compact_turn = 0; - - assert!(should_auto_compact(&many_big, &meta, 512_000)); - } - #[test] #[allow(clippy::vec_init_then_push)] fn find_tool_groups_detects_consecutive_tool_calls() { @@ -975,29 +976,6 @@ mod tests { assert_eq!(groups[1].tool_result_indices, vec![4]); } - #[test] - fn should_auto_compact_needs_minimum_message_count() { - let meta = ConversationMeta::new(); - let few: Vec = (0..5) - .map(|_| ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("x".repeat(100_000))), - tool_calls: None, - tool_call_id: None, - }) - .collect(); - assert!(!should_auto_compact(&few, &meta, 512_000)); - } - - #[test] - fn conversation_meta_defaults_to_zero() { - let meta = ConversationMeta::new(); - assert_eq!(meta.last_compact_turn, 0); - assert!(!meta.has_attempted_reactive_compact); - assert!(!meta.is_compact_agent); - assert_eq!(meta.current_turn, 0); - } - #[test] fn prepare_messages_applies_tier2_when_tier1_not_enough() { let ctx = 50000; @@ -1066,6 +1044,54 @@ mod tests { assert!(has_boundary); } + #[test] + fn hard_cap_fallback_drops_oldest_only_keeps_latest_user_intent() { + let mut msgs = vec![chat_msg("system", "sys")]; + msgs.push(chat_msg("user", "request 0")); + // 7 tool groups (16 msgs + sys + last user = 17 > compact_min_message_count(15)). + // Preserved tool groups alone exceed PROMPT_HARD_CAP_BYTES, so the + // obs/coll tiers cannot reduce below the cap → the hard-cap branch fires. + for i in 0..7 { + msgs.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![ToolCall { + id: format!("c{i}"), + call_type: "function".to_string(), + function: FunctionCall { + name: "tool".to_string(), + arguments: "x".repeat(60_000), + }, + }]), + tool_call_id: None, + }); + msgs.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("y".repeat(60_000))), + tool_calls: None, + tool_call_id: Some(format!("c{i}")), + }); + } + msgs.push(chat_msg("user", "request last")); + let prepared = prepare_messages_for_llm(&msgs, 1_000); + assert!( + prepared.messages.iter().any(|m| m + .content + .as_ref() + .map(|c| c.as_text() == "request last") + .unwrap_or(false)), + "last user turn survives the hard cap" + ); + assert!( + !prepared.messages.iter().any(|m| m + .content + .as_ref() + .map(|c| c.as_text() == "request 0") + .unwrap_or(false)), + "oldest traffic dropped" + ); + } + #[test] fn compact_summary_prompt_contains_state_keywords() { let msg = build_compact_summary_prompt(); @@ -1087,16 +1113,195 @@ mod tests { assert!(text.contains("state"), "Should hint at state format"); } + fn chat_msg(role: &str, text: &str) -> ChatMessage { + ChatMessage { + role: role.to_string(), + content: Some(MessageContent::Text(text.to_string())), + tool_calls: None, + tool_call_id: None, + } + } + + fn tool_call_msg(id: &str, name: &str) -> ChatMessage { + ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![crate::llm::ToolCall { + id: id.to_string(), + call_type: "function".to_string(), + function: crate::llm::FunctionCall { + name: name.to_string(), + arguments: "{}".to_string(), + }, + }]), + tool_call_id: None, + } + } + + fn tool_result_msg(id: &str) -> ChatMessage { + ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("result payload".to_string())), + tool_calls: None, + tool_call_id: Some(id.to_string()), + } + } + + #[test] + fn protected_tail_start_keeps_last_two_user_turns() { + let mut msgs = vec![chat_msg("system", "sys")]; + for i in 0..10 { + msgs.push(chat_msg("user", &format!("turn {i}"))); + msgs.push(chat_msg("assistant", &format!("reply {i}"))); + } + let start = protected_tail_start(&msgs, 1_000_000); + let tail: Vec<&str> = msgs[start..].iter().map(|m| m.role.as_str()).collect(); + assert!(tail.contains(&"user"), "tail keeps user turns"); + assert_eq!( + msgs[start..].iter().filter(|m| m.role == "user").count(), + 2, + "exactly the last two user turns survive" + ); + assert!( + msgs[start..].iter().any(|m| { + m.content + .as_ref() + .map(|c| c.as_text() == "turn 8") + .unwrap_or(false) + }), + "second-to-last user turn verbatim" + ); + assert!( + msgs[start..].iter().any(|m| { + m.content + .as_ref() + .map(|c| c.as_text() == "turn 9") + .unwrap_or(false) + }), + "last user turn verbatim" + ); + } + + #[test] + fn protected_tail_start_never_splits_tool_pair() { + // user, call, result, user — boundary must not land between call and result. + let msgs = vec![ + chat_msg("system", "sys"), + chat_msg("user", "old request"), + tool_call_msg("call_a", "lookup_thing"), + tool_result_msg("call_a"), + chat_msg("assistant", "old answer"), + chat_msg("user", "latest request"), + ]; + let start = protected_tail_start(&msgs, 1_000_000); + let tail = &msgs[start..]; + assert!( + !(tail + .iter() + .any(|m| m.tool_call_id.as_deref() == Some("call_a")) + && !tail.iter().any(|m| { + m.has_tool_calls() + && m.tool_calls + .as_ref() + .is_some_and(|calls| calls.iter().any(|c| c.id == "call_a")) + })), + "orphaned tool result in tail" + ); + assert!( + !(tail.iter().any(|m| { + m.has_tool_calls() + && m.tool_calls + .as_ref() + .is_some_and(|calls| calls.iter().any(|c| c.id == "call_a")) + }) && !tail + .iter() + .any(|m| m.tool_call_id.as_deref() == Some("call_a"))), + "orphaned tool call in tail" + ); + } + + #[test] + fn protected_tail_start_caps_at_20_percent() { + let mut msgs = vec![chat_msg("system", "sys")]; + for i in 0..8 { + msgs.push(chat_msg("user", &format!("request {i}"))); + msgs.push(chat_msg("assistant", &"reply ".repeat(500))); + } + // window sized so the full tail (~4K chars) exceeds 20% of window tokens + let window = estimate_tokens(&msgs) * 5 / 2; // tail cap = window/5 < tail tokens + let start = protected_tail_start(&msgs, window); + let tail_tokens = estimate_tokens(&msgs[start..]); + assert!( + tail_tokens <= window / 5 + estimate_tokens(&msgs[msgs.len() - 2..]), + "tail must be capped near 20% (plus the mandatory last turn): {tail_tokens} > {}", + window / 5 + ); + // last user turn always survives the cap + assert!( + msgs[start..].iter().any(|m| { + m.content + .as_ref() + .map(|c| c.as_text().starts_with("request 7")) + .unwrap_or(false) + }), + "last user turn must never be dropped by the cap" + ); + } + + #[test] + fn protected_tail_start_returns_zero_without_user_messages() { + let msgs = vec![ + chat_msg("system", "sys"), + chat_msg("assistant", "a"), + chat_msg("tool", "t"), + ]; + assert_eq!(protected_tail_start(&msgs, 1_000_000), 0); + } + #[test] - fn compact_fraction_ladder() { - let approx = |a: f64, b: f64| (a - b).abs() < 1e-9; - assert!(approx(compact_fraction(0.69), 0.0)); - assert!(approx(compact_fraction(0.70), 0.10)); - assert!(approx(compact_fraction(0.80), 0.25)); - assert!(approx(compact_fraction(0.90), 0.55)); - assert!(approx(compact_fraction(0.99), 0.70)); - // Saturates at the largest ladder entry. - assert!(approx(compact_fraction(2.0), 0.70)); + fn estimate_tokens_counts_cjk_and_latin() { + use crate::llm::{ChatMessage, MessageContent}; + + fn msg(text: &str) -> ChatMessage { + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(text.to_string())), + tool_calls: None, + tool_call_id: None, + } + } + + // 4 Latin chars ≈ 1 token. + let latin = vec![msg("abcd")]; + assert_eq!(estimate_tokens(&latin), 1, "4 latin chars ≈ 1 token"); + + // CJK chars cost 1 token each. + let cjk = vec![msg("中文测试")]; + assert_eq!(estimate_tokens(&cjk), 4, "CJK chars ≈ 1 token each"); + + // Mixed. + let mixed = vec![msg("hello中文")]; + assert_eq!(estimate_tokens(&mixed), 1 + 2, "latin/4 + cjk"); + + // Tool-call arguments count toward the total. + let with_tool = vec![ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![crate::llm::ToolCall { + id: "c1".to_string(), + call_type: "function".to_string(), + function: crate::llm::FunctionCall { + name: "search".to_string(), + arguments: r#"{"q":"abcd"}"#.to_string(), + }, + }]), + tool_call_id: None, + }]; + assert_eq!( + estimate_tokens(&with_tool), + 3, + "tool args count (12 latin chars)" + ); } #[test] diff --git a/src/config.rs b/src/config.rs index e933851..d152a01 100644 --- a/src/config.rs +++ b/src/config.rs @@ -389,6 +389,10 @@ pub struct LearningConfig { /// Cron expression for weekly user model update (default: Sunday 3am). #[serde(default = "default_user_model_cron")] pub user_model_cron: String, + /// Optional model override for compaction summary + USER.md flush turns + /// (ADR 0003 Q9). Empty default = the conversation's current model. + #[serde(default)] + pub compaction_model: Option, } fn default_model() -> String { @@ -559,6 +563,7 @@ fn default_learning_config() -> LearningConfig { skill_extraction_threshold: default_skill_extraction_threshold(), user_model_update_interval: default_user_model_update_interval(), user_model_cron: default_user_model_cron(), + compaction_model: None, } } diff --git a/src/conversation.rs b/src/conversation.rs index 1d8cf5a..c616e90 100644 --- a/src/conversation.rs +++ b/src/conversation.rs @@ -1,18 +1,33 @@ use anyhow::Result; -use crate::agent_prompt::{prepare_messages_for_llm, CompressedMessage}; +use crate::agent_prompt::prepare_messages_for_llm; use crate::config::Config; use crate::llm::{ChatMessage, ContentPart, LlmClient, MessageContent}; use crate::memory::MemoryStore; use crate::platform::IncomingMessage; use crate::skills::SkillRegistry; -use std::collections::HashMap; + +/// Inputs for one compaction pass (ADR 0003). +pub struct CompactionContext<'a> { + pub llm: &'a LlmClient, + /// Provider window in tokens (from `registry.effective_context_window`). + pub context_window: usize, + /// Optional cheaper model for summary + flush turns (Q9). + pub compaction_model: Option<&'a str>, + /// USER.md path for the durable-memory flush (Q5); `None` disables flush. + pub user_model_path: Option<&'a std::path::Path>, +} pub struct ConversationManager { messages: Vec, system_prompt: String, memory: MemoryStore, conversation_id: String, + /// Running summary of compacted history (ADR 0003 Q2) — layered, + /// persisted as `[SUMMARY]` rows (Q8), injected as a system message. + summary: Option, + /// Highest message index whose user turn was already flushed to USER.md (Q6). + last_flush_turn: Option, } impl ConversationManager { @@ -30,6 +45,24 @@ impl ConversationManager { .await .unwrap_or_default(); + let mut folded_summary: Vec = Vec::new(); + let mut raw: Vec = Vec::new(); + for m in history { + if m.role == "system" { + if let Some(text) = m.content.as_ref().map(|c| c.as_text()) { + if let Some(rest) = text.strip_prefix("[SUMMARY]") { + folded_summary.push(rest.trim().to_string()); + continue; + } + } + } + if m.role == "user" && m.tool_call_id.as_deref() == Some("summary") { + continue; // legacy marker-style summary entries are superseded + } + raw.push(m); + } + let summary = (!folded_summary.is_empty()).then(|| folded_summary.join("\n\n")); + let now = chrono::Local::now(); let context_prompt = format!( "\n\nCurrent date and time: {} ({})", @@ -47,13 +80,25 @@ impl ConversationManager { }; let mut messages = vec![system_msg]; - messages.extend(history); + if let Some(s) = &summary { + messages.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!( + "Previously compacted context:\n{s}" + ))), + tool_calls: None, + tool_call_id: None, + }); + } + messages.extend(raw); Ok(Self { messages, system_prompt, memory: memory.clone(), conversation_id, + summary, + last_flush_turn: None, }) } @@ -89,6 +134,70 @@ impl ConversationManager { self.messages.push(msg); } + /// ADR 0003 Q6: flush only when the range contains a user-authored + /// message newer than the last flushed one. + pub(crate) fn should_flush( + range_user_max: Option, + last_flush_turn: Option, + ) -> bool { + match (range_user_max, last_flush_turn) { + (Some(max), Some(last)) => max > last, + (Some(_), None) => true, + (None, _) => false, + } + } + + /// Apply a new summary layer (ADR 0003 Q2/Q8): fold into the running + /// summary, rebuild the message list as [system, summary block, + /// protected tail], and persist the layer as a `[SUMMARY]` system + /// message. Persistence failures are logged and ignored — the in-memory + /// state wins. + pub(crate) async fn apply_summary_layer( + &mut self, + layer: &str, + tail_start: usize, + ) -> Result<()> { + let layer = layer.trim(); + if layer.is_empty() { + anyhow::bail!("empty summary layer"); + } + self.summary = Some(match self.summary.take() { + Some(prev) => format!("{prev}\n\n{layer}"), + None => layer.to_string(), + }); + + let mut new_msgs = Vec::with_capacity(2 + self.messages.len().saturating_sub(tail_start)); + if let Some(system) = self.messages.first().cloned() { + new_msgs.push(system); + } + new_msgs.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!( + "Previously compacted context:\n{}", + self.summary.as_deref().unwrap_or_default() + ))), + tool_calls: None, + tool_call_id: None, + }); + new_msgs.extend(self.messages.iter().skip(tail_start).cloned()); + self.messages = new_msgs; + + let persisted = ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!("[SUMMARY]\n{layer}"))), + tool_calls: None, + tool_call_id: None, + }; + if let Err(e) = self + .memory + .save_message(&self.conversation_id, &persisted) + .await + { + tracing::warn!(error = %format!("{e:#}"), "Failed to persist summary layer"); + } + Ok(()) + } + pub fn add_assistant_turn(&mut self, msg: ChatMessage) { self.messages.push(msg); } @@ -114,138 +223,77 @@ impl ConversationManager { self.messages.push(steer_msg); } - /// Unified compaction pipeline: compress the oldest messages once total - /// utilization crosses `agent_prompt::COMPACT_TRIGGER_PCT` of the context - /// window. The fraction of oldest messages summarized follows the graduated - /// `agent_prompt::COMPACT_LADDER`; the system message (index 0) and the - /// newest 8 messages always stay verbatim. - /// - /// Summarization is attempted via the LLM and rendered as structured marker - /// lines; on LLM failure a sync structured extraction is used instead (no - /// LLM call). Returns `Ok(true)` when compaction happened, `Ok(false)` when - /// nothing was summarized. - pub async fn compact_messages( - &mut self, - llm: &LlmClient, - context_window: usize, - ) -> Result { - if context_window == 0 { + /// Unified compaction pipeline (ADR 0003 Q1): compress the oldest + /// messages once total estimated tokens cross 85% of the real provider + /// window. The protected tail (last two user turns + active exchange, + /// never mid-tool-pair) stays verbatim. Durable facts are flushed to + /// USER.md before the running summary is extended. On summarizer + /// failure the pass is DEFERRED — nothing is truncated (Q7). + pub async fn compact_messages(&mut self, ctx: &CompactionContext<'_>) -> Result { + if ctx.context_window == 0 { return Ok(false); } - let total: usize = self - .messages - .iter() - .map(|m| m.content.as_ref().map(|c| c.as_text().len()).unwrap_or(0)) - .sum(); - let utilization = total as f64 / context_window as f64; - if utilization < crate::agent_prompt::COMPACT_TRIGGER_PCT { + let trigger_tokens = + (ctx.context_window as f64 * crate::agent_prompt::COMPACT_TRIGGER_PCT) as usize; + if crate::agent_prompt::estimate_tokens(&self.messages) <= trigger_tokens { return Ok(false); } - // Graduated fraction of the oldest (non-system) messages to compress, - // clamped so at most len-9 messages are summarized (system + newest 8 - // stay verbatim). - let fraction = crate::agent_prompt::compact_fraction(utilization); - let max_summarize = self.messages.len().saturating_sub(9); - let mut summarize_count = - ((self.messages.len().saturating_sub(1)) as f64 * fraction) as usize; - summarize_count = summarize_count.min(max_summarize); - if summarize_count == 0 { + let tail_start = + crate::agent_prompt::protected_tail_start(&self.messages, ctx.context_window); + if tail_start == 0 || tail_start >= self.messages.len() { + return Ok(false); + } + let range: Vec<&ChatMessage> = self.messages.iter().skip(1).take(tail_start - 1).collect(); + if range.is_empty() { return Ok(false); } - // Round the summarized range down to tool-group boundaries so a - // [tool_call, tool_result] pair is never split across the - // summarize/preserve boundary. - let mut i = 1usize; - while i <= summarize_count { - let msg = &self.messages[i]; - if msg.has_tool_calls() { - let call_ids: Vec<&str> = msg - .tool_calls - .iter() - .flatten() - .map(|c| c.id.as_str()) - .collect(); - let mut last_result: Option = None; - for j in (i + 1)..self.messages.len() { - let m = &self.messages[j]; - if m.role == "tool" - && m.tool_call_id - .as_deref() - .is_some_and(|id| call_ids.contains(&id)) - { - last_result = Some(j); - } else if m.role != "tool" { - break; + // Q5/Q6: durable-memory flush before the summary is written. + let range_user_max = range + .iter() + .enumerate() + .filter(|(_, m)| m.role == "user") + .map(|(i, _)| i + 1) // range index 0 == message index 1 + .max(); + if Self::should_flush(range_user_max, self.last_flush_turn) { + if let Some(path) = ctx.user_model_path { + match crate::learning::flush_user_model(ctx.llm, path, &range, ctx.compaction_model) + .await + { + Ok(true) => { + self.last_flush_turn = range_user_max; } - } - if let Some(r) = last_result { - if r > summarize_count { - summarize_count = r; + Ok(false) => tracing::info!("User-model flush skipped: no durable facts"), + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "User-model flush failed"); } } - } else if msg.role == "tool" { - // A result inside the range whose matching call is preserved - // in the tail: stop the range before this result. - let call_in_tail = self.messages[(summarize_count + 1)..].iter().any(|m| { - m.has_tool_calls() - && m.tool_calls.as_ref().is_some_and(|calls| { - calls - .iter() - .any(|c| Some(c.id.as_str()) == msg.tool_call_id.as_deref()) - }) - }); - if call_in_tail { - summarize_count = i.saturating_sub(1); - break; - } } - i += 1; } - if summarize_count == 0 { - return Ok(false); - } - - // System (index 0) and the newest 8 messages stay verbatim. - let preserved_tail_start = (self.messages.len().saturating_sub(8)).max(summarize_count + 1); - let to_summarize: Vec<&ChatMessage> = - self.messages.iter().skip(1).take(summarize_count).collect(); - let summary_text = match self.summarize_with_llm(llm, &to_summarize).await { + // Q2/Q7: extend the running summary; defer on failure. + let layer = match self.summarize_with_llm(ctx, &range).await { Ok(text) => text, Err(e) => { - tracing::warn!("LLM compaction failed ({e}); using sync structured summary"); - self.build_sync_summary(&to_summarize) + tracing::warn!( + error = %format!("{e:#}"), + range = range.len(), + "Compaction summary failed; deferring (no truncation)" + ); + return Ok(false); } }; - let heading = format!( - "★ COMPACTED CONTEXT — {} messages summarized ★\n", - to_summarize.len() - ); - let summary_entry = ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text(format!("{heading}{summary_text}"))), - tool_calls: None, - tool_call_id: Some("summary".to_string()), - }; - - let mut new_msgs = - Vec::with_capacity(2 + self.messages.len().saturating_sub(preserved_tail_start)); - if let Some(system) = self.messages.first().cloned() { - new_msgs.push(system); - } - new_msgs.push(summary_entry); - new_msgs.extend(self.messages.iter().skip(preserved_tail_start).cloned()); - self.messages = new_msgs; + self.apply_summary_layer(&layer, tail_start).await?; Ok(true) } - /// Ask the LLM to summarize `to_summarize` as structured marker lines. + /// Ask the summarizer (Q9 model override, else current model) to EXTEND + /// the running summary with the new portion of the conversation. async fn summarize_with_llm( &self, - llm: &LlmClient, + ctx: &CompactionContext<'_>, to_summarize: &[&ChatMessage], ) -> Result { let summary_text: String = to_summarize @@ -260,15 +308,23 @@ impl ConversationManager { .collect::>() .join("\n"); + let previous = self.summary.as_deref().unwrap_or(""); let summary_prompt = format!( - "Summarize the following conversation, preserving key decisions and facts.\n\ - Output the summary AS STRUCTURED MARKER LINES, one per message, in exactly these formats:\n\ - [Tool: NAME] description | result: SUMMARY | status: ok|error\n\ - [User] TOPIC: SUMMARY\n\ - [Assistant] ACTION: DECISION_SUMMARY\n\ - [System] EVENT: NOTABLE_INFO\n\n\ - Conversation:\n{summary_text}" + "You are maintaining a running summary of a long conversation.\n\ + {prev_block}\ + Below is the new portion of the conversation. EXTEND the previous summary with it:\n\ + - Preserve key facts, decisions, preferences, and open questions\n\ + - Merge new information; never contradict or repeat the previous summary\n\ + - Be concise — at most 300 words\n\ + - Output ONLY the new summary text (no preamble, no markers)\n\n\ + New conversation:\n{summary_text}", + prev_block = if previous.is_empty() { + String::new() + } else { + format!("Previous summary:\n{previous}\n\n") + }, ); + let summary_msg = vec![ ChatMessage { role: "system".to_string(), @@ -286,7 +342,15 @@ impl ConversationManager { }, ]; - let response = llm.chat(&summary_msg, &[]).await?; + let response = match ctx.compaction_model { + Some(model) => { + ctx.llm + .chat_completion_with_model(&summary_msg, &[], model) + .await? + .message + } + None => ctx.llm.chat(&summary_msg, &[]).await?, + }; Ok(response .content .as_ref() @@ -294,76 +358,6 @@ impl ConversationManager { .unwrap_or_default()) } - /// Sync fallback: build structured marker lines from `to_summarize` - /// without any LLM call. - fn build_sync_summary(&self, to_summarize: &[&ChatMessage]) -> String { - const MAX_CHARS: usize = 200; - // Resolve tool identity by NAME: map tool_call_id -> function name so - // tool results can render the call's name instead of the raw id. - let mut tool_names: HashMap<&str, &str> = HashMap::new(); - for m in &self.messages { - if let Some(calls) = &m.tool_calls { - for c in calls { - tool_names.entry(c.id.as_str()).or_insert(&c.function.name); - } - } - } - - let mut compacted: Vec = Vec::new(); - for m in to_summarize { - let text = m.content.as_ref().map(|c| c.as_text()).unwrap_or_default(); - if text.is_empty() && !m.has_tool_calls() { - continue; - } - let truncated: String = text.chars().take(MAX_CHARS).collect(); - if m.has_tool_calls() { - // Assistant tool-call message -> [Tool: NAME] marker. - let call = m - .tool_calls - .iter() - .flatten() - .next() - .expect("has_tool_calls checked"); - let name = call.function.name.clone(); - compacted.push(CompressedMessage { - role: m.role.clone(), - original_type: "tool_call".to_string(), - summary: name.clone(), - key_data: Some(serde_json::json!({ - "name": name, - "args": call.function.arguments.chars().take(MAX_CHARS).collect::(), - })), - }); - } else if m.role == "tool" { - let id = m.tool_call_id.as_deref().unwrap_or("unknown"); - let name = tool_names.get(id).copied().unwrap_or(id).to_string(); - let status = if text.contains("Error") || text.contains("error") { - "error" - } else { - "ok" - }; - compacted.push(CompressedMessage { - role: m.role.clone(), - original_type: "tool_result".to_string(), - summary: truncated, - key_data: Some(serde_json::json!({"name": name, "status": status})), - }); - } else { - compacted.push(CompressedMessage { - role: m.role.clone(), - original_type: m.role.clone(), - summary: truncated, - key_data: None, - }); - } - } - compacted - .iter() - .map(|c| c.to_marker()) - .collect::>() - .join("\n") - } - pub fn prepare(&self, context_window: usize) -> crate::agent_prompt::PreparedPrompt { prepare_messages_for_llm(&self.messages, context_window) } @@ -418,108 +412,122 @@ mod tests { system_prompt: String::new(), memory: crate::memory::MemoryStore::open_in_memory().unwrap(), conversation_id: String::new(), + summary: None, + last_flush_turn: None, } } #[tokio::test] - async fn compact_messages_never_splits_tool_pair() { - use crate::llm::{FunctionCall, ToolCall}; + async fn should_flush_gate() { + // no user message in range → never flush + assert!(!ConversationManager::should_flush(None, None)); + // first flush with a user message → yes + assert!(ConversationManager::should_flush(Some(3), None)); + // same range as last flush → no + assert!(!ConversationManager::should_flush(Some(3), Some(3))); + // newer user message than last flush → yes + assert!(ConversationManager::should_flush(Some(7), Some(3))); + } - let mut messages = vec![ChatMessage { - role: "system".to_string(), - content: Some(MessageContent::Text("system prompt".to_string())), - tool_calls: None, - tool_call_id: None, - }]; - messages.push(ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text(format!( - "first request {}", - "x".repeat(100) - ))), - tool_calls: None, - tool_call_id: None, - }); - messages.push(ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text(format!( - "second request {}", - "x".repeat(100) - ))), - tool_calls: None, - tool_call_id: None, - }); - // idx 3: assistant with tool call — the naive boundary (len-9 = 3) - // lands exactly here, which would orphan the result below. - messages.push(ChatMessage { - role: "assistant".to_string(), - content: None, - tool_calls: Some(vec![ToolCall { - id: "call_split".to_string(), - call_type: "function".to_string(), - function: FunctionCall { - name: "lookup_thing".to_string(), - arguments: r#"{"query":"x"}"#.to_string(), - }, - }]), - tool_call_id: None, - }); - // idx 4: matching tool result — naive range summarizes the call but - // preserves the result, orphaning the pair. - messages.push(ChatMessage { - role: "tool".to_string(), - content: Some(MessageContent::Text("lookup result payload".to_string())), - tool_calls: None, - tool_call_id: Some("call_split".to_string()), - }); - for i in 0..7 { - messages.push(ChatMessage { + #[tokio::test] + async fn apply_summary_layer_rebuilds_messages_and_persists() { + let store = crate::memory::MemoryStore::open_in_memory().unwrap(); + let conv = store + .get_or_create_conversation("test", "layer_u1") + .await + .unwrap(); + let mut cm = manager(vec![ + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("old request".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { role: "assistant".to_string(), - content: Some(MessageContent::Text(format!( - "filler {} {}", - "y".repeat(120), - i - ))), + content: Some(MessageContent::Text("old reply".to_string())), tool_calls: None, tool_call_id: None, - }); - } - // len == 12: naive summarize_count = min(11 * 0.70, 12 - 9) = 3 → mid-pair. - let mut cm = manager(messages); - let llm = failing_llm(); + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("latest request".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ]); + cm.memory = store.clone(); + cm.conversation_id = conv.clone(); - let result = cm.compact_messages(&llm, 1_000).await.unwrap(); - assert!(result, "compaction must happen"); + cm.apply_summary_layer("layer one content", 3) + .await + .unwrap(); - let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); - // Tool call rendered by NAME, not as [Assistant] ACTION. + // Rebuilt: system + summary block + tail from index 3. + assert_eq!(cm.messages.len(), 3); + assert_eq!(cm.messages[0].role, "system"); + assert_eq!(cm.messages[1].role, "system"); assert!( - summary_text.contains("[Tool: lookup_thing]"), - "missing tool-call marker: {summary_text}" + cm.messages[1] + .content + .as_ref() + .unwrap() + .as_text() + .contains("Previously compacted context:\nlayer one content"), + "summary injected as system message: {}", + cm.messages[1].content.as_ref().unwrap().as_text() ); - // The pair was summarized together: the result is in the summary. + assert_eq!( + cm.messages[2].content.as_ref().unwrap().as_text(), + "latest request" + ); + + // Second layer extends, not replaces. + cm.apply_summary_layer("layer two content", 2) + .await + .unwrap(); + let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); assert!( - summary_text.contains("lookup result payload"), - "tool result must be summarized with its call: {summary_text}" + summary_text.contains("layer one content") + && summary_text.contains("layer two content"), + "layered extension: {summary_text}" ); - // Preserved tail: no orphaned tool result or call for call_split. - for m in &cm.messages[2..] { - assert_ne!( - m.tool_call_id.as_deref(), - Some("call_split"), - "preserved tail must not contain a tool result whose call was summarized" - ); - assert!( - !m.has_tool_calls() - || !m - .tool_calls - .as_ref() - .unwrap() - .iter() - .any(|c| c.id == "call_split"), - "preserved tail must not contain a summarized tool call" - ); - } + assert_eq!( + cm.summary.as_deref().unwrap(), + "layer one content\n\nlayer two content" + ); + + // Persisted: [SUMMARY] rows reload. + let reloaded = store.load_messages(&conv).await.unwrap(); + let summary_rows: Vec = reloaded + .iter() + .filter_map(|m| { + m.content + .as_ref() + .map(|c| c.as_text()) + .filter(|t| t.starts_with("[SUMMARY]")) + }) + .collect(); + assert_eq!(summary_rows.len(), 2, "one [SUMMARY] row per layer"); + assert!(summary_rows[0].contains("layer one content")); + assert!(summary_rows[1].contains("layer two content")); + } + + #[tokio::test] + async fn apply_summary_layer_rejects_empty() { + let mut cm = manager(vec![ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }]); + assert!(cm.apply_summary_layer(" ", 1).await.is_err()); } #[tokio::test] @@ -557,14 +565,89 @@ mod tests { }; let before = texts(&cm); let llm = failing_llm(); + let ctx = CompactionContext { + llm: &llm, + context_window: 100_000, + compaction_model: None, + user_model_path: None, + }; - let result = cm.compact_messages(&llm, 100_000).await.unwrap(); + let result = cm.compact_messages(&ctx).await.unwrap(); assert!(!result, "tiny conversation must not trigger compaction"); assert_eq!(texts(&cm), before, "messages must be unchanged"); } + #[test] + fn compact_range_boundary_lands_after_tool_pair() { + use crate::agent_prompt::protected_tail_start; + use crate::llm::{FunctionCall, ToolCall}; + + let mut messages = vec![ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("system prompt".to_string())), + tool_calls: None, + tool_call_id: None, + }]; + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "first request {}", + "x".repeat(100) + ))), + tool_calls: None, + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![ToolCall { + id: "call_split".to_string(), + call_type: "function".to_string(), + function: FunctionCall { + name: "lookup_thing".to_string(), + arguments: r#"{"query":"x"}"#.to_string(), + }, + }]), + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("lookup result payload".to_string())), + tool_calls: None, + tool_call_id: Some("call_split".to_string()), + }); + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "second request {}", + "x".repeat(100) + ))), + tool_calls: None, + tool_call_id: None, + }); + + let start = protected_tail_start(&messages, 1_000_000); + // Boundary must not orphan the pair: both call and result are either + // both in the tail or both summarized. + let call_in_tail = messages[start..].iter().any(|m| { + m.has_tool_calls() + && m.tool_calls + .as_ref() + .is_some_and(|calls| calls.iter().any(|c| c.id == "call_split")) + }); + let result_in_tail = messages[start..] + .iter() + .any(|m| m.tool_call_id.as_deref() == Some("call_split")); + assert_eq!( + call_in_tail, result_in_tail, + "tool pair must not be split at the boundary (start={start})" + ); + } + #[tokio::test] - async fn compact_messages_sync_fallback_produces_markers() { + async fn compact_messages_defers_on_llm_failure_never_truncates() { + use crate::agent_prompt::{estimate_tokens, protected_tail_start}; + let mut messages = vec![ChatMessage { role: "system".to_string(), content: Some(MessageContent::Text("system prompt".to_string())), @@ -574,74 +657,166 @@ mod tests { messages.push(ChatMessage { role: "user".to_string(), content: Some(MessageContent::Text(format!( - "user question {} {}", - "x".repeat(90), - 0 + "UNIQUE_KEYWORD_A long initial request {}", + "x".repeat(900) ))), tool_calls: None, tool_call_id: None, }); - for i in 0..10 { + for i in 0..15 { messages.push(ChatMessage { role: "assistant".to_string(), - content: Some(MessageContent::Text(format!( - "assistant reply {} {}", - "x".repeat(80), - i - ))), - tool_calls: None, + content: None, + tool_calls: Some(vec![crate::llm::ToolCall { + id: format!("call_{i}"), + call_type: "function".to_string(), + function: crate::llm::FunctionCall { + name: "search".to_string(), + arguments: format!(r#"{{"q":"{}"}}"#, "y".repeat(120)), + }, + }]), tool_call_id: None, }); - let result = if i == 3 { - format!("Error: file not found {}", "y".repeat(70)) - } else { - format!("tool result {} {}", "y".repeat(80), i) - }; messages.push(ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text(result)), + content: Some(MessageContent::Text(format!( + "tool result {}", + "z".repeat(200) + ))), tool_calls: None, - tool_call_id: Some(format!("tool_{i}")), + tool_call_id: Some(format!("call_{i}")), }); } - let last_content = messages.last().unwrap().content.clone(); + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text( + "UNIQUE_KEYWORD_B follow-up request".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }); + let mut cm = manager(messages); let llm = failing_llm(); + let original_len = cm.messages.len(); + let window = estimate_tokens(&cm.messages); + assert!(window > 0); + + let ctx = CompactionContext { + llm: &llm, + context_window: window, + compaction_model: None, + user_model_path: None, + }; + let result = cm.compact_messages(&ctx).await.unwrap(); - let result = cm.compact_messages(&llm, 1_000).await.unwrap(); - assert!(result, "LLM failure must still compact via sync fallback"); - - // system + 1 summary entry + newest 8 preserved - assert_eq!(cm.messages.len(), 10); - assert_eq!(cm.messages[0].role, "system"); - assert_eq!(cm.messages[1].role, "user"); - assert_eq!(cm.messages[1].tool_call_id.as_deref(), Some("summary")); + // LLM failure → defer: no compaction, no truncation, nothing lost. + assert!(!result, "must defer when summarization fails"); + assert_eq!(cm.messages.len(), original_len, "messages unchanged"); - let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); + let texts: Vec = cm + .messages + .iter() + .map(|m| m.content.as_ref().map(|c| c.as_text()).unwrap_or_default()) + .collect(); + assert!( + texts.iter().any(|t| t.contains("UNIQUE_KEYWORD_A")), + "initial request preserved verbatim" + ); assert!( - summary_text.contains("★ COMPACTED CONTEXT — 13 messages summarized ★"), - "missing heading: {}", - summary_text + texts.last().unwrap().contains("UNIQUE_KEYWORD_B"), + "latest user intent preserved verbatim" ); - assert!(summary_text.contains("[User] TOPIC:"), "{summary_text}"); assert!( - summary_text.contains("[Assistant] ACTION:"), - "{summary_text}" + texts + .iter() + .all(|t| t.len() >= 200 || !t.contains("UNIQUE_KEYWORD_A")), + "no 200-char truncation anywhere" ); - assert!(summary_text.contains("[Tool: tool_3]"), "{summary_text}"); - assert!(summary_text.contains("| status: error"), "{summary_text}"); - // Preserved tail: newest 8 messages verbatim, in order. + // Second attempt: protected tail must include both user turns. + let tail = protected_tail_start(&cm.messages, window); + assert!( + cm.messages[tail..].iter().any(|m| m + .content + .as_ref() + .map(|c| c.as_text()) + .is_some_and(|t| t.contains("UNIQUE_KEYWORD_B"))), + "protected tail contains the latest user turn" + ); + } + + #[tokio::test] + async fn compact_success_path_preserves_user_intent() { + let store = crate::memory::MemoryStore::open_in_memory().unwrap(); + let conv = store + .get_or_create_conversation("test", "compact_u1") + .await + .unwrap(); + let mut cm = manager(vec![ + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "UNIQUE_KEYWORD_A old request {}", + "x".repeat(800) + ))), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("old reply".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("middle message".to_string())), + tool_calls: None, + tool_call_id: None, + }, + ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text( + "UNIQUE_KEYWORD_B follow-up".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }, + ]); + cm.conversation_id = conv.clone(); + + let tail = crate::agent_prompt::protected_tail_start(&cm.messages, 1_000_000); + assert_eq!( + tail, 3, + "old request + reply summarized, follow-up protected" + ); + cm.apply_summary_layer("user asked about UNIQUE_KEYWORD_A topic", tail) + .await + .unwrap(); + + // System message at index 1 carries the summary; the latest intent is verbatim. + assert_eq!(cm.messages[1].role, "system"); + let summary_text = cm.messages[1].content.as_ref().unwrap().as_text(); + assert!( + summary_text.contains("UNIQUE_KEYWORD_A"), + "summary preserves the old intent: {summary_text}" + ); assert_eq!( cm.messages .last() .unwrap() .content .as_ref() - .map(|c| c.as_text()), - last_content.as_ref().map(|c| c.as_text()) + .unwrap() + .as_text(), + "UNIQUE_KEYWORD_B follow-up" ); - assert_eq!(cm.messages[2].role, "assistant"); - assert!(cm.messages[2].content.as_ref().unwrap().as_text().len() > 50); } } diff --git a/src/learning.rs b/src/learning.rs index 13746db..21c3cd3 100644 --- a/src/learning.rs +++ b/src/learning.rs @@ -473,54 +473,23 @@ pub fn truncate_to(value: &str, max_chars: usize) -> String { value.chars().take(max_chars).collect() } -/// Update the user model by summarizing recent conversations through the LLM. -pub async fn update_user_model( - llm: &LlmClient, - memory: &crate::memory::MemoryStore, - user_model_path: &Path, -) { - match update_user_model_inner(llm, memory, user_model_path).await { - Ok(true) => info!("User model updated: {}", user_model_path.display()), - Ok(false) => info!("User model: not enough data to update"), - Err(e) => warn!("User model update failed: {:#}", e), - } -} - -async fn update_user_model_inner( - llm: &LlmClient, - memory: &crate::memory::MemoryStore, - user_model_path: &Path, -) -> Result { - // Load recent conversation messages for context. - let recent = memory - .search_messages("user preferences interests communication", 20) - .await - .unwrap_or_default(); - - if recent.len() < MIN_MESSAGES_FOR_USER_MODEL { - return Ok(false); // Not enough data yet - } - - let conversation_snippets: String = recent +/// Shared: format message excerpts for the user-model update prompt. +fn format_snippets(messages: &[&ChatMessage]) -> String { + messages .iter() + .filter(|m| m.role == "user" || m.role == "assistant") .filter_map(|m| { m.content .as_ref() .map(|c| format!("[{}]: {}", m.role, c.as_text())) }) .collect::>() - .join("\n"); - - // Read existing model. - let existing = if user_model_path.exists() { - tokio::fs::read_to_string(user_model_path) - .await - .unwrap_or_default() - } else { - DEFAULT_USER_MODEL.to_string() - }; + .join("\n") +} - let prompt = format!( +/// Shared: build the user-model update prompt from existing content + snippets. +fn build_user_model_prompt(existing: &str, snippets: &str) -> String { + format!( "You maintain a concise user profile for an AI assistant.\n\ \n\ Current user model:\n```\n{existing}\n```\n\ @@ -534,11 +503,45 @@ async fn update_user_model_inner( - Be concise — max 500 words total\n\ - Only add information the user explicitly stated or strongly implied\n\ - Do not remove existing valid entries — merge new info\n\ - - Output the COMPLETE updated file (frontmatter + body), nothing else", - existing = existing, - snippets = conversation_snippets, - ); + - Output the COMPLETE updated file (frontmatter + body), nothing else" + ) +} +/// Shared: validated write with `.bak` backup before overwrite. +async fn write_user_model_with_backup(user_model_path: &Path, new_content: &str) -> Result<()> { + if let Some(parent) = user_model_path.parent() { + tokio::fs::create_dir_all(parent).await.ok(); + } + if user_model_path.exists() { + let mut bak_path = user_model_path.to_string_lossy().to_string(); + bak_path.push_str(".bak"); + let _ = tokio::fs::copy(user_model_path, &bak_path).await; + } + tokio::fs::write(user_model_path, new_content) + .await + .with_context(|| format!("Failed to write user model: {}", user_model_path.display()))?; + Ok(()) +} + +/// Shared: prompt → LLM (optionally model-overridden) → frontmatter-validated +/// write. Returns `Ok(false)` when there is nothing to write. +async fn write_user_model_from_snippets( + llm: &LlmClient, + user_model_path: &Path, + snippets: &str, + model: Option<&str>, +) -> Result { + if snippets.trim().is_empty() { + return Ok(false); + } + let existing = if user_model_path.exists() { + tokio::fs::read_to_string(user_model_path) + .await + .unwrap_or_default() + } else { + DEFAULT_USER_MODEL.to_string() + }; + let prompt = build_user_model_prompt(&existing, snippets); let messages = vec![ChatMessage { role: "user".to_string(), content: Some(MessageContent::Text(prompt)), @@ -546,7 +549,14 @@ async fn update_user_model_inner( tool_call_id: None, }]; - let response = llm.chat(&messages, &[]).await?; + let response = match model { + Some(m) => { + llm.chat_completion_with_model(&messages, &[], m) + .await? + .message + } + None => llm.chat(&messages, &[]).await?, + }; let new_content = response.content.unwrap_or_default().as_text(); // Strict validation: must start with `---` and contain a closing `---` @@ -557,37 +567,54 @@ async fn update_user_model_inner( return Ok(false); } - // Ensure parent directory exists. - if let Some(parent) = user_model_path.parent() { - tokio::fs::create_dir_all(parent).await.ok(); - } + write_user_model_with_backup(user_model_path, &new_content).await?; + info!("User model updated: {}", user_model_path.display()); + Ok(true) +} - // Create backup before overwriting - if user_model_path.exists() { - let mut bak_path = user_model_path.to_string_lossy().to_string(); - bak_path.push_str(".bak"); - let bak = std::path::PathBuf::from(&bak_path); - let _ = tokio::fs::copy(user_model_path, &bak).await; +/// Pre-compaction flush (ADR 0003 Q5): bank durable facts from the +/// to-be-summarized range into USER.md so compaction cannot erase them. +pub async fn flush_user_model( + llm: &LlmClient, + user_model_path: &Path, + range: &[&ChatMessage], + model: Option<&str>, +) -> Result { + let snippets = format_snippets(range); + write_user_model_from_snippets(llm, user_model_path, &snippets, model).await +} + +/// Update the user model by summarizing recent conversations through the LLM. +pub async fn update_user_model( + llm: &LlmClient, + memory: &crate::memory::MemoryStore, + user_model_path: &Path, +) { + match update_user_model_inner(llm, memory, user_model_path).await { + Ok(true) => info!("User model updated: {}", user_model_path.display()), + Ok(false) => info!("User model: not enough data to update"), + Err(e) => warn!("User model update failed: {:#}", e), } +} - tokio::fs::write(user_model_path, &new_content) +async fn update_user_model_inner( + llm: &LlmClient, + memory: &crate::memory::MemoryStore, + user_model_path: &Path, +) -> Result { + // Load recent conversation messages for context. + let recent = memory + .search_messages("user preferences interests communication", 20) .await - .with_context(|| format!("Failed to write user model: {}", user_model_path.display()))?; + .unwrap_or_default(); - // Log diff summary - let old_lines: usize = existing.lines().count(); - let new_lines: usize = new_content.lines().count(); - let added = new_lines.saturating_sub(old_lines); - let removed = old_lines.saturating_sub(new_lines); - tracing::info!( - "User model updated: {} ({} lines, +{}/-{})", - user_model_path.display(), - new_lines, - added, - removed - ); + if recent.len() < MIN_MESSAGES_FOR_USER_MODEL { + return Ok(false); // Not enough data yet + } - Ok(true) + let refs: Vec<&ChatMessage> = recent.iter().collect(); + let snippets = format_snippets(&refs); + write_user_model_from_snippets(llm, user_model_path, &snippets, None).await } // ─── Feature 4: Self-Update ───────────────────────────────────────────────── @@ -923,6 +950,29 @@ mod tests { use super::*; use tempfile::tempdir; + #[tokio::test] + async fn test_flush_user_model_writes_valid_content() { + use crate::llm::{ChatMessage, MessageContent}; + + let dir = tempfile::tempdir().unwrap(); + let _path = dir.path().join("USER.md"); + let msg = ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text( + "I prefer replies in Traditional Chinese and short answers.".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }; + + let snippets = format_snippets(&[&msg]); + assert!(snippets.contains("[user]: I prefer replies")); + + // Frontmatter validation gate. + assert!(has_valid_frontmatter("---\nname: user-model\n---\n\nbody")); + assert!(!has_valid_frontmatter("no frontmatter here")); + } + #[test] fn test_extract_line_value() { let text = "SKILL_NAME: my-new-skill\nSKILL_DESCRIPTION: Does things\n"; diff --git a/src/loop_runner.rs b/src/loop_runner.rs index 453c645..0ddbee9 100644 --- a/src/loop_runner.rs +++ b/src/loop_runner.rs @@ -27,7 +27,7 @@ pub type ToolHandlerFn = Box< pub struct LoopConfig { pub max_iterations: u32, pub empty_response_retry_limit: u32, - pub compaction_enabled: bool, + pub context_window: usize, pub loop_detection_enabled: bool, pub interactive_loop_callback: bool, pub allowed_tools: Option>, @@ -94,29 +94,16 @@ impl<'a> AgenticLoop<'a> { user_id: &str, chat_id: &str, ) -> Result { - let context_window = 128_000; + let context_window = self.config.context_window; let mut empty_count = 0u32; - let mut last_compact_turn = 0usize; - for iteration in 0..self.config.max_iterations { + for _iteration in 0..self.config.max_iterations { if let Some(ref cancel) = self.cancel { if cancel.is_cancelled() { return Ok(LoopOutcome::Cancelled); } } - if self.config.compaction_enabled - && iteration > 0 - && ((iteration as usize).saturating_sub(last_compact_turn) >= 5 - || last_compact_turn == 0) - { - if let MessageContainer::Conversation(cm) = messages { - if let Ok(true) = cm.compact_messages(self.llm, context_window).await { - last_compact_turn = iteration as usize; - } - } - } - let prepared = messages.prepare(context_window); let tool_defs = if let Some(ref whitelist) = self.config.allowed_tools { diff --git a/tests/compaction_preserves_user_intent.rs b/tests/compaction_preserves_user_intent.rs new file mode 100644 index 0000000..2c6ac20 --- /dev/null +++ b/tests/compaction_preserves_user_intent.rs @@ -0,0 +1,157 @@ +//! Regression test: compaction must never lose the user's request, even when +//! the summarizer fails (ADR 0003 Q7). Uses an in-memory store and an LLM +//! client whose provider always fails (empty base_url → relative URL → no +//! network traffic). + +use std::collections::HashMap; +use std::sync::Arc; + +use rustfox::config::ProviderType; +use rustfox::conversation::{CompactionContext, ConversationManager}; +use rustfox::llm::{ChatMessage, LlmClient, MessageContent}; +use rustfox::memory::MemoryStore; +use rustfox::provider::{OpenRouterProvider, ProviderConfig, ProviderRegistry}; + +fn failing_llm() -> LlmClient { + let config = ProviderConfig { + name: "test".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: String::new(), + api_key: None, + default_model: "test-model".to_string(), + supports_vision: false, + max_tokens: 100, + discover_models: false, + context_window: 4096, + context_window_cache: Arc::new(tokio::sync::RwLock::new(None)), + parse_retry_limit: 0, + }; + let provider: Arc = Arc::new(OpenRouterProvider::new(config)); + let mut providers = HashMap::new(); + providers.insert("test".to_string(), provider); + LlmClient::new(Arc::new(ProviderRegistry::new( + providers, + "test".to_string(), + ))) +} + +fn msg(role: &str, text: &str) -> ChatMessage { + ChatMessage { + role: role.to_string(), + content: Some(MessageContent::from_text(text.to_string())), + tool_calls: None, + tool_call_id: None, + } +} + +#[tokio::test] +async fn compaction_never_loses_user_request() { + let store = MemoryStore::open_in_memory().unwrap(); + let conv = store + .get_or_create_conversation("telegram", "intent_u1") + .await + .unwrap(); + + // Seed history: long initial request A + 15 tool exchanges + follow-up B. + let mut history: Vec = vec![msg( + "user", + &format!("UNIQUE_KEYWORD_A initial request {}", "x".repeat(900)), + )]; + for i in 0..15 { + history.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![rustfox::llm::ToolCall { + id: format!("call_{i}"), + call_type: "function".to_string(), + function: rustfox::llm::FunctionCall { + name: "search".to_string(), + arguments: format!(r#"{{"q":"{}"}}"#, "y".repeat(120)), + }, + }]), + tool_call_id: None, + }); + history.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::from_text(format!( + "tool result {}", + "z".repeat(200) + ))), + tool_calls: None, + tool_call_id: Some(format!("call_{i}")), + }); + } + for m in &history { + store.save_message(&conv, m).await.unwrap(); + } + + // Load via the real conversation path, then add the live follow-up. + let mut cmgr = ConversationManager::new( + &store, + "telegram", + "intent_u1", + "system prompt".to_string(), + &rustfox::skills::SkillRegistry::new(), + &minimal_config(), + ) + .await + .unwrap(); + cmgr.add_user_turn(msg("user", "UNIQUE_KEYWORD_B follow-up request")); + let original_len = cmgr.messages().len(); + + let llm = failing_llm(); + let window = rustfox::agent_prompt::estimate_tokens(cmgr.messages()); + let ctx = CompactionContext { + llm: &llm, + context_window: window, + compaction_model: None, + user_model_path: None, + }; + + // Two passes: both must defer (LLM failure), never truncate. + for pass in 0..2 { + let compacted = cmgr.compact_messages(&ctx).await.unwrap(); + assert!(!compacted, "pass {pass}: must defer on summarizer failure"); + assert_eq!( + cmgr.messages().len(), + original_len, + "pass {pass}: messages unchanged" + ); + } + + let texts: Vec = cmgr + .messages() + .iter() + .map(|m| m.content.as_ref().map(|c| c.as_text()).unwrap_or_default()) + .collect(); + assert!( + texts.iter().any(|t| t.contains("UNIQUE_KEYWORD_A")), + "initial request preserved verbatim" + ); + assert!( + texts.last().unwrap().contains("UNIQUE_KEYWORD_B"), + "latest user intent preserved verbatim and last" + ); +} + +fn minimal_config() -> rustfox::config::Config { + // keep(): the temp dir must outlive the loaded config file. + let dir = tempfile::tempdir().unwrap().keep(); + let path = dir.join("config.toml"); + std::fs::write( + &path, + r#" +[telegram] +bot_token = "test" +allowed_user_ids = [1] + +[openrouter] +api_key = "test" + +[sandbox] +allowed_directory = "." +"#, + ) + .unwrap(); + rustfox::config::Config::load(&path).unwrap() +}