Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1128,6 +1128,7 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet
}),
success: true,
result_for_assistant: None,
image_attachments: None,
error: None,
duration_ms: Some(outcome.duration_ms),
}),
Expand Down Expand Up @@ -1203,6 +1204,7 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet
result: serde_json::Value::Null,
success: false,
result_for_assistant: None,
image_attachments: None,
error: Some(error.to_string()),
duration_ms: None,
}),
Expand Down
4 changes: 4 additions & 0 deletions src/crates/assembly/core/src/agentic/memories/transcript.rs
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,7 @@ mod tests {
result_for_assistant: Some(
"Fetched page with token=ghp_abcdefghijklmnopqrstuvwxyz".to_string(),
),
image_attachments: None,
error: None,
duration_ms: Some(1),
}),
Expand Down Expand Up @@ -499,6 +500,7 @@ mod tests {
r#"{"description":"full schema definition","input_schema":{"type":"object"}}"#
.to_string(),
),
image_attachments: None,
error: None,
duration_ms: Some(1),
}),
Expand Down Expand Up @@ -558,6 +560,7 @@ mod tests {
result_for_assistant: Some(
"external page content that should not enter memory extraction".to_string(),
),
image_attachments: None,
error: None,
duration_ms: Some(1),
}),
Expand Down Expand Up @@ -617,6 +620,7 @@ mod tests {
result: json!({"content": "r".repeat(TOOL_RESULT_TOKEN_LIMIT * 5)}),
success: true,
result_for_assistant: None,
image_attachments: None,
error: Some("e".repeat(TOOL_ERROR_TOKEN_LIMIT * 5)),
duration_ms: Some(1),
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4984,6 +4984,7 @@ impl SessionManager {
tool_id,
result,
result_for_assistant,
image_attachments,
is_error,
..
} = &msg.content
Expand All @@ -4998,6 +4999,7 @@ impl SessionManager {
result: result.clone(),
success: !is_error,
result_for_assistant: assistant_text,
image_attachments: image_attachments.clone(),
error: if *is_error {
serde_json::to_string(result).ok()
} else {
Expand Down Expand Up @@ -7474,6 +7476,7 @@ mod tests {
}),
success: true,
result_for_assistant: Some(assistant_output.clone()),
image_attachments: None,
error: None,
duration_ms: Some(1),
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,13 @@ impl ToolStateManager {
} => result_for_assistant.clone(),
_ => None,
},
image_attachments: match result {
crate::agentic::tools::framework::ToolResult::Result {
image_attachments,
..
} => image_attachments.clone(),
_ => None,
},
duration_ms: *duration_ms,
queue_wait_ms: *queue_wait_ms,
preflight_ms: *preflight_ms,
Expand Down
2 changes: 2 additions & 0 deletions src/crates/assembly/core/src/service/session_usage/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2321,6 +2321,7 @@ mod tests {
}),
success: false,
result_for_assistant: None,
image_attachments: None,
error: Some("operation timed out".to_string()),
duration_ms: Some(95_000),
});
Expand Down Expand Up @@ -2783,6 +2784,7 @@ mod tests {
result: serde_json::json!({}),
success,
result_for_assistant: None,
image_attachments: None,
error: (!success).then(|| "tool failed".to_string()),
duration_ms: Some(duration_ms),
}),
Expand Down
27 changes: 27 additions & 0 deletions src/crates/contracts/events/src/agentic.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Agentic Events Definition
pub use bitfun_core_types::errors::{AiErrorDetail, ErrorCategory};
use bitfun_core_types::ToolImageAttachment;
use serde::{Deserialize, Serialize};
use std::time::SystemTime;

Expand Down Expand Up @@ -444,6 +445,8 @@ pub enum ToolEventData {
result: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
result_for_assistant: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
image_attachments: Option<Vec<ToolImageAttachment>>,
duration_ms: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
queue_wait_ms: Option<u64>,
Expand Down Expand Up @@ -727,6 +730,7 @@ mod tests {
identity: ToolEventIdentity::direct("tool-1", "write_file"),
result: serde_json::json!({ "ok": true }),
result_for_assistant: None,
image_attachments: None,
duration_ms: 120,
queue_wait_ms: Some(10),
preflight_ms: Some(20),
Expand Down Expand Up @@ -764,6 +768,29 @@ mod tests {
assert_eq!(decoded.effective_tool_name(), "CreatePlan");
}

#[test]
fn completed_tool_serializes_image_attachments() {
let event = ToolEventData::Completed {
identity: ToolEventIdentity::direct("tool-image-1", "view_image"),
result: serde_json::json!({ "path": "preview.png" }),
result_for_assistant: Some("Image attached".to_string()),
image_attachments: Some(vec![bitfun_core_types::ToolImageAttachment {
mime_type: "image/png".to_string(),
data_base64: "AAAA".to_string(),
}]),
duration_ms: 12,
queue_wait_ms: None,
preflight_ms: None,
confirmation_wait_ms: None,
execution_ms: Some(12),
};

let json = serde_json::to_value(&event).expect("serialize tool event");

assert_eq!(json["image_attachments"][0]["mime_type"], "image/png");
assert_eq!(json["image_attachments"][0]["data_base64"], "AAAA");
}

#[test]
fn failed_tool_reports_best_effort_total_duration() {
let event = ToolEventData::Failed {
Expand Down
1 change: 1 addition & 0 deletions src/crates/contracts/events/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ pub use backend::{
BackgroundCommandLifecycleInfo, ToolExecutionCompletedInfo, ToolExecutionErrorInfo,
ToolExecutionProgressInfo, ToolExecutionStartedInfo, ToolTerminalReadyInfo,
};
pub use bitfun_core_types::ToolImageAttachment;
pub use emitter::EventEmitter;
pub use frontend_projection::{project_agentic_frontend_event, AgenticFrontendEvent};
pub use types::*;
36 changes: 35 additions & 1 deletion src/crates/execution/tool-execution/src/pipeline.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Provider-neutral tool pipeline planning helpers.

use bitfun_events::{ToolEventData, ToolEventIdentity};
use bitfun_events::{ToolEventData, ToolEventIdentity, ToolImageAttachment};
use dashmap::DashMap;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
Expand Down Expand Up @@ -111,6 +111,7 @@ pub enum ToolStateEventKind {
Completed {
result: serde_json::Value,
result_for_assistant: Option<String>,
image_attachments: Option<Vec<ToolImageAttachment>>,
duration_ms: u64,
queue_wait_ms: Option<u64>,
preflight_ms: Option<u64>,
Expand Down Expand Up @@ -305,6 +306,7 @@ pub fn tool_state_event_data(facts: ToolStateEventFacts) -> ToolEventData {
ToolStateEventKind::Completed {
result,
result_for_assistant,
image_attachments,
duration_ms,
queue_wait_ms,
preflight_ms,
Expand All @@ -314,6 +316,7 @@ pub fn tool_state_event_data(facts: ToolStateEventFacts) -> ToolEventData {
identity,
result: sanitize_tool_result_for_event(&result),
result_for_assistant,
image_attachments,
duration_ms,
queue_wait_ms,
preflight_ms,
Expand Down Expand Up @@ -383,6 +386,36 @@ mod tests {
use bitfun_events::{ToolEventData, ToolEventIdentity};
use serde_json::json;

#[test]
fn completed_event_preserves_image_attachments() {
let data = tool_state_event_data(ToolStateEventFacts {
identity: ToolEventIdentity::direct("tool-image-1", "view_image"),
state: ToolStateEventKind::Completed {
result: json!({ "path": "preview.png" }),
result_for_assistant: Some("Image attached".to_string()),
image_attachments: Some(vec![bitfun_events::ToolImageAttachment {
mime_type: "image/png".to_string(),
data_base64: "AAAA".to_string(),
}]),
duration_ms: 10,
queue_wait_ms: None,
preflight_ms: None,
confirmation_wait_ms: None,
execution_ms: Some(10),
},
});

let ToolEventData::Completed {
image_attachments, ..
} = data
else {
panic!("expected completed event");
};
let attachments = image_attachments.expect("image attachments");
assert_eq!(attachments[0].mime_type, "image/png");
assert_eq!(attachments[0].data_base64, "AAAA");
}

#[test]
fn completed_event_redacts_data_urls_recursively() {
let data = tool_state_event_data(ToolStateEventFacts {
Expand All @@ -393,6 +426,7 @@ mod tests {
"nested": [{ "data_url": "data:image/png;base64,BBBB" }]
}),
result_for_assistant: Some("done".to_string()),
image_attachments: None,
duration_ms: 10,
queue_wait_ms: Some(1),
preflight_ms: None,
Expand Down
2 changes: 2 additions & 0 deletions src/crates/interfaces/acp/src/client/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ fn acp_tool_call_events(
Some(tool_call.locations),
),
result_for_assistant: None,
image_attachments: None,
duration_ms: 0,
queue_wait_ms: None,
preflight_ms: None,
Expand Down Expand Up @@ -320,6 +321,7 @@ fn acp_tool_call_update_events(
update.fields.locations,
),
result_for_assistant: None,
image_attachments: None,
duration_ms: 0,
queue_wait_ms: None,
preflight_ms: None,
Expand Down
2 changes: 2 additions & 0 deletions src/crates/interfaces/acp/src/runtime/events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,7 @@ mod tests {
identity: identity("Bash"),
result: serde_json::json!({ "stdout": "ok" }),
result_for_assistant: Some("done".to_string()),
image_attachments: None,
duration_ms: 42,
queue_wait_ms: None,
preflight_ms: None,
Expand Down Expand Up @@ -880,6 +881,7 @@ mod tests {
"success": true,
}),
result_for_assistant: None,
image_attachments: None,
duration_ms: 15,
queue_wait_ms: None,
preflight_ms: None,
Expand Down
8 changes: 7 additions & 1 deletion src/crates/services/services-core/src/session/types.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Types for session persistence

use bitfun_core_types::SessionKind;
use bitfun_core_types::{SessionKind, ToolImageAttachment};
use serde::{Deserialize, Serialize};

pub const SESSION_STORAGE_SCHEMA_VERSION: u32 = 2;
Expand Down Expand Up @@ -759,6 +759,12 @@ pub struct ToolResultData {
alias = "result_for_assistant"
)]
pub result_for_assistant: Option<String>,
#[serde(
default,
skip_serializing_if = "Option::is_none",
alias = "image_attachments"
)]
pub image_attachments: Option<Vec<ToolImageAttachment>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", alias = "duration_ms")]
Expand Down
4 changes: 4 additions & 0 deletions src/web-ui/src/flow_chat/services/EventBatcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,10 @@ export type RejectedToolEvent = BaseToolEvent<'Rejected'>;
export interface CompletedToolEvent extends BaseToolEvent<'Completed'> {
result: unknown;
result_for_assistant?: string;
image_attachments?: Array<{
mime_type: string;
data_base64: string;
}>;
duration_ms: number;
queue_wait_ms?: number;
preflight_ms?: number;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,54 @@ describe('processToolEvent rejected event behavior', () => {
});
});

describe('processToolEvent completed image behavior', () => {
afterEach(() => {
resetStore();
});

it('preserves image attachments on the completed tool item', () => {
const tool: FlowToolItem = {
id: 'tool-image-1',
type: 'tool',
toolName: 'view_image',
timestamp: 1001,
status: 'running',
toolCall: {
id: 'tool-image-1',
input: { path: 'screenshots/preview.png' },
},
};

FlowChatStore.getInstance().setState(() => ({
sessions: new Map([['session-1', createSessionWithTool(tool)]]),
activeSessionId: 'session-1',
}));

processToolEvent(
makeToolContext(),
'session-1',
'turn-1',
'round-1',
{
event_type: 'Completed',
tool_id: 'tool-image-1',
tool_name: 'view_image',
result: { path: 'screenshots/preview.png', width: 80, height: 60 },
image_attachments: [{ mime_type: 'image/png', data_base64: 'AAAA' }],
duration_ms: 12,
},
);

const updatedTool = FlowChatStore.getInstance()
.findToolItem('session-1', 'turn-1', 'tool-image-1') as FlowToolItem;

expect(updatedTool.status).toBe('completed');
expect(updatedTool.toolResult?.imageAttachments).toEqual([
{ mime_type: 'image/png', data_base64: 'AAAA' },
]);
});
});

describe('processToolEvent AskUserQuestion retry superseded handling', () => {
afterEach(() => {
resetStore();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -539,6 +539,7 @@ function handleCompleted(
result: toolEvent.result,
success: true,
resultForAssistant: toolEvent.result_for_assistant,
imageAttachments: toolEvent.image_attachments,
duration_ms: toolEvent.duration_ms
},
status: 'completed' as const,
Expand Down
Loading
Loading