diff --git a/docs/agents/system-prompt.mdx b/docs/agents/system-prompt.mdx index ed2a73bd0de..b2454ba533c 100644 --- a/docs/agents/system-prompt.mdx +++ b/docs/agents/system-prompt.mdx @@ -55,7 +55,7 @@ When the user asks for "best of n" work, assume they want the \`task\` tool's \` Before spawning the batch, do a small amount of preliminary analysis to capture shared context, constraints, or evaluation criteria that would otherwise be repeated by every child. Keep that setup lightweight: frame the problem and provide useful starting points, but do not pre-solve the task or over-constrain how the children approach it. Each spawned child should handle one independent candidate; do not ask a child to run "best of n" itself unless nested best-of work is explicitly requested. -Picking the best candidate requires every report, so await the full batch (pass \`task_await\` \`min_completed\` equal to the batch size, or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. +Picking the best candidate requires every report, so await the full batch with \`task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })\` (or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. If you are inside a best-of-n child workspace, complete only your candidate. diff --git a/docs/hooks/tools.mdx b/docs/hooks/tools.mdx index 2c3af99d6e9..d5ffc58a7ca 100644 --- a/docs/hooks/tools.mdx +++ b/docs/hooks/tools.mdx @@ -333,17 +333,17 @@ If a value is too large for the environment, it may be omitted (not set). Mux al
bash (9) -| Env var | JSON path | Type | Description | -| --------------------------------------- | ------------------------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. | -| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. | -| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. | -| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. | -| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. | -| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await (returns only new output since last check). Stop with task_stop using the taskId. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the returned taskId first. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. | -| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute | -| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive | +| Env var | JSON path | Type | Description | +| --------------------------------------- | ------------------------ | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. | +| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. | +| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. | +| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. | +| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. | +| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with `task\_await({ task\_ids: [result.taskId] })` (returns only new output since last check). Stop with `task\_stop({ task\_ids: [result.taskId] })`. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. | +| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute | +| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |
@@ -693,8 +693,8 @@ If a value is too large for the environment, it may be omitted (not set). Mux al | `MUX_TOOL_INPUT_FILTER` | `filter` | string | Optional regex to filter bash task output lines. By default, only matching lines are returned. When filter_exclude is true, matching lines are excluded instead. Non-matching lines are discarded and cannot be retrieved later. | | `MUX_TOOL_INPUT_FILTER_EXCLUDE` | `filter_exclude` | boolean | When true, lines matching 'filter' are excluded instead of kept. Requires 'filter' to be set. | | `MUX_TOOL_INPUT_MIN_COMPLETED` | `min_completed` | number | Number of awaited tasks that must complete before this call returns. Defaults to 1, so by default task_await returns as soon as the FIRST awaited task completes, letting you act on it while the rest keep running. The result still includes every task complete at that moment plus current status (running/queued) for the rest. Tasks that have not yet completed keep running and remain re-awaitable on a later task_await call. Raise this (e.g. set it to the total number of awaited tasks) when you genuinely need more before proceeding — for example best-of-N synthesis that must compare every candidate. Clamped to the number of awaited tasks; values above that behave like 'wait for all'. | -| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. | -| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) | +| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. | +| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) | | `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Maximum time to wait in seconds for each task. For bash tasks, this waits for NEW output (or process exit). If exceeded, the result returns status=queued\|starting\|running\|awaiting_report (task is still active). Defaults to 600 seconds (10 minutes) if not specified. Set to 0 for a non-blocking status check. | @@ -737,7 +737,7 @@ If a value is too large for the environment, it may be omitted (not set). Mux al | ------------------------------------ | --------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- | | `MUX_TOOL_INPUT_MESSAGE` | `message` | string | Updated guidance to send to the sub-agent. | | `MUX_TOOL_INPUT_QUEUE_DISPATCH_MODE` | `queue_dispatch_mode` | enum | When the child is busy, dispatch the guidance at "tool-end" after its next tool call (default) or at "turn-end" after its current turn. | -| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active descendant sub-agent task ID returned by task or task_list. | +| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active or inactive persistent descendant sub-agent task ID returned by task or task_list. | @@ -798,19 +798,19 @@ If a value is too large for the environment, it may be omitted (not set). Mux al | ---------------------------------- | ------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `MUX_TOOL_INPUT_MODE` | `mode` | enum | Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. Use 'retry_from_checkpoint' only for failed runs; it re-executes work after the last checkpoint and is rejected when unsafe. | | `MUX_TOOL_INPUT_RUN_ID` | `run_id` | string | Workflow run ID (wfr\_...) to resume. Must belong to the current workspace. | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await the runId with task_await when you need the result. | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with `task\_await({ task\_ids: [result.runId] })` when you need the result. |
workflow_run (4) -| Env var | JSON path | Type | Description | -| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using the result. | -| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. | -| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. | +| Env var | JSON path | Type | Description | +| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with `task\_await({ task\_ids: [result.runId] })` before using the result. | +| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. | +| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |
diff --git a/src/cli/workflow.ts b/src/cli/workflow.ts index 7ae4a933260..c04c0c503d1 100644 --- a/src/cli/workflow.ts +++ b/src/cli/workflow.ts @@ -436,6 +436,8 @@ function createWorkflowService(input: { return new WorkflowService({ runStore: new WorkflowRunStore({ sessionDir: workspaceSessionDir }), runtimeFactory: new QuickJSRuntimeFactory(), + withRunStartLock: (ownerWorkspaceId, operation) => + input.ctx.services.taskService.withWorkspaceOwnedWorkStartLock(ownerWorkspaceId, operation), taskAdapterFactory: (runId) => new WorkflowTaskServiceAdapter({ taskService: input.ctx.services.taskService, @@ -445,6 +447,8 @@ function createWorkflowService(input: { experiments, modelString: input.model, thinkingLevel: input.thinkingLevel, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + input.ctx.services.backgroundProcessManager.cleanup(taskWorkspaceId), getProjectTrusted: () => input.ctx.projectTrusted, patchToolConfig: { workspaceId: input.ctx.workspaceId, diff --git a/src/common/utils/tools/toolDefinitions.ts b/src/common/utils/tools/toolDefinitions.ts index 6ab3ac78dd8..2324e8965d3 100644 --- a/src/common/utils/tools/toolDefinitions.ts +++ b/src/common/utils/tools/toolDefinitions.ts @@ -320,7 +320,7 @@ export function buildTaskToolDescription(runtimeMode: RuntimeMode | undefined): "\n\nWhen the user explicitly asks for best-of-n work, the parent should begin with light preliminary analysis to extract shared context, constraints, or evaluation criteria that would otherwise be duplicated across children. " + "Keep that pre-work lightweight: frame the task and provide useful starting points, but do not pre-solve the problem or over-constrain how the children reason about it. Then delegate the substantive analysis to the spawned sub-agents. " + "Do not also do a full parallel analysis in the parent. Call task_await when you are ready to act on child output; do not await reflexively just because tasks are running. " + - "task_await returns as soon as the first awaited task completes by default (min_completed), so you can start dependent work on each result as it lands instead of blocking on the whole batch; for best-of-N synthesis that must compare every candidate, pass min_completed equal to the batch size (or use a foreground grouped spawn, below). " + + "task_await returns as soon as the first awaited task completes by default (min_completed), so you can start dependent work on each result as it lands instead of blocking on the whole batch. Pass returned camelCase IDs through task_await's snake_case input: `task_await({ task_ids: [result.taskId] })` for one handle, or `task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })` when every grouped result is required (or use a foreground grouped spawn, below). " + "\n\nWhen delegating, include a compact task brief (Task / Background / Scope / Starting points / Acceptance / Deliverables / Constraints). " + "For now, persisted sub-agent goals are not supported; pass sub-agent objectives, success criteria, and deliverables directly in the prompt. " + "Sub-agents observe the same system instructions as the parent (project/global AGENTS.md and custom instructions), so do not restate that shared context in the prompt; spend the prompt on task-specific information the sub-agent cannot infer from those instructions. " + @@ -617,8 +617,8 @@ export const TaskAwaitToolArgsSchema = z .nullish() .describe( "List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. " + - "task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. " + - "When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs." + "task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. " + + "When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs." ), filter: z .string() @@ -725,7 +725,7 @@ const TaskAwaitToolArtifactsSchema = z * and can be re-fetched by ID after context compaction instead of re-running the work. */ export const COMPLETED_REPORT_REFETCH_NOTE = - 'Report persisted on disk; re-fetch anytime (even after context compaction) with task_await(task_ids: [""], timeout_secs: 0).'; + 'Report persisted on disk; re-fetch anytime (even after context compaction) with `task_await({ task_ids: [""], timeout_secs: 0 })`.'; export const TaskAwaitToolCompletedResultSchema = z .object({ @@ -946,7 +946,9 @@ export const TaskSendMessageToolArgsSchema = z task_id: z .string() .min(1) - .describe("Active descendant sub-agent task ID returned by task or task_list."), + .describe( + "Active or inactive persistent descendant sub-agent task ID returned by task or task_list." + ), message: z.string().trim().min(1).describe("Updated guidance to send to the sub-agent."), queue_dispatch_mode: z .enum(["tool-end", "turn-end"]) @@ -1131,7 +1133,6 @@ export const TaskRemoveToolResultSchema = z TaskRemoveToolBaseResultSchema.extend({ status: z.literal("removed") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("already_removed") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("active") }).strict(), - TaskRemoveToolBaseResultSchema.extend({ status: z.literal("not_found") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("invalid_scope") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("error") }).strict(), ]) @@ -1140,7 +1141,7 @@ export const TaskRemoveToolResultSchema = z .strict(); // ----------------------------------------------------------------------------- -// task_terminate (terminate sub-agent/bash tasks, interrupt workflow runs) +// task_terminate (legacy schema retained to render historical tool calls) // ----------------------------------------------------------------------------- export const TaskTerminateToolArgsSchema = z .object({ @@ -1256,7 +1257,7 @@ export const TaskWorkspaceLifecycleToolArgsSchema = z .boolean() .nullish() .describe( - "When true, interrupt active workspace turns for the target before performing an otherwise-eligible lifecycle action. Active sub-agents must be discarded with task_terminate instead. Defaults to false." + "When true, interrupt active workspace turns for the target before performing an otherwise-eligible lifecycle action. Active sub-agents must be stopped with task_stop instead. Defaults to false." ), force: z .boolean() @@ -1405,7 +1406,7 @@ export const WorkflowRunToolArgsSchema = z .default(false) .describe( "Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. " + - "Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using the result." + "Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with `task_await({ task_ids: [result.runId] })` before using the result." ), }) .strict() @@ -1448,7 +1449,7 @@ export const WorkflowResumeToolArgsSchema = z .default(false) .describe( "Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. " + - "Set true to resume in the background and continue other work; await the runId with task_await when you need the result." + "Set true to resume in the background and continue other work; await it with `task_await({ task_ids: [result.runId] })` when you need the result." ), mode: WorkflowResumeModeSchema.nullish().describe( "Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. " + @@ -1681,15 +1682,15 @@ export const TOOL_DEFINITIONS = { "Do NOT use for quick commands (<5s), interactive processes (no stdin support), " + "or processes requiring real-time output (use foreground with larger timeout instead). " + "Returns immediately with a taskId (bash:) and backgroundProcessId. " + - "Read output with task_await (returns only new output since last check). " + - "Stop with task_stop using the taskId. " + + "Read output with `task_await({ task_ids: [result.taskId] })` (returns only new output since last check). " + + "Stop with `task_stop({ task_ids: [result.taskId] })`. " + "List active tasks with task_list. " + "Process persists until timeout_secs expires, terminated, or workspace is removed." + "\\n\\nFor long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. " + "Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. " + "With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. " + "Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. " + - "Do not call task_await in the same parallel tool-call batch; wait for the returned taskId first. " + + "Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. " + "When you actually need the output, read it with task_await; do not poll task_await just because the process is still running." ), monitor: BashMonitorSchema.nullish().describe( @@ -2243,8 +2244,8 @@ export const TOOL_DEFINITIONS = { "When a terminal wake-up says a sub-agent report or failure is already injected into context, integrate it directly — do NOT call task_await for it. When a wake-up asks you to retrieve a workspace turn's terminal output, call task_await with the listed IDs and timeout_secs: 0 (a one-shot retrieval, not a wait). " + "\n\nIMPORTANT: Do not call task_await in the same parallel tool-call batch as task, bash, or workflow_run — " + "the taskId/runId is not available until the spawning tool returns. " + - "Always wait for the task/bash/workflow_run tool result first, then call task_await in a subsequent step. " + - "When omitting task_ids to await active tasks/workflows, ensure at least one background task or workflow was already spawned in a prior step. Omitted task_ids discover top-level workflow runs only and exclude workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. " + + "Always wait for the task/bash/workflow_run tool result first, then call task_await in a subsequent step. Map returned result fields explicitly: `task_await({ task_ids: [result.taskId] })` or `task_await({ task_ids: result.taskIds })`. " + + "When omitting task_ids to await active tasks/workflows, ensure at least one background task or workflow was already spawned in a prior step. Omitted task_ids discover active in-scope descendant agent tasks, public workspace turns, background bash tasks, and top-level workflow runs, while excluding workflow-owned internal workers because their results are consumed through parent workflow runs. " + "\n\nAgent tasks and workflow runs return reports when completed. " + "Completed reports are persisted on disk and survive context compaction: calling task_await on an already-completed task/workflow run ID (timeout_secs: 0 for non-blocking) re-fetches the full report instead of re-running the work. " + "Bash tasks return incremental output while running and a final reportMarkdown when they exit. " + @@ -2263,8 +2264,8 @@ export const TOOL_DEFINITIONS = { }, task_send_message: { description: - "Send guidance to a descendant sub-agent. Queued/running work is interrupted or queued at the requested boundary so the child can incorporate the update. An inactive child is reawakened in the same persistent workspace under a fresh internal execution. " + - "The stable sub-agent task ID and durable role title remain unchanged. If the new assignment changes the child's reusable responsibility, call task_retitle as well; do not retitle it for ordinary one-off assignments. Best-of children retain candidate metadata, so reawaken them only to continue that same candidate; use a standalone specialist for unrelated work. This tool does not target bash tasks, workflow runs, or workspace-turn handles.", + "Send guidance to a user-owned descendant sub-agent. Queued/running work is interrupted or queued at the requested boundary so the child can incorporate the update. An inactive child is reawakened in the same persistent workspace under a fresh internal execution. " + + "The stable sub-agent task ID and durable role title remain unchanged. If the new assignment changes the child's reusable responsibility, call task_retitle as well; do not retitle it for ordinary one-off assignments. Best-of children retain candidate metadata, so reawaken them only to continue that same candidate; use a standalone specialist for unrelated work. This tool does not target workflow-owned internal workers, bash tasks, workflow runs, or workspace-turn handles.", schema: TaskSendMessageToolArgsSchema, }, task_retitle: { @@ -2274,18 +2275,18 @@ export const TOOL_DEFINITIONS = { }, task_stop: { description: - "Stop one or more tasks without removing persistent child workspaces. Sub-agent trees are stopped leaf-first and unfinished children become interrupted; workspace turns and workflow runs are interrupted; bash processes are terminated. Use this to cancel or abandon work, not to mark useful progress as completed—ask a child to finalize with task_send_message and await its report instead. Stopping an already-inactive task is idempotent.", + "Stop one or more user-owned tasks without removing persistent child workspaces. Sub-agent trees are stopped leaf-first and unfinished children become interrupted; workspace turns and workflow runs are interrupted; bash processes are terminated. Workflow-owned internal workers are controlled through their owning workflow run and return invalid_scope when targeted directly. Use this to cancel or abandon work, not to mark useful progress as completed—ask a child to finalize with task_send_message and await its report instead. Stopping any already-terminal handle is idempotent.", schema: TaskStopToolArgsSchema, }, task_remove: { description: - "Irreversibly remove inactive child task workspaces owned by the current workspace. Removed sub-agents cannot be restored or reawakened. Active targets are rejected; descendants must be removed first, so nested batches are processed deepest-first.", + "Irreversibly remove inactive user-owned child task workspaces. Workflow-owned internal workers are controlled through their owning workflow run and return invalid_scope when targeted directly. Removed sub-agents cannot be restored or reawakened. Active targets are rejected; descendants must be removed first, so nested batches are processed deepest-first.", schema: TaskRemoveToolArgsSchema, }, task_list: { description: "List descendant tasks for the current workspace, including status + metadata. " + - "This includes sub-agent tasks, background bash tasks, and top-level workflow runs, but omits workflow-owned sub-agents/background bash tasks whose reports are consumed through parent workflow runs. " + + "This includes sub-agent tasks, public workspace turns, background bash tasks, and top-level workflow runs, but omits workflow-owned sub-agents/background bash tasks whose reports are consumed through parent workflow runs. " + "Use this after compaction, interruptions, workflow_run errors/aborts, or an app restart to rediscover active tasks, inactive persistent sub-agents, and resumable workflow runs. The default statuses find unfinished work; request `reported` explicitly for completed persistent sub-agents. " + "When recovering an uncertain workflow_run, omit statuses first or include pending/running/backgrounded as well as interrupted/failed/completed; terminal-only filters can hide unfinished workflow runs. Pending runs may need workflow_resume because no runner may be active yet. " + "Workflow rows may include compact `workflowProgress` so callers can see the latest phase before deciding whether to await, resume, or leave the run alone. " + @@ -2302,7 +2303,7 @@ export const TOOL_DEFINITIONS = { "the conductor follows the documented phases more faithfully and gains durable checkpoints, resume, and fresh delegated context per phase. " + "Use agent_skill_read / agent_skill_read_file to discover and inspect skill-packaged workflows; non-skill workflow files must be addressed by an explicit known path and can be inspected with normal file tools. " + "Prefer the default foreground mode (`run_in_background` omitted or false) so completed workflows return their result without an extra task_await round-trip. " + - "If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using or reporting the workflow output. " + + "If workflow_run returns status=running or status=backgrounded, await it with `task_await({ task_ids: [result.runId] })` before using or reporting the workflow output. " + "After a previous workflow_run error, abort, timeout, or uncertain result, do not start a fresh run until you rediscover existing workflow runs: either omit task_list statuses first, or query pending/running/backgrounded/interrupted/failed/completed together. " + "Use task_await for running/backgrounded runs, workflow_resume for pending/interrupted runs, workflow_resume({ mode: 'retry_from_checkpoint' }) only for eligible failed runs, and inspect/refetch completed results instead of rerunning. " + "Use background mode only when you intend to start another workflow/task or do independent work while the workflow runs; a background run is non-blocking and Mux wakes this workspace with the terminal workflow result, so call task_await only when the current request depends on the output before you can answer.", @@ -2316,7 +2317,7 @@ export const TOOL_DEFINITIONS = { "For failed runs, pass mode='retry_from_checkpoint' explicitly; it re-executes work after the last checkpoint, so only use it when that is acceptable, and start a fresh workflow_run when it is rejected as unsafe. " + "Calling this on a completed run returns its existing result without re-running anything. " + "Prefer foreground mode (run_in_background omitted or false) to get the final result directly; " + - "if the returned status is running or backgrounded, await the runId with task_await before using the result.", + "if the returned status is running or backgrounded, await it with `task_await({ task_ids: [result.runId] })` before using the result.", schema: WorkflowResumeToolArgsSchema, }, agent_report: { diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index ef41a0b22ac..deb0fe85fa0 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -140,6 +140,67 @@ export interface WorkflowServiceScriptInput { sourceKind: "skill" | "workspace-file" | "inline"; } +export interface ToolWorkflowService { + getRun?(input: { workspaceId: string; runId: string }): Promise; + listRuns?(input: { workspaceId: string }): Promise; + startWorkflowInBackground?(input: { + script: WorkflowServiceScriptInput; + workspaceId: string; + projectTrusted: boolean; + args: unknown; + attentionPolicy?: BackgroundWorkAttentionPolicy; + onRunCreated?: (event: { + runId: string; + status: "pending"; + result: null; + run: unknown; + }) => Promise | void; + }): Promise<{ runId: string; status: string; result: unknown }>; + startWorkflow?(input: { + script: WorkflowServiceScriptInput; + workspaceId: string; + projectTrusted: boolean; + args: unknown; + abortSignal?: AbortSignal; + onRunCreated?: (event: { + runId: string; + status: "pending"; + result: null; + run: unknown; + }) => Promise | void; + }): Promise<{ runId: string; status: string; result: unknown }>; + interruptRun?(input: { + workspaceId: string; + runId: string; + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }): Promise; + resumeRun?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + abortSignal?: AbortSignal; + }): Promise<{ runId: string; status: string; result: unknown }>; + resumeRunInBackground?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + }): Promise<{ runId: string; status: string; result: unknown }>; + retryRunFromCheckpoint?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + abortSignal?: AbortSignal; + }): Promise<{ runId: string; status: string; result: unknown }>; + retryRunFromCheckpointInBackground?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + }): Promise<{ runId: string; status: string; result: unknown }>; +} + export interface ToolConfiguration { /** Working directory for command execution - actual path in runtime's context (local or remote) */ cwd: string; @@ -202,59 +263,9 @@ export interface ToolConfiguration { /** Task orchestration for sub-agent tasks */ taskService?: TaskService; /** Durable workflow lifecycle service for dynamic workflow tools. */ - workflowService?: { - getRun?(input: { workspaceId: string; runId: string }): Promise; - listRuns?(input: { workspaceId: string }): Promise; - startWorkflowInBackground?(input: { - script: WorkflowServiceScriptInput; - workspaceId: string; - projectTrusted: boolean; - args: unknown; - attentionPolicy?: BackgroundWorkAttentionPolicy; - onRunCreated?: (event: { - runId: string; - status: "pending"; - result: null; - run: unknown; - }) => Promise | void; - }): Promise<{ runId: string; status: string; result: unknown }>; - startWorkflow?(input: { - script: WorkflowServiceScriptInput; - workspaceId: string; - projectTrusted: boolean; - args: unknown; - abortSignal?: AbortSignal; - onRunCreated?: (event: { - runId: string; - status: "pending"; - result: null; - run: unknown; - }) => Promise | void; - }): Promise<{ runId: string; status: string; result: unknown }>; - interruptRun?(input: { workspaceId: string; runId: string }): Promise; - resumeRun?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - abortSignal?: AbortSignal; - }): Promise<{ runId: string; status: string; result: unknown }>; - resumeRunInBackground?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - }): Promise<{ runId: string; status: string; result: unknown }>; - retryRunFromCheckpoint?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - abortSignal?: AbortSignal; - }): Promise<{ runId: string; status: string; result: unknown }>; - retryRunFromCheckpointInBackground?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - }): Promise<{ runId: string; status: string; result: unknown }>; - }; + workflowService?: ToolWorkflowService; + /** Resolve the workflow lifecycle service for a descendant workspace's session. */ + workflowServiceForWorkspace?: (workspaceId: string) => ToolWorkflowService | null; /** Workspace heartbeat settings service for model-facing heartbeat configuration. */ workspaceHeartbeatService?: WorkspaceHeartbeatToolService; /** Workspace goal lifecycle service for model-facing goal tools. */ @@ -765,7 +776,7 @@ export async function getToolsForModel( task_remove: wrap(createTaskRemoveTool(config)), task_list: wrap(createTaskListTool(config)), - // Bash execution (foreground/background). Manage background output via task_await/task_list/task_terminate. + // Bash execution (foreground/background). Manage background output via task_await/task_list/task_stop. bash: wrap(createBashTool(config)), // Legacy bash process tools (deprecated) diff --git a/src/node/builtinSkills/orchestrate.md b/src/node/builtinSkills/orchestrate.md index eb07ac90811..f785b051c24 100644 --- a/src/node/builtinSkills/orchestrate.md +++ b/src/node/builtinSkills/orchestrate.md @@ -138,7 +138,7 @@ In a workflow, the verifier becomes `agent(prompt, { id, schema, onRefusal: "fai ## Sequential protocol (only for dependency chains) 1. Spawn the prerequisite `exec` implementation task with `run_in_background: false`. -2. If step 1 returns `queued`/`running` without a completed report, call `task_await` with the returned `taskId` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application. +2. If step 1 returns `queued`/`running` without a completed report, call `task_await({ task_ids: [result.taskId] })` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application. 3. Dry-run apply its patch (`dry_run: true`); then apply for real (`dry_run: false`). If either step fails, follow the conflict playbook above (including `git am --abort` only when a real apply leaves a git-am session in progress). 4. Only then spawn the dependent task. diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index 51b1d5f7214..ab42995698e 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -128,6 +128,7 @@ describe("router workflow routes", () => { config, aiService: { waitForInit: mock(async () => undefined), + cleanupWorkspaceBackgroundProcesses: mock(async () => undefined), getWorkspaceMetadata: mock(async () => ({ success: true, data: { @@ -149,7 +150,13 @@ describe("router workflow routes", () => { getWorkflowContinuationSendOptions: mock(() => null), sendMessage: mock(async () => ({ success: true, data: undefined })), }, - taskService: {}, + taskService: { + withWorkspaceOwnedWorkStartLock: mock( + async (_workspaceId: string, operation: () => Promise) => await operation() + ), + terminateAllDescendantAgentTasks: mock(async () => []), + markWorkflowRunEnded: mock(async () => undefined), + }, experimentsService: { isExperimentEnabled: mock(() => options.enabled), }, @@ -616,6 +623,9 @@ export default function workflow() { return { reportMarkdown: "should not run" } let waitCalls = 0; context.taskService = { + withWorkspaceOwnedWorkStartLock: mock( + async (_workspaceId: string, operation: () => Promise) => await operation() + ), create: mock(async () => ({ success: true, data: { taskId: "task_slow" } })), waitForAgentReport: mock(async () => { waitCalls += 1; @@ -624,6 +634,8 @@ export default function workflow() { return { reportMarkdown: "should not run" } } return { reportMarkdown: "done", structuredOutput: {} }; }), + terminateAllDescendantAgentTasks: mock(async () => []), + markWorkflowRunEnded: mock(async () => undefined), } as unknown as ORPCContext["taskService"]; const client = createRouterClient(router(), { context }); diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index a94fcada3d1..ff284f9fe00 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -457,6 +457,8 @@ export async function resolveWorkflowContext( options.notifyInterruptedBackgroundRunTerminal === true, runStore: new WorkflowRunStore({ sessionDir: context.config.getSessionDir(workspaceId) }), runtimeFactory: context.workflowRuntimeFactory, + withRunStartLock: (ownerWorkspaceId, operation) => + context.taskService.withWorkspaceOwnedWorkStartLock(ownerWorkspaceId, operation), taskAdapterFactory: (runId, workflowName) => new WorkflowTaskServiceAdapter({ taskService: context.taskService, @@ -472,6 +474,8 @@ export async function resolveWorkflowContext( workspaceSessionDir: context.config.getSessionDir(workspaceId), trusted: projectTrusted, }, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + context.aiService.cleanupWorkspaceBackgroundProcesses(taskWorkspaceId), getProjectTrusted: resolveWorkflowProjectTrusted, experiments: { dynamicWorkflows: true, diff --git a/src/node/services/agentSkills/builtInSkillContent.generated.ts b/src/node/services/agentSkills/builtInSkillContent.generated.ts index 119658ede5f..6e73e508429 100644 --- a/src/node/services/agentSkills/builtInSkillContent.generated.ts +++ b/src/node/services/agentSkills/builtInSkillContent.generated.ts @@ -2701,7 +2701,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "Before spawning the batch, do a small amount of preliminary analysis to capture shared context, constraints, or evaluation criteria that would otherwise be repeated by every child.", "Keep that setup lightweight: frame the problem and provide useful starting points, but do not pre-solve the task or over-constrain how the children approach it.", 'Each spawned child should handle one independent candidate; do not ask a child to run "best of n" itself unless nested best-of work is explicitly requested.', - "Picking the best candidate requires every report, so await the full batch (pass \\`task_await\\` \\`min_completed\\` equal to the batch size, or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands.", + "Picking the best candidate requires every report, so await the full batch with \\`task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })\\` (or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands.", "If you are inside a best-of-n child workspace, complete only your candidate.", "", "", @@ -5229,17 +5229,17 @@ export const BUILTIN_SKILL_FILES: Record> = { "
", "bash (9)", "", - "| Env var | JSON path | Type | Description |", - "| --------------------------------------- | ------------------------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", - "| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. |", - "| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. |", - "| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. |", - "| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. |", - "| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. |", - "| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await (returns only new output since last check). Stop with task_stop using the taskId. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the returned taskId first. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. |", - "| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute |", - "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |", + "| Env var | JSON path | Type | Description |", + "| --------------------------------------- | ------------------------ | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", + "| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. |", + "| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. |", + "| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. |", + "| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. |", + "| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. |", + "| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with `task\\_await({ task\\_ids: [result.taskId] })` (returns only new output since last check). Stop with `task\\_stop({ task\\_ids: [result.taskId] })`. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. |", + "| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute |", + "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |", "", "
", "", @@ -5589,8 +5589,8 @@ export const BUILTIN_SKILL_FILES: Record> = { "| `MUX_TOOL_INPUT_FILTER` | `filter` | string | Optional regex to filter bash task output lines. By default, only matching lines are returned. When filter_exclude is true, matching lines are excluded instead. Non-matching lines are discarded and cannot be retrieved later. |", "| `MUX_TOOL_INPUT_FILTER_EXCLUDE` | `filter_exclude` | boolean | When true, lines matching 'filter' are excluded instead of kept. Requires 'filter' to be set. |", "| `MUX_TOOL_INPUT_MIN_COMPLETED` | `min_completed` | number | Number of awaited tasks that must complete before this call returns. Defaults to 1, so by default task_await returns as soon as the FIRST awaited task completes, letting you act on it while the rest keep running. The result still includes every task complete at that moment plus current status (running/queued) for the rest. Tasks that have not yet completed keep running and remain re-awaitable on a later task_await call. Raise this (e.g. set it to the total number of awaited tasks) when you genuinely need more before proceeding — for example best-of-N synthesis that must compare every candidate. Clamped to the number of awaited tasks; values above that behave like 'wait for all'. |", - "| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. |", - "| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) |", + "| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. |", + "| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) |", "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Maximum time to wait in seconds for each task. For bash tasks, this waits for NEW output (or process exit). If exceeded, the result returns status=queued\\|starting\\|running\\|awaiting_report (task is still active). Defaults to 600 seconds (10 minutes) if not specified. Set to 0 for a non-blocking status check. |", "", "", @@ -5633,7 +5633,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "| ------------------------------------ | --------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- |", "| `MUX_TOOL_INPUT_MESSAGE` | `message` | string | Updated guidance to send to the sub-agent. |", '| `MUX_TOOL_INPUT_QUEUE_DISPATCH_MODE` | `queue_dispatch_mode` | enum | When the child is busy, dispatch the guidance at "tool-end" after its next tool call (default) or at "turn-end" after its current turn. |', - "| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active descendant sub-agent task ID returned by task or task_list. |", + "| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active or inactive persistent descendant sub-agent task ID returned by task or task_list. |", "", "", "", @@ -5694,19 +5694,19 @@ export const BUILTIN_SKILL_FILES: Record> = { "| ---------------------------------- | ------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", "| `MUX_TOOL_INPUT_MODE` | `mode` | enum | Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. Use 'retry_from_checkpoint' only for failed runs; it re-executes work after the last checkpoint and is rejected when unsafe. |", "| `MUX_TOOL_INPUT_RUN_ID` | `run_id` | string | Workflow run ID (wfr\\_...) to resume. Must belong to the current workspace. |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await the runId with task_await when you need the result. |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with `task\\_await({ task\\_ids: [result.runId] })` when you need the result. |", "", "", "", "
", "workflow_run (4)", "", - "| Env var | JSON path | Type | Description |", - "| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |", - "| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using the result. |", - '| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. |', - "| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |", + "| Env var | JSON path | Type | Description |", + "| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |", + "| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with `task\\_await({ task\\_ids: [result.runId] })` before using the result. |", + '| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. |', + "| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |", "", "
", "", @@ -7880,7 +7880,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "## Sequential protocol (only for dependency chains)", "", "1. Spawn the prerequisite `exec` implementation task with `run_in_background: false`.", - "2. If step 1 returns `queued`/`running` without a completed report, call `task_await` with the returned `taskId` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application.", + "2. If step 1 returns `queued`/`running` without a completed report, call `task_await({ task_ids: [result.taskId] })` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application.", "3. Dry-run apply its patch (`dry_run: true`); then apply for real (`dry_run: false`). If either step fails, follow the conflict playbook above (including `git am --abort` only when a real apply leaves a git-am session in progress).", "4. Only then spawn the dependent task.", "", diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index af1eccb4697..6d9e2d413e8 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -557,6 +557,10 @@ export class AIService extends EventEmitter { private analyticsService?: { executeRawQuery(sql: string): Promise }; private desktopSessionManager?: DesktopSessionManager; + async cleanupWorkspaceBackgroundProcesses(workspaceId: string): Promise { + await this.backgroundProcessManager?.cleanup(workspaceId); + } + constructor( config: Config, historyService: HistoryService, @@ -2186,6 +2190,41 @@ export class AIService extends EventEmitter { }); const getWorkflowProjectTrusted = () => isWorkspaceProjectTrusted(this.config, metadata); + const createWorkflowLifecycleService = (ownerWorkspaceId: string) => + new WorkflowService({ + runStore: new WorkflowRunStore({ + sessionDir: this.config.getSessionDir(ownerWorkspaceId), + }), + onRunStatusChanged: async (event) => { + if (!isTerminalWorkflowRunStatus(event.status)) { + await this.taskService?.resetWorkflowRunTerminalAttention({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } + await this.onWorkflowRunStatusChanged?.(event); + }, + runtimeFactory: new QuickJSRuntimeFactory(), + withRunStartLock: (lockedWorkspaceId, operation) => + this.taskService!.withWorkspaceOwnedWorkStartLock(lockedWorkspaceId, operation), + taskAdapterFactory: (runId, workflowName) => + new WorkflowTaskServiceAdapter({ + taskService: this.taskService!, + parentWorkspaceId: ownerWorkspaceId, + workflowRunId: runId, + workflowName, + defaultAgentId: DEFAULT_WORKFLOW_AGENT_ID, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + this.cleanupWorkspaceBackgroundProcesses(taskWorkspaceId), + experiments: { + ...experiments, + dynamicWorkflows: dynamicWorkflowsExperimentEnabled, + workspaceHeartbeats: workspaceHeartbeatsExperimentEnabled, + }, + }), + runnerId: `workflow-runner:${ownerWorkspaceId}`, + }); + const workflowService = dynamicWorkflowsExperimentEnabled && this.taskService != null ? new WorkflowService({ @@ -2202,6 +2241,8 @@ export class AIService extends EventEmitter { await this.onWorkflowRunStatusChanged?.(event); }, runtimeFactory: new QuickJSRuntimeFactory(), + withRunStartLock: (ownerWorkspaceId, operation) => + this.taskService!.withWorkspaceOwnedWorkStartLock(ownerWorkspaceId, operation), taskAdapterFactory: (runId, workflowName) => new WorkflowTaskServiceAdapter({ taskService: this.taskService!, @@ -2217,6 +2258,8 @@ export class AIService extends EventEmitter { workspaceSessionDir: this.config.getSessionDir(workspaceId), trusted: getWorkflowProjectTrusted(), }, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + this.cleanupWorkspaceBackgroundProcesses(taskWorkspaceId), getProjectTrusted: getWorkflowProjectTrusted, experiments: { ...experiments, @@ -2495,6 +2538,13 @@ export class AIService extends EventEmitter { muxScope, timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, + workflowServiceForWorkspace: + this.taskService != null + ? (ownerWorkspaceId) => + ownerWorkspaceId === workspaceId && workflowService != null + ? workflowService + : createWorkflowLifecycleService(ownerWorkspaceId) + : undefined, workflowService, goalService: workspaceGoalService, goalDefaults: effectiveGoalDefaults, diff --git a/src/node/services/systemMessage.ts b/src/node/services/systemMessage.ts index f4bc8f41431..dd349605a83 100644 --- a/src/node/services/systemMessage.ts +++ b/src/node/services/systemMessage.ts @@ -91,7 +91,7 @@ When the user asks for "best of n" work, assume they want the \`task\` tool's \` Before spawning the batch, do a small amount of preliminary analysis to capture shared context, constraints, or evaluation criteria that would otherwise be repeated by every child. Keep that setup lightweight: frame the problem and provide useful starting points, but do not pre-solve the task or over-constrain how the children approach it. Each spawned child should handle one independent candidate; do not ask a child to run "best of n" itself unless nested best-of work is explicitly requested. -Picking the best candidate requires every report, so await the full batch (pass \`task_await\` \`min_completed\` equal to the batch size, or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. +Picking the best candidate requires every report, so await the full batch with \`task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })\` (or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. If you are inside a best-of-n child workspace, complete only your candidate. diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4ad1017e040..5f1b490e741 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -2121,6 +2121,18 @@ describe("TaskService", () => { expect(aiMocks.stopStream).not.toHaveBeenCalled(); }); + test("interruptWorkspaceTurn is idempotent after interruption", async () => { + const { parentId, taskService, aiMocks } = await startWorkspaceTurnForTest(); + + expect(await taskService.interruptWorkspaceTurn(parentId, "wst_handle")).toEqual( + Ok({ workspaceId: "childworkspace" }) + ); + expect(await taskService.interruptWorkspaceTurn(parentId, "wst_handle")).toEqual( + Ok({ workspaceId: "childworkspace", alreadyInactive: true }) + ); + expect(aiMocks.stopStream).toHaveBeenCalledTimes(1); + }); + test("createWorkspaceTurn reserves a slot before queueing a manually busy existing workspace", async () => { const config = await createTestConfig(rootDir); stubStableIds(config, ["queuedhandle", "queuedturn"]); @@ -5300,7 +5312,7 @@ describe("TaskService", () => { test("late correlated stream-end does not resettle an explicitly interrupted workspace turn", async () => { const { config, parentId, taskService } = await startWorkspaceTurnForTest(); - // Explicit interrupt (user Esc / task_terminate): status interrupted WITHOUT the + // Explicit interrupt (user Esc / task_stop): status interrupted WITHOUT the // stale-restart marker. An in-flight stream-end completing after the cancel must not // make the canceled turn appear completed. await new TaskHandleStore(config).upsertWorkspaceTurn({ @@ -11852,6 +11864,97 @@ describe("TaskService", () => { expect(updateTitle).not.toHaveBeenCalled(); }); + test("direct lifecycle operations reject workflow-owned descendants", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-workflow-owned-lifecycle"; + const workflowChildId = "workflow-owned-lifecycle-root"; + const messageChildId = "workflow-owned-message-child"; + const stopChildId = "workflow-owned-stop-child"; + const removeChildId = "workflow-owned-remove-child"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "workflow-child", workflowChildId, { + parentWorkspaceId, + taskStatus: "running", + workflowTask: { runId: "wfr_lifecycle", stepId: "step" }, + }), + projectWorkspace(projectPath, "message-child", messageChildId, { + parentWorkspaceId: workflowChildId, + taskStatus: "queued", + taskPrompt: "Original workflow-owned assignment", + }), + projectWorkspace(projectPath, "stop-child", stopChildId, { + parentWorkspaceId: workflowChildId, + taskStatus: "running", + }), + projectWorkspace(projectPath, "remove-child", removeChildId, { + parentWorkspaceId: workflowChildId, + taskStatus: "reported", + }), + ], + testTaskSettings() + ); + const { taskService } = createTaskServiceHarness(config); + + expect( + await taskService.sendMessageToDescendantAgentTask( + parentWorkspaceId, + messageChildId, + "Bypass the owning workflow", + "tool-end" + ) + ).toEqual(Err({ code: "invalid_scope" })); + expect(await taskService.stopDescendantAgentTask(parentWorkspaceId, stopChildId)).toEqual( + Err("Task is not a descendant of this workspace") + ); + expect( + await taskService.removeInactiveDescendantAgentTask(parentWorkspaceId, removeChildId) + ).toMatchObject({ success: true, data: { status: "invalid_scope" } }); + }); + + test("stopDescendantAgentTask leaves workflow-owned descendant branches to WorkflowService", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-stop-workflow-branch"; + const childTaskId = "user-owned-stop-root"; + const workflowChildId = "workflow-owned-stop-descendant"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "child", childTaskId, { + parentWorkspaceId, + taskStatus: "running", + }), + projectWorkspace(projectPath, "workflow-child", workflowChildId, { + parentWorkspaceId: childTaskId, + taskStatus: "running", + workflowTask: { runId: "wfr_stop_branch", stepId: "step" }, + }), + ], + testTaskSettings() + ); + const stopStream = mock((): Promise> => Promise.resolve(Ok(undefined))); + const { aiService } = createAIServiceMocks(config, { + isStreaming: mock(() => true), + stopStream, + }); + const { taskService } = createTaskServiceHarness(config, { aiService }); + + expect(await taskService.stopDescendantAgentTask(parentWorkspaceId, childTaskId)).toEqual( + Ok({ stoppedTaskIds: [childTaskId] }) + ); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(stopStream).toHaveBeenCalledWith(childTaskId, { abandonPartial: false }); + expect(findWorkspaceInConfig(config, childTaskId)?.taskStatus).toBe("interrupted"); + expect(findWorkspaceInConfig(config, workflowChildId)?.taskStatus).toBe("running"); + }); + test("retitleDescendantAgentTask surfaces title update failures", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -12901,13 +13004,30 @@ describe("TaskService", () => { await stopStarted; const creation = createAgentTask(taskService, childTaskId, "Spawn after stop"); + const createWorkflowRun = mock(() => Promise.resolve("created")); + const workflowCreation = taskService.withWorkspaceOwnedWorkStartLock( + childTaskId, + createWorkflowRun + ); await Promise.resolve(); expect(create).not.toHaveBeenCalled(); + expect(createWorkflowRun).not.toHaveBeenCalled(); releaseStop?.(); expect(await stopping).toEqual(Ok({ stoppedTaskIds: [childTaskId] })); expect(await creation).toEqual(Err("Task.create: cannot spawn new tasks after task_stop")); + let workflowCreationError: unknown; + try { + await workflowCreation; + } catch (error: unknown) { + workflowCreationError = error; + } + expect(workflowCreationError).toBeInstanceOf(Error); + expect((workflowCreationError as Error).message).toBe( + "Cannot start workflow work after task_stop" + ); expect(create).not.toHaveBeenCalled(); + expect(createWorkflowRun).not.toHaveBeenCalled(); }); test("bulk task creation waits for task stop and rejects the interrupted parent", async () => { @@ -13050,6 +13170,25 @@ describe("TaskService", () => { expect(nestedTasks.every((workspace) => workspace.taskStatus === "queued")).toBe(true); }); + test("withWorkspaceOwnedWorkStartLock rejects missing workspaces", async () => { + const config = await createTestConfig(rootDir); + const { taskService } = createTaskServiceHarness(config); + const operation = mock(() => Promise.resolve("started")); + + let startError: unknown; + try { + await taskService.withWorkspaceOwnedWorkStartLock("removed-workspace", operation); + } catch (error: unknown) { + startError = error; + } + + expect(startError).toBeInstanceOf(Error); + expect((startError as Error).message).toBe( + "Cannot start workflow work from a missing workspace" + ); + expect(operation).not.toHaveBeenCalled(); + }); + test("task tree lifecycle locks serialize descendants with their ancestor", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -13260,6 +13399,112 @@ describe("TaskService", () => { ).toEqual(Err({ code: "not_found" })); }); + test("removeInactiveDescendantAgentTask removes archived workflow-owned descendants with their user-owned parent", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const ownerWorkspaceId = "owner-remove-archived-workflow"; + const parentTaskId = "parent-remove-archived-workflow"; + const workflowTaskId = "workflow-remove-archived-child"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "owner", ownerWorkspaceId), + projectWorkspace(projectPath, "parent", parentTaskId, { + parentWorkspaceId: ownerWorkspaceId, + taskStatus: "reported", + }), + projectWorkspace(projectPath, "workflow-child", workflowTaskId, { + parentWorkspaceId: parentTaskId, + taskStatus: "interrupted", + workflowTask: { runId: "wfr_remove_archived", stepId: "worker" }, + archivedAt: "2026-08-16T00:00:00.000Z", + }), + ], + testTaskSettings() + ); + const remove = mock(async (workspaceId: string): Promise> => { + await removeWorkspaceFromTestConfig(config, workspaceId); + return Ok(undefined); + }); + const { workspaceService } = createWorkspaceServiceMocks({ remove }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + expect( + await taskService.removeInactiveDescendantAgentTask(ownerWorkspaceId, parentTaskId) + ).toMatchObject({ success: true, data: { status: "removed", taskId: parentTaskId } }); + expect(remove.mock.calls.map((call) => call[0])).toEqual([workflowTaskId, parentTaskId]); + expect(findWorkspaceInConfig(config, workflowTaskId)).toBeUndefined(); + expect(findWorkspaceInConfig(config, parentTaskId)).toBeUndefined(); + }); + + test("rejected parent removals preserve archived workflow-owned descendants", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const ownerWorkspaceId = "owner-preserve-archived-workflow"; + const activeParentId = "active-parent-preserve-workflow"; + const activeWorkflowChildId = "active-parent-workflow-child"; + const blockedParentId = "blocked-parent-preserve-workflow"; + const userChildId = "blocked-parent-user-child"; + const blockedWorkflowChildId = "blocked-parent-workflow-child"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "owner", ownerWorkspaceId), + projectWorkspace(projectPath, "active-parent", activeParentId, { + parentWorkspaceId: ownerWorkspaceId, + taskStatus: "running", + }), + projectWorkspace(projectPath, "active-workflow-child", activeWorkflowChildId, { + parentWorkspaceId: activeParentId, + taskStatus: "interrupted", + workflowTask: { runId: "wfr_active_parent", stepId: "worker" }, + archivedAt: "2026-08-16T00:00:00.000Z", + }), + projectWorkspace(projectPath, "blocked-parent", blockedParentId, { + parentWorkspaceId: ownerWorkspaceId, + taskStatus: "reported", + }), + projectWorkspace(projectPath, "user-child", userChildId, { + parentWorkspaceId: blockedParentId, + taskStatus: "reported", + }), + projectWorkspace(projectPath, "blocked-workflow-child", blockedWorkflowChildId, { + parentWorkspaceId: blockedParentId, + taskStatus: "interrupted", + workflowTask: { runId: "wfr_blocked_parent", stepId: "worker" }, + archivedAt: "2026-08-16T00:00:00.000Z", + }), + ], + testTaskSettings() + ); + const remove = mock((): Promise> => Promise.resolve(Ok(undefined))); + const { workspaceService } = createWorkspaceServiceMocks({ remove }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + expect( + await taskService.removeInactiveDescendantAgentTask(ownerWorkspaceId, activeParentId) + ).toMatchObject({ success: true, data: { status: "active", taskId: activeParentId } }); + const blockedRemoval = await taskService.removeInactiveDescendantAgentTask( + ownerWorkspaceId, + blockedParentId + ); + expect(blockedRemoval).toMatchObject({ + success: true, + data: { status: "error", taskId: blockedParentId }, + }); + if (!blockedRemoval.success || blockedRemoval.data.status !== "error") { + throw new Error("Expected blocked parent removal to return an error lifecycle result"); + } + expect(new Set(blockedRemoval.data.descendantTaskIds ?? [])).toEqual( + new Set([userChildId, blockedWorkflowChildId]) + ); + expect(remove).not.toHaveBeenCalled(); + expect(findWorkspaceInConfig(config, activeWorkflowChildId)).toBeTruthy(); + expect(findWorkspaceInConfig(config, blockedWorkflowChildId)).toBeTruthy(); + }); + test("requestAgentFinalReportForTimeout records finalization token only after prompt send succeeds", async () => { const config = await createTestConfig(rootDir); @@ -13860,15 +14105,37 @@ describe("TaskService", () => { testTaskSettings() ); - const { aiService } = createAIServiceMocks(config); + const lifecycleEvents: string[] = []; + const stopStream = mock((workspaceId: string): Promise> => { + lifecycleEvents.push(`stop:${workspaceId}`); + return Promise.resolve(Ok(undefined)); + }); + const { aiService } = createAIServiceMocks(config, { stopStream }); const { workspaceService } = createWorkspaceServiceMocks(); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + const cleanupWorkspaceBackgroundProcesses = mock((workspaceId: string): Promise => { + lifecycleEvents.push(`cleanup:${workspaceId}`); + return workspaceId === workflowChildTaskId + ? Promise.reject(new Error("background cleanup failed")) + : Promise.resolve(); + }); const interruptedTaskIds = await taskService.terminateAllDescendantAgentTasks(rootWorkspaceId, { workflowRunId: "wfr_target", + cleanupWorkspaceBackgroundProcesses, }); expect(interruptedTaskIds).toEqual([workflowChildTaskId, workflowTaskId]); + expect(lifecycleEvents).toEqual([ + `stop:${workflowChildTaskId}`, + `cleanup:${workflowChildTaskId}`, + `stop:${workflowTaskId}`, + `cleanup:${workflowTaskId}`, + ]); + expect(cleanupWorkspaceBackgroundProcesses.mock.calls.map((call) => call[0])).toEqual([ + workflowChildTaskId, + workflowTaskId, + ]); const saved = config.loadConfigOrDefault(); const tasks = saved.projects.get(projectPath)?.workspaces ?? []; expect(tasks.find((workspace) => workspace.id === workflowTaskId)?.taskStatus).toBe( @@ -13917,9 +14184,18 @@ describe("TaskService", () => { const { workspaceService } = createWorkspaceServiceMocks(); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - expect( - new Set(taskService.listDescendantAgentTasks(rootWorkspaceId).map((task) => task.taskId)) - ).toEqual(new Set([regularTaskId, workflowChildTaskId, workflowTaskId])); + const descendants = taskService.listDescendantAgentTasks(rootWorkspaceId); + expect(new Set(descendants.map((task) => task.taskId))).toEqual( + new Set([regularTaskId, workflowChildTaskId, workflowTaskId]) + ); + expect(descendants.find((task) => task.taskId === workflowTaskId)).toMatchObject({ + workflowRunId: "wfr_target", + workflowOwnerWorkspaceId: rootWorkspaceId, + }); + expect(descendants.find((task) => task.taskId === workflowChildTaskId)).toMatchObject({ + workflowRunId: "wfr_target", + workflowOwnerWorkspaceId: rootWorkspaceId, + }); expect( taskService .listDescendantAgentTasks(rootWorkspaceId, { @@ -14774,7 +15050,7 @@ describe("TaskService", () => { expect(userInterrupted?.taskStatus).toBe("interrupted"); }); - test("terminateAllDescendantAgentTasks archives run-scoped interrupted children immediately", async () => { + test("terminateAllDescendantAgentTasks can defer run-scoped archive sweeps", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -14814,14 +15090,18 @@ describe("TaskService", () => { const { workspaceService } = createWorkspaceServiceMocks({ archive }); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - // Run-scoped interrupt (WorkflowService.interruptRun path): the sweep archives the - // freshly interrupted workflow child even if the runner's onRunEnded hook already - // fired before the children were interrupted. - await taskService.terminateAllDescendantAgentTasks(rootId, { workflowRunId }); + // task_stop can already hold the non-reentrant tree lock while interrupting the owning run. + // Defer archive work until that outer lock releases, then run the normal idempotent sweep. + await taskService.terminateAllDescendantAgentTasks(rootId, { + workflowRunId, + deferWorkflowSweep: true, + }); - const workflowChild = findWorkspaceInConfig(config, workflowChildId); - expect(workflowChild?.taskStatus).toBe("interrupted"); - expect(workflowChild?.archivedAt).toBeString(); + expect(findWorkspaceInConfig(config, workflowChildId)?.taskStatus).toBe("interrupted"); + expect(findWorkspaceInConfig(config, workflowChildId)?.archivedAt).toBeUndefined(); + + await taskService.markWorkflowRunEnded(workflowRunId); + expect(findWorkspaceInConfig(config, workflowChildId)?.archivedAt).toBeString(); // The run-scoped filter leaves the user-spawned sibling running and unarchived. const userChild = findWorkspaceInConfig(config, userChildId); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 539aa61ffac..6da013c4d29 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -192,6 +192,7 @@ export interface AgentTaskTimestamps { } type WorkspaceLifecycleResult = z.infer; +type RemoveInactiveAgentTaskResult = Exclude; export interface TaskCreateArgs { parentWorkspaceId: string; @@ -613,6 +614,10 @@ export interface DescendantAgentTaskInfo { executionStatus?: WorkspaceTurnTaskStatus; modelString?: string; thinkingLevel?: ThinkingLevel; + /** Effective owning workflow run for this task branch, inherited through workflow ancestry. */ + workflowRunId?: string; + /** Workspace session that persists the effective owning workflow run. */ + workflowOwnerWorkspaceId?: string; depth: number; } @@ -687,7 +692,7 @@ const WORKSPACE_TURN_STALE_RESTART_ERROR = "Workspace turn interrupted after res * Settled workspace-turn records eligible for self-heal correction (resettle from a * correlated stream-end, or read-time repair/revive): transient stream-error settlements * (status "error") and stale restart-recovery interrupts. Explicit interrupts — user Esc, - * task_terminate, cancel reasons — must stay terminal even if a late correlated stream-end + * task_stop, cancel reasons — must stay terminal even if a late correlated stream-end * or same-turn retry evidence arrives, so canceled work never resurfaces as completed. */ function isSelfHealEligibleSettledWorkspaceTurn( @@ -2095,6 +2100,38 @@ export class TaskService { return await acquire(0); } + async withWorkspaceOwnedWorkStartLock( + workspaceId: string, + operation: () => Promise + ): Promise { + assert(workspaceId.length > 0, "withWorkspaceOwnedWorkStartLock requires workspaceId"); + return await this.withTaskTreeLifecycleLock(workspaceId, async () => { + { + await using _lock = await this.mutex.acquire(); + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + if (entry == null) { + throw new Error("Cannot start workflow work from a missing workspace"); + } + if (isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt)) { + throw new Error("Cannot start workflow work from an archived workspace"); + } + if ( + entry?.workspace.taskStatus === "interrupted" && + !isActiveWorkspaceTurnTaskStatus(entry.workspace.taskExecutionStatus) + ) { + throw new Error("Cannot start workflow work after task_stop"); + } + if ( + entry?.workspace.taskStatus === "reported" && + !isActiveWorkspaceTurnTaskStatus(entry.workspace.taskExecutionStatus) + ) { + throw new Error("Cannot start workflow work after agent_report"); + } + } + return await operation(); + }); + } + private async editWorkspaceEntry( workspaceId: string, updater: (workspace: WorkspaceConfigEntry) => void, @@ -4444,7 +4481,10 @@ export class TaskService { taskIndex.parentById, ancestorWorkspaceId, taskId - ) + ) || + // Workflow-owned workers are private implementation details; their lifecycle is consumed + // through the owning workflow run rather than direct parent task tools. + this.isWorkflowOwnedTaskUsingIndex(taskIndex, taskId) ) { return Err({ code: "invalid_scope" as const }); } @@ -4487,7 +4527,8 @@ export class TaskService { taskIndex.parentById, ancestorWorkspaceId, taskId - ) + ) || + this.isWorkflowOwnedTaskUsingIndex(taskIndex, taskId) ) { return Err({ code: "invalid_scope" as const }); } @@ -4656,14 +4697,46 @@ export class TaskService { async stopDescendantAgentTask( ancestorWorkspaceId: string, - taskId: string + taskId: string, + options?: { beforeStop?: () => Promise } ): Promise> { assert(ancestorWorkspaceId.length > 0, "stopDescendantAgentTask: ancestorWorkspaceId required"); assert(taskId.length > 0, "stopDescendantAgentTask: taskId required"); - return await this.withTaskTreeLifecycleLock(taskId, () => - this.stopDescendantAgentTaskUnderLifecycleLock(ancestorWorkspaceId, taskId) - ); + return await this.withTaskTreeLifecycleLock(taskId, async () => { + const scopeResult = await this.validateDirectDescendantAgentTaskScope( + ancestorWorkspaceId, + taskId + ); + if (!scopeResult.success) { + return scopeResult; + } + const beforeStopError = await options?.beforeStop?.(); + if (beforeStopError != null) { + return Err(beforeStopError); + } + return await this.stopDescendantAgentTaskUnderLifecycleLock(ancestorWorkspaceId, taskId); + }); + } + + private async validateDirectDescendantAgentTaskScope( + ancestorWorkspaceId: string, + taskId: string + ): Promise> { + await using _lock = await this.mutex.acquire(); + const cfg = this.config.loadConfigOrDefault(); + const entry = findWorkspaceEntry(cfg, taskId); + if (!entry?.workspace.parentWorkspaceId) { + return Err("Task not found"); + } + const index = this.buildAgentTaskIndex(cfg); + if ( + !this.isDescendantAgentTaskUsingParentById(index.parentById, ancestorWorkspaceId, taskId) || + this.isWorkflowOwnedTaskUsingIndex(index, taskId) + ) { + return Err("Task is not a descendant of this workspace"); + } + return Ok(undefined); } private async stopDescendantAgentTaskUnderLifecycleLock( @@ -4682,12 +4755,20 @@ export class TaskService { } const index = this.buildAgentTaskIndex(cfg); if ( - !this.isDescendantAgentTaskUsingParentById(index.parentById, ancestorWorkspaceId, taskId) + !this.isDescendantAgentTaskUsingParentById(index.parentById, ancestorWorkspaceId, taskId) || + this.isWorkflowOwnedTaskUsingIndex(index, taskId) ) { return Err("Task is not a descendant of this workspace"); } - const taskIds = [taskId, ...this.listDescendantAgentTaskIdsFromIndex(index, taskId)]; + const taskIds = [ + taskId, + ...this.listDescendantAgentTaskIdsFromIndex(index, taskId).filter( + // Workflow-owned branches are stopped through WorkflowService so the durable run and + // its workers transition together; direct tree stopping must not mutate them behind it. + (descendantTaskId) => !this.isWorkflowOwnedTaskUsingIndex(index, descendantTaskId) + ), + ]; taskIds.sort( (left, right) => this.getTaskDepthFromParentById(index.parentById, right) - @@ -5139,7 +5220,11 @@ export class TaskService { */ async terminateAllDescendantAgentTasks( workspaceId: string, - options?: { workflowRunId?: string } + options?: { + workflowRunId?: string; + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; + deferWorkflowSweep?: boolean; + } ): Promise { assert( workspaceId.length > 0, @@ -5198,6 +5283,18 @@ export class TaskService { log.debug("terminateAllDescendantAgentTasks: stopStream threw", { taskId: id, error }); } + // Stop the worker stream before taking the final process snapshot: a background bash tool + // can register its process while the stream is still winding down. Cleanup remains + // best-effort so a disposal failure cannot block task status teardown. + try { + await options?.cleanupWorkspaceBackgroundProcesses?.(id); + } catch (error: unknown) { + log.warn("terminateAllDescendantAgentTasks: background cleanup failed", { + taskId: id, + error: getErrorMessage(error), + }); + } + let preservedCompletedDescendant = false; let transitionedToInterrupted = false; let parentWorkspaceId: string | undefined; @@ -5245,7 +5342,7 @@ export class TaskService { await this.emitWorkspaceMetadata(taskId); } - if (options?.workflowRunId != null) { + if (options?.workflowRunId != null && options.deferWorkflowSweep !== true) { // Run-scoped interrupts arrive after the owning run's terminal status write // (WorkflowService.interruptRun aborts the runner, persists "interrupted", THEN // terminates descendants), so the children just interrupted above can be archived @@ -8313,7 +8410,7 @@ export class TaskService { async interruptWorkspaceTurn( ownerWorkspaceId: string, handleId: string - ): Promise> { + ): Promise> { let workspaceId: string | undefined; let shouldClearQueuedPrompt = false; let shouldStopStream = false; @@ -8324,8 +8421,12 @@ export class TaskService { if (record == null) { return Err("Workspace turn not found or out of scope"); } - if (record.status === "completed" || record.status === "error") { - return Err(`Workspace turn is already ${record.status} and cannot be interrupted.`); + if ( + record.status === "completed" || + record.status === "error" || + record.status === "interrupted" + ) { + return Ok({ workspaceId: record.workspaceId, alreadyInactive: true }); } workspaceId = record.workspaceId; @@ -8357,7 +8458,10 @@ export class TaskService { return Ok({ workspaceId: record.workspaceId }); }); - if (!result.success) { + if ( + !result.success || + ("alreadyInactive" in result.data && result.data.alreadyInactive === true) + ) { return result; } @@ -8426,10 +8530,64 @@ export class TaskService { return Ok(didUnarchive); } + private async removeArchivedWorkflowOwnedDescendantsUnderLifecycleLock( + taskId: string + ): Promise> { + const config = this.config.loadConfigOrDefault(); + const index = this.buildAgentTaskIndex(config); + const candidateTaskIds = this.listDescendantAgentTaskIdsFromIndex(index, taskId).filter( + (descendantTaskId) => { + const descendant = index.byId.get(descendantTaskId); + return ( + descendant != null && + this.isWorkflowOwnedTaskUsingIndex(index, descendantTaskId) && + isWorkspaceArchived(descendant.archivedAt, descendant.unarchivedAt) + ); + } + ); + candidateTaskIds.sort( + (left, right) => + this.getTaskDepthFromParentById(index.parentById, right) - + this.getTaskDepthFromParentById(index.parentById, left) + ); + + for (const descendantTaskId of candidateTaskIds) { + const freshConfig = this.config.loadConfigOrDefault(); + const freshIndex = this.buildAgentTaskIndex(freshConfig); + const descendant = freshIndex.byId.get(descendantTaskId); + if ( + descendant == null || + !this.isWorkflowOwnedTaskUsingIndex(freshIndex, descendantTaskId) || + !isWorkspaceArchived(descendant.archivedAt, descendant.unarchivedAt) + ) { + continue; + } + if (this.isActiveAgentTaskEntry(descendant) || this.aiService.isStreaming(descendantTaskId)) { + return Err(`Archived workflow-owned descendant ${descendantTaskId} is still active.`); + } + if ((freshIndex.childrenByParent.get(descendantTaskId) ?? []).length > 0) { + continue; + } + + const tombstoneResult = await this.persistRemovedAgentTaskTombstones(descendantTaskId); + if (!tombstoneResult.success) { + return tombstoneResult; + } + const removeResult = await this.workspaceService.removeWhileTaskTreeLocked( + descendantTaskId, + true + ); + if (!removeResult.success) { + return Err(removeResult.error); + } + } + return Ok(undefined); + } + async removeInactiveDescendantAgentTask( ownerWorkspaceId: string, taskId: string - ): Promise> { + ): Promise> { assert(ownerWorkspaceId.length > 0, "removeInactiveDescendantAgentTask requires owner"); assert(taskId.length > 0, "removeInactiveDescendantAgentTask requires taskId"); @@ -8448,7 +8606,10 @@ export class TaskService { } const index = this.buildAgentTaskIndex(config); - if (!this.isDescendantAgentTaskUsingParentById(index.parentById, ownerWorkspaceId, taskId)) { + if ( + !this.isDescendantAgentTaskUsingParentById(index.parentById, ownerWorkspaceId, taskId) || + this.isWorkflowOwnedTaskUsingIndex(index, taskId) + ) { return Ok({ status: "invalid_scope", action: "remove", taskId }); } @@ -8458,8 +8619,31 @@ export class TaskService { workspaceId: taskId, ...(displayName != null ? { displayName } : {}), }; + if ( + this.isActiveAgentTaskEntry({ ...entry.workspace, projectPath: entry.projectPath }) || + this.aiService.isStreaming(taskId) + ) { + return Ok({ + status: "active", + action: "remove", + ...target, + activeTaskIds: [taskId], + note: "Stop the sub-agent before removing it.", + }); + } + const descendantTaskIds = this.listDescendantAgentTasks(taskId).map((task) => task.taskId); - if (descendantTaskIds.length > 0) { + const blockingDescendantTaskIds = descendantTaskIds.filter((descendantTaskId) => { + const descendant = index.byId.get(descendantTaskId); + return ( + descendant == null || + !this.isWorkflowOwnedTaskUsingIndex(index, descendantTaskId) || + !isWorkspaceArchived(descendant.archivedAt, descendant.unarchivedAt) || + this.isActiveAgentTaskEntry(descendant) || + this.aiService.isStreaming(descendantTaskId) + ); + }); + if (blockingDescendantTaskIds.length > 0) { return Ok({ status: "error", action: "remove", @@ -8469,16 +8653,28 @@ export class TaskService { }); } - if ( - this.isActiveAgentTaskEntry({ ...entry.workspace, projectPath: entry.projectPath }) || - this.aiService.isStreaming(taskId) - ) { + // Workflow-owned workers are hidden from public lifecycle tools, but their archived + // workspaces must not permanently block an otherwise-eligible parent removal. + const workflowCleanupResult = + await this.removeArchivedWorkflowOwnedDescendantsUnderLifecycleLock(taskId); + if (!workflowCleanupResult.success) { return Ok({ - status: "active", + status: "error", action: "remove", ...target, - activeTaskIds: [taskId], - note: "Stop the sub-agent before removing it.", + error: workflowCleanupResult.error, + }); + } + const remainingDescendantTaskIds = this.listDescendantAgentTasks(taskId).map( + (task) => task.taskId + ); + if (remainingDescendantTaskIds.length > 0) { + return Ok({ + status: "error", + action: "remove", + ...target, + descendantTaskIds: remainingDescendantTaskIds, + error: "Cannot remove a sub-agent while descendant sub-agents remain.", }); } @@ -8598,6 +8794,9 @@ export class TaskService { ); const workflowOwned = next.workflowOwned || entry.workflowTask != null; + const workflowOwner = workflowOwned + ? this.findWorkflowTaskOwnerInAncestry(index, next.taskId) + : null; const status: AgentTaskStatus = entry.taskStatus ?? "running"; if ( (!statusFilter || statusFilter.has(status)) && @@ -8613,6 +8812,14 @@ export class TaskService { createdAt: entry.createdAt, executionTaskId: entry.taskExecutionId, executionStatus: entry.taskExecutionStatus, + ...(workflowOwner != null + ? { + workflowRunId: workflowOwner.workflowTask.runId, + ...(workflowOwner.workspace.parentWorkspaceId != null + ? { workflowOwnerWorkspaceId: workflowOwner.workspace.parentWorkspaceId } + : {}), + } + : {}), modelString: entry.aiSettings?.model, thinkingLevel: entry.aiSettings?.thinkingLevel, depth: next.depth, diff --git a/src/node/services/tools/task.bash.test.ts b/src/node/services/tools/task.bash.test.ts index ca5a94a2ac6..46b79e4c9bc 100644 --- a/src/node/services/tools/task.bash.test.ts +++ b/src/node/services/tools/task.bash.test.ts @@ -306,8 +306,37 @@ describe("bash + task_* (background bash tasks)", () => { }); }); - it("task_terminate can terminate bash tasks", async () => { - using tempDir = new TestTempDir("test-task-terminate-bash"); + it("task_stop rejects bash tasks from workflow-owned descendants", async () => { + using tempDir = new TestTempDir("test-task-stop-bash-workflow-owned"); + + const getProcess = mock(() => ({ id: "workflow-proc", workspaceId: "workflow-task" })); + const terminate = mock(() => ({ success: true as const })); + const backgroundProcessManager = { + getProcess, + terminate, + } as unknown as BackgroundProcessManager; + const taskService = { + isDescendantAgentTask: mock(() => Promise.resolve(true)), + isWorkflowOwnedDescendantAgentTask: mock(() => Promise.resolve(true)), + } as unknown as TaskService; + const tool = createTaskStopTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "ws-1" }), + backgroundProcessManager, + taskService, + }); + + expect( + await Promise.resolve( + tool.execute!({ task_ids: ["bash:workflow-proc"] }, mockToolCallOptions) + ) + ).toEqual({ + results: [{ status: "invalid_scope", taskId: "bash:workflow-proc" }], + }); + expect(terminate).not.toHaveBeenCalled(); + }); + + it("task_stop can stop bash tasks", async () => { + using tempDir = new TestTempDir("test-task-stop-bash"); const getProcess = mock(() => ({ id: "proc-1", workspaceId: "ws-1" })); const terminate = mock(() => ({ success: true as const })); diff --git a/src/node/services/tools/task_await.test.ts b/src/node/services/tools/task_await.test.ts index 73ed99ab8fb..c52f1fd649b 100644 --- a/src/node/services/tools/task_await.test.ts +++ b/src/node/services/tools/task_await.test.ts @@ -933,7 +933,7 @@ describe("task_await tool", () => { expect(waitForAgentReport).toHaveBeenCalledTimes(1); }); - it("returns an error with descendant task suggestions for hallucinated IDs", async () => { + it("preserves not_found while suggesting active descendant task IDs", async () => { using tempDir = new TestTempDir("test-task-await-tool-hallucinated-descendant-suggestions"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "parent-workspace" }); @@ -954,27 +954,22 @@ describe("task_await tool", () => { const tool = createTaskAwaitTool({ ...baseConfig, taskService }); - const result = (await Promise.resolve( - tool.execute!({ task_ids: ["hallucinated"] }, mockToolCallOptions) - )) as { results: Array<{ status: string; taskId: string; error?: string }> }; - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - status: "error", - taskId: "hallucinated", + expect( + await Promise.resolve(tool.execute!({ task_ids: ["hallucinated"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "not_found", + taskId: "hallucinated", + activeTaskIds: ["real-child"], + }, + ], }); - const descendantSuggestionError = result.results[0]?.error; - expect(typeof descendantSuggestionError).toBe("string"); - if (typeof descendantSuggestionError !== "string") { - throw new Error("Expected hallucinated descendant result to include an error message"); - } - expect(descendantSuggestionError).toContain("same parallel tool-call batch"); - expect(descendantSuggestionError).toContain("real-child"); expect(getAgentTaskStatuses).toHaveBeenCalledTimes(1); expect(waitForAgentReport).toHaveBeenCalledTimes(0); }); - it("returns an error with bash task suggestions for out-of-scope IDs", async () => { + it("preserves invalid_scope while suggesting active bash task IDs", async () => { using tempDir = new TestTempDir("test-task-await-tool-hallucinated-bash-suggestions"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "parent-workspace" }); @@ -1010,22 +1005,17 @@ describe("task_await tool", () => { taskService, }); - const result = (await Promise.resolve( - tool.execute!({ task_ids: ["other-workspace"] }, mockToolCallOptions) - )) as { results: Array<{ status: string; taskId: string; error?: string }> }; - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - status: "error", - taskId: "other-workspace", + expect( + await Promise.resolve(tool.execute!({ task_ids: ["other-workspace"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "invalid_scope", + taskId: "other-workspace", + activeTaskIds: ["bash:proc-1"], + }, + ], }); - const bashSuggestionError = result.results[0]?.error; - expect(typeof bashSuggestionError).toBe("string"); - if (typeof bashSuggestionError !== "string") { - throw new Error("Expected out-of-scope bash suggestion result to include an error message"); - } - expect(bashSuggestionError).toContain("same parallel tool-call batch"); - expect(bashSuggestionError).toContain("bash:proc-1"); expect(getAgentTaskStatuses).toHaveBeenCalledTimes(1); expect(waitForAgentReport).toHaveBeenCalledTimes(0); }); @@ -1062,6 +1052,7 @@ describe("task_await tool", () => { reportMarkdown?: string; title?: string; note?: string; + activeTaskIds?: string[]; }>; }; @@ -1073,16 +1064,11 @@ describe("task_await tool", () => { title: undefined, note: COMPLETED_REPORT_REFETCH_NOTE, }); - expect(result.results[1]).toMatchObject({ - status: "error", + expect(result.results[1]).toEqual({ + status: "not_found", taskId: "hallucinated", + activeTaskIds: ["real-child"], }); - const mixedResultError = result.results[1]?.error; - expect(typeof mixedResultError).toBe("string"); - if (typeof mixedResultError !== "string") { - throw new Error("Expected mixed-result hallucinated task to include an error message"); - } - expect(mixedResultError).toContain("real-child"); expect(waitForAgentReport).toHaveBeenCalledTimes(1); expect(waitForAgentReport).toHaveBeenCalledWith("real-child", expect.any(Object)); expect(getAgentTaskStatuses).toHaveBeenCalledTimes(1); diff --git a/src/node/services/tools/task_await.ts b/src/node/services/tools/task_await.ts index 7cb92785033..5bd1a24fd89 100644 --- a/src/node/services/tools/task_await.ts +++ b/src/node/services/tools/task_await.ts @@ -93,17 +93,6 @@ function withElapsedMs(elapsedMs: number | undefined): { elapsed_ms?: number } { return elapsedMs == null ? {} : { elapsed_ms: elapsedMs }; } -function buildTaskAwaitSequencingError(taskId: string, suggestedTaskIds: string[]) { - return { - status: "error" as const, - taskId, - error: - "Do not call task_await in the same parallel tool-call batch as task or bash. " + - "Wait for the spawning tool result first, then call task_await in a later step. " + - `Use one of these returned task IDs instead: ${suggestedTaskIds.join(", ")}.`, - }; -} - function parseWorkflowRun(value: unknown): WorkflowRunRecord { return WorkflowRunRecordSchema.parse(value); } @@ -770,19 +759,15 @@ export const createTaskAwaitTool: ToolFactory = (config: ToolConfiguration) => { if (!descendantAgentTaskIdSet.has(taskId)) { const lookup = rejectedAgentTaskStatuses.get(taskId); - const activeTaskIds = - activeDescendantAgentTaskIds.length > 0 ? activeDescendantAgentTaskIds : undefined; - if (requestedIds) { - const suggestedTaskIds = dedupeStrings([ - ...activeDescendantAgentTaskIds, - ...(await listInScopeWorkspaceTurnTaskIds().catch(() => [])), - ...(await getSuggestionBashTaskIds()), - ...(await getSuggestionWorkflowRunIds()), - ]); - if (suggestedTaskIds.length > 0) { - return buildTaskAwaitSequencingError(taskId, suggestedTaskIds); - } - } + const suggestedTaskIds = requestedIds + ? dedupeStrings([ + ...activeDescendantAgentTaskIds, + ...(await listInScopeWorkspaceTurnTaskIds().catch(() => [])), + ...(await getSuggestionBashTaskIds()), + ...(await getSuggestionWorkflowRunIds()), + ]) + : activeDescendantAgentTaskIds; + const activeTaskIds = suggestedTaskIds.length > 0 ? suggestedTaskIds : undefined; if (!lookup?.exists) { return { status: "not_found" as const, taskId, activeTaskIds }; } diff --git a/src/node/services/tools/task_remove.ts b/src/node/services/tools/task_remove.ts index 0e53f77dbc8..c2de5ccf3a4 100644 --- a/src/node/services/tools/task_remove.ts +++ b/src/node/services/tools/task_remove.ts @@ -43,7 +43,6 @@ export const createTaskRemoveTool: ToolFactory = (config: ToolConfiguration) => case "removed": case "already_removed": case "active": - case "not_found": case "invalid_scope": results.push({ status: data.status, diff --git a/src/node/services/tools/task_stop.test.ts b/src/node/services/tools/task_stop.test.ts index eff272effe2..b622e4c6297 100644 --- a/src/node/services/tools/task_stop.test.ts +++ b/src/node/services/tools/task_stop.test.ts @@ -41,15 +41,23 @@ describe("task_stop tool", () => { using tempDir = new TestTempDir("test-task-terminate-invalid-scope"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const listRuns = mock(() => Promise.resolve([])); + const stopDescendantAgentTask = mock( + (): Promise> => + Promise.resolve(Err("Task is not a descendant of this workspace")) + ); const taskService = { listActiveDescendantAgentTaskIds: mock(() => ["child-task"]), - stopDescendantAgentTask: mock( - (): Promise> => - Promise.resolve(Err("Task is not a descendant of this workspace")) - ), + listDescendantAgentTasks: mock(() => []), + isDescendantAgentTask: mock(() => Promise.resolve(false)), + stopDescendantAgentTask, } as unknown as TaskService; - const tool = createTaskStopTool({ ...baseConfig, taskService }); + const tool = createTaskStopTool({ + ...baseConfig, + taskService, + workflowService: { listRuns }, + }); const result: unknown = await Promise.resolve( tool.execute!({ task_ids: ["other-task"] }, mockToolCallOptions) @@ -58,6 +66,8 @@ describe("task_stop tool", () => { expect(result).toEqual({ results: [{ status: "invalid_scope", taskId: "other-task" }], }); + expect(listRuns).not.toHaveBeenCalled(); + expect(stopDescendantAgentTask).toHaveBeenCalledTimes(1); }); it("reports aggregated cleanup failures as error, not invalid_scope", async () => { @@ -114,11 +124,39 @@ describe("task_stop tool", () => { }); }); + it("stops ordinary task trees when dynamic workflow services are unavailable", async () => { + using tempDir = new TestTempDir("test-task-stop-without-workflow-service"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const taskService = { + listDescendantAgentTasks: mock(() => [ + { taskId: "ordinary-grandchild", status: "running" as const }, + ]), + stopDescendantAgentTask: mock( + (): Promise> => + Promise.resolve(Ok({ stoppedTaskIds: ["ordinary-grandchild", "ordinary-child"] })) + ), + } as unknown as TaskService; + const tool = createTaskStopTool({ ...baseConfig, taskService }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["ordinary-child"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "stopped", + taskId: "ordinary-child", + stoppedTaskIds: ["ordinary-grandchild", "ordinary-child"], + }, + ], + }); + }); + it("returns an interrupted error promptly while completed task IDs still resolve", async () => { using tempDir = new TestTempDir("test-task-terminate-abort"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); const controller = new AbortController(); + const finished = Promise.withResolvers(); const taskService = { stopDescendantAgentTask: mock( ( @@ -128,6 +166,7 @@ describe("task_stop tool", () => { if (taskId === "stuck-task") { return new Promise(() => undefined); } + finished.resolve(); return Promise.resolve(Ok({ stoppedTaskIds: [taskId] })); } ), @@ -140,8 +179,11 @@ describe("task_stop tool", () => { { ...mockToolCallOptions, abortSignal: controller.signal } ) ); - await Promise.resolve(); - await Promise.resolve(); + await finished.promise; + // Let the completed branch propagate through the tool's abort race before aborting the stuck one. + for (let i = 0; i < 10; i += 1) { + await Promise.resolve(); + } controller.abort(); expect(await resultPromise).toEqual({ @@ -193,6 +235,25 @@ describe("task_stop tool", () => { }); }); + it("treats stopping a terminal workspace turn as idempotent", async () => { + using tempDir = new TestTempDir("test-task-stop-terminal-workspace-turn"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const interruptWorkspaceTurn = mock( + (): Promise> => + Promise.resolve(Ok({ workspaceId: "child-workspace", alreadyInactive: true })) + ); + const tool = createTaskStopTool({ + ...baseConfig, + taskService: { interruptWorkspaceTurn } as unknown as TaskService, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wst_turn"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "already_inactive", taskId: "wst_turn" }], + }); + }); + const buildWorkflowRun = (status: string) => ({ id: "wfr_run_1", workspaceId: "root-workspace", @@ -217,7 +278,10 @@ describe("task_stop tool", () => { const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); const getRun = mock(() => Promise.resolve(buildWorkflowRun("running"))); - const interruptRun = mock(() => Promise.resolve(buildWorkflowRun("interrupted"))); + const interruptRun = mock((input: { onRunInterrupted?: (runId: string) => void }) => { + input.onRunInterrupted?.("wfr_run_1"); + return Promise.resolve(buildWorkflowRun("interrupted")); + }); const taskService = { stopDescendantAgentTask: mock(() => { throw new Error("workflow IDs must not reach agent task termination"); @@ -241,6 +305,7 @@ describe("task_stop tool", () => { expect(interruptRun).toHaveBeenCalledWith({ workspaceId: "root-workspace", runId: "wfr_run_1", + onRunInterrupted: expect.any(Function), }); expect(result).toEqual({ results: [ @@ -253,6 +318,182 @@ describe("task_stop tool", () => { }); }); + it("interrupts workflow runs owned by an agent subtree before stopping user-owned tasks", async () => { + using tempDir = new TestTempDir("test-task-stop-agent-workflows-first"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const events: string[] = []; + const workflowRun = { ...buildWorkflowRun("running"), workspaceId: "child-task" }; + const workerWorkflowRun = { + ...buildWorkflowRun("running"), + id: "wfr_worker", + workspaceId: "workflow-worker", + }; + const taskService = { + listDescendantAgentTasks: mock( + (_taskId: string, options?: { excludeWorkflowTasks?: boolean }) => + options?.excludeWorkflowTasks === true + ? [] + : [ + { + taskId: "workflow-worker", + status: "running" as const, + workflowRunId: "wfr_run_1", + workflowOwnerWorkspaceId: "child-task", + }, + ] + ), + isDescendantAgentTask: mock(() => Promise.resolve(true)), + isWorkflowOwnedDescendantAgentTask: mock(() => Promise.resolve(false)), + stopDescendantAgentTask: mock( + async ( + _workspaceId: string, + _taskId: string, + options?: { beforeStop?: () => Promise } + ): Promise> => { + const beforeStopError = await options?.beforeStop?.(); + if (beforeStopError != null) { + return Err(beforeStopError); + } + events.push("task"); + return Ok({ stoppedTaskIds: ["child-task"] }); + } + ), + markWorkflowRunEnded: mock((workflowRunId: string) => { + events.push(`sweep:${workflowRunId}`); + return Promise.resolve(); + }), + } as unknown as TaskService; + const interruptRun = + (runId: string, run: ReturnType, event: string) => + (input: { + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + onRunInterrupted?: (interruptedRunId: string) => void; + }) => { + expect(input.deferTaskSweep).toBe(true); + expect(input.lockAlreadyHeld).toBe(true); + input.onRunInterrupted?.(runId); + events.push(event); + return Promise.resolve({ ...run, status: "interrupted" }); + }; + const childWorkflowService = { + listRuns: mock(() => Promise.resolve([workflowRun])), + getRun: mock(() => Promise.resolve(workflowRun)), + interruptRun: mock(interruptRun("wfr_run_1", workflowRun, "workflow:child")), + }; + const workerWorkflowService = { + listRuns: mock(() => Promise.resolve([workerWorkflowRun])), + getRun: mock(() => Promise.resolve(workerWorkflowRun)), + interruptRun: mock(interruptRun("wfr_worker", workerWorkflowRun, "workflow:workflow-worker")), + }; + const workflowServiceForWorkspace = mock((ownerWorkspaceId: string) => { + if (ownerWorkspaceId === "child-task") return childWorkflowService; + if (ownerWorkspaceId === "workflow-worker") return workerWorkflowService; + throw new Error(`Unexpected workflow owner ${ownerWorkspaceId}`); + }); + const tool = createTaskStopTool({ + ...baseConfig, + taskService, + workflowService: { + listRuns: mock(() => { + throw new Error("Parent workflow store must not be used for child-owned runs"); + }), + }, + workflowServiceForWorkspace, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["child-task"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "stopped", taskId: "child-task", stoppedTaskIds: ["child-task"] }], + }); + expect(workflowServiceForWorkspace.mock.calls.map((call) => call[0])).toEqual([ + "child-task", + "workflow-worker", + ]); + expect(events).toEqual([ + "workflow:child", + "workflow:workflow-worker", + "task", + "sweep:wfr_run_1", + "sweep:wfr_worker", + ]); + }); + + it("does not let a healthy workflow run mask an orphaned active workflow worker", async () => { + using tempDir = new TestTempDir("test-task-stop-orphaned-workflow-worker"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const healthyRun = { ...buildWorkflowRun("running"), id: "wfr_healthy", workspaceId: "child" }; + const stopDescendantAgentTask = mock( + async ( + _workspaceId: string, + _taskId: string, + options?: { beforeStop?: () => Promise } + ): Promise> => { + const error = await options?.beforeStop?.(); + return error == null ? Ok({ stoppedTaskIds: ["child"] }) : Err(error); + } + ); + const taskService = { + listDescendantAgentTasks: mock( + (_taskId: string, options?: { excludeWorkflowTasks?: boolean }) => + options?.excludeWorkflowTasks === true + ? [] + : [ + { + taskId: "healthy-worker", + status: "running" as const, + workflowRunId: "wfr_healthy", + workflowOwnerWorkspaceId: "child", + }, + { + taskId: "orphan-worker", + status: "running" as const, + workflowRunId: "wfr_missing", + workflowOwnerWorkspaceId: "child", + }, + ] + ), + stopDescendantAgentTask, + } as unknown as TaskService; + const ownerWorkflowService = { + listRuns: mock(() => Promise.resolve([healthyRun])), + getRun: mock((input: { runId: string }) => + Promise.resolve(input.runId === "wfr_healthy" ? healthyRun : null) + ), + interruptRun: mock((input: { onRunInterrupted?: (runId: string) => void }) => { + input.onRunInterrupted?.("wfr_healthy"); + return Promise.resolve({ ...healthyRun, status: "interrupted" }); + }), + }; + const emptyWorkflowService = { + listRuns: mock(() => Promise.resolve([])), + getRun: mock(() => Promise.resolve(null)), + interruptRun: mock(() => Promise.resolve(null)), + }; + const tool = createTaskStopTool({ + ...baseConfig, + taskService, + workflowServiceForWorkspace: (ownerWorkspaceId) => + ownerWorkspaceId === "child" ? ownerWorkflowService : emptyWorkflowService, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["child"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "error", + taskId: "child", + error: + "Owning workflow run wfr_missing for active descendant orphan-worker is missing or unreadable", + }, + ], + }); + expect(ownerWorkflowService.interruptRun).toHaveBeenCalledTimes(1); + expect(stopDescendantAgentTask).toHaveBeenCalledTimes(1); + }); + it("does not start termination when the signal is already aborted", async () => { using tempDir = new TestTempDir("test-task-terminate-preaborted"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); @@ -313,7 +554,13 @@ describe("task_stop tool", () => { using tempDir = new TestTempDir("test-task-terminate-workflow-idempotent"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); - const interruptRun = mock(() => Promise.reject(new Error("must not re-interrupt"))); + const interruptRun = mock( + (input: { retryTaskCleanup?: boolean; onRunInterrupted?: (runId: string) => void }) => { + expect(input.retryTaskCleanup).toBe(true); + input.onRunInterrupted?.("wfr_run_1"); + return Promise.resolve(buildWorkflowRun("interrupted")); + } + ); const tool = createTaskStopTool({ ...baseConfig, taskService: {} as unknown as TaskService, @@ -327,7 +574,7 @@ describe("task_stop tool", () => { tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions) ); - expect(interruptRun).not.toHaveBeenCalled(); + expect(interruptRun).toHaveBeenCalledTimes(1); expect(result).toEqual({ results: [ { @@ -338,8 +585,8 @@ describe("task_stop tool", () => { }); }); - it("rejects interrupting terminal workflow runs", async () => { - using tempDir = new TestTempDir("test-task-terminate-workflow-terminal"); + it("treats stopping terminal workflow runs as idempotent", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-terminal"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); const interruptRun = mock(() => Promise.reject(new Error("must not interrupt terminal runs"))); @@ -358,14 +605,101 @@ describe("task_stop tool", () => { expect(interruptRun).not.toHaveBeenCalled(); expect(result).toEqual({ + results: [{ status: "already_inactive", taskId: "wfr_run_1" }], + }); + }); + + it("treats a workflow that settles during interruption as already inactive", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-settlement-race"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const runs = [buildWorkflowRun("running"), buildWorkflowRun("completed")]; + const tool = createTaskStopTool({ + ...baseConfig, + taskService: {} as unknown as TaskService, + workflowService: { + getRun: mock(() => Promise.resolve(runs.shift() ?? buildWorkflowRun("completed"))), + interruptRun: mock(() => Promise.reject(new Error("invalid workflow transition"))), + }, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "already_inactive", taskId: "wfr_run_1" }], + }); + }); + + it("surfaces cleanup failure after this interruption persisted terminal state", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-post-persist-failure"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const getRun = mock(() => Promise.resolve(buildWorkflowRun("running"))); + const tool = createTaskStopTool({ + ...baseConfig, + taskService: {} as unknown as TaskService, + workflowService: { + getRun, + interruptRun: mock( + (input: { onRunInterrupted?: (runId: string) => void }): Promise => { + input.onRunInterrupted?.("wfr_run_1"); + return Promise.reject(new Error("workflow worker cleanup failed")); + } + ), + }, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ results: [ { status: "error", taskId: "wfr_run_1", - error: expect.stringContaining("already completed"), + error: "workflow worker cleanup failed", }, ], }); + expect(getRun).toHaveBeenCalledTimes(1); + }); + + it("retries workflow task cleanup after a post-persistence failure", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-cleanup-retry"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const runs = [buildWorkflowRun("running"), buildWorkflowRun("interrupted")]; + let interruptCalls = 0; + const tool = createTaskStopTool({ + ...baseConfig, + taskService: {} as unknown as TaskService, + workflowService: { + getRun: mock(() => Promise.resolve(runs.shift() ?? buildWorkflowRun("interrupted"))), + interruptRun: mock( + (input: { retryTaskCleanup?: boolean; onRunInterrupted?: (runId: string) => void }) => { + interruptCalls += 1; + input.onRunInterrupted?.("wfr_run_1"); + return interruptCalls === 1 + ? Promise.reject(new Error("workflow worker cleanup failed")) + : Promise.resolve(buildWorkflowRun("interrupted")); + } + ), + }, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "error", + taskId: "wfr_run_1", + error: "workflow worker cleanup failed", + }, + ], + }); + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "already_inactive", taskId: "wfr_run_1" }], + }); + expect(interruptCalls).toBe(2); }); it("reports workflow runs outside this workspace as not found", async () => { diff --git a/src/node/services/tools/task_stop.ts b/src/node/services/tools/task_stop.ts index c819c68a56d..1f25ad89946 100644 --- a/src/node/services/tools/task_stop.ts +++ b/src/node/services/tools/task_stop.ts @@ -1,12 +1,22 @@ import { tool } from "ai"; import { getErrorMessage } from "@/common/utils/errors"; -import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; +import type { + ToolConfiguration, + ToolFactory, + ToolWorkflowService, +} from "@/common/utils/tools/tools"; import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; +import { + isActiveWorkflowRunStatus, + isNestedWorkflowRun, + isTerminalWorkflowRunStatus, +} from "@/common/types/workflow"; import { TaskStopToolResultSchema, TOOL_DEFINITIONS } from "@/common/utils/tools/toolDefinitions"; import { TASK_TERMINATION_TOOL_TIMEOUT_MS } from "@/constants/terminationTimeouts"; import { log } from "@/node/services/log"; +import type { TaskService } from "@/node/services/taskService"; import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { isWorkspaceTurnTaskId } from "@/node/services/taskHandleStore"; import { fromBashTaskId, isWorkflowRunTaskId } from "./taskId"; @@ -26,11 +36,15 @@ const WORKFLOW_STOPPED_NOTE = * (whose contract says in-progress work is discarded). */ async function interruptWorkflowRun( - config: ToolConfiguration, + workflowService: ToolWorkflowService | null | undefined, workspaceId: string, - taskId: string + taskId: string, + options?: { + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + onRunInterrupted?: (runId: string) => void; + } ) { - const workflowService = config.workflowService; if (workflowService?.getRun == null || workflowService.interruptRun == null) { return { status: "error" as const, @@ -56,26 +70,195 @@ async function interruptWorkflowRun( } const run = parsedRun.data; - if (run.status === "interrupted") { - // Idempotent: re-interrupting an interrupted run is a no-op success. + if (run.status === "completed" || run.status === "failed") { return { status: "already_inactive" as const, taskId }; } - if (run.status === "completed" || run.status === "failed") { - return { - status: "error" as const, - taskId, - error: `Workflow run is already ${run.status} and cannot be interrupted.`, - }; + + if (run.status === "interrupted") { + try { + await workflowService.interruptRun({ + workspaceId, + runId: taskId, + retryTaskCleanup: true, + ...(options?.deferTaskSweep === true ? { deferTaskSweep: true } : {}), + ...(options?.lockAlreadyHeld === true ? { lockAlreadyHeld: true } : {}), + onRunInterrupted: options?.onRunInterrupted, + }); + return { status: "already_inactive" as const, taskId }; + } catch (error: unknown) { + return { status: "error" as const, taskId, error: getErrorMessage(error) }; + } } + let persistedByThisCall = false; try { - await workflowService.interruptRun({ workspaceId, runId: taskId }); + await workflowService.interruptRun({ + workspaceId, + runId: taskId, + ...(options?.deferTaskSweep === true ? { deferTaskSweep: true } : {}), + ...(options?.lockAlreadyHeld === true ? { lockAlreadyHeld: true } : {}), + onRunInterrupted: (runId) => { + persistedByThisCall ||= runId === taskId; + options?.onRunInterrupted?.(runId); + }, + }); } catch (error: unknown) { + // A terminal re-read means idempotent success only when another actor won the transition race. + // If this invocation persisted interrupted and cleanup then failed, surface that failure so the + // caller does not report success while workflow-owned workers may still be active. + if (!persistedByThisCall) { + const latestRawRun = await workflowService + .getRun({ workspaceId, runId: taskId }) + .catch(() => null); + const latestRun = WorkflowRunRecordSchema.safeParse(latestRawRun); + if (latestRun.success && isTerminalWorkflowRunStatus(latestRun.data.status)) { + return { status: "already_inactive" as const, taskId }; + } + } return { status: "error" as const, taskId, error: getErrorMessage(error) }; } return { status: "stopped" as const, taskId, note: WORKFLOW_STOPPED_NOTE }; } +async function interruptWorkflowRunsOwnedByAgentTaskTree( + config: ToolConfiguration, + taskService: TaskService, + taskId: string, + deferredWorkflowRunIds: string[] +): Promise { + const listDescendants = taskService.listDescendantAgentTasks?.bind(taskService); + if (listDescendants == null) { + return null; + } + + const descendants = listDescendants(taskId); + const userOwnedTaskIds = new Set( + listDescendants(taskId, { excludeWorkflowTasks: true }).map((task) => task.taskId) + ); + const activeWorkflowOwnedDescendants = descendants.filter( + (task) => + !userOwnedTaskIds.has(task.taskId) && + (task.status === "queued" || + task.status === "starting" || + task.status === "running" || + task.status === "awaiting_report") + ); + const workflowServices = new Map(); + const resolveWorkflowService = (ownerWorkspaceId: string): ToolWorkflowService | null => { + if (workflowServices.has(ownerWorkspaceId)) { + return workflowServices.get(ownerWorkspaceId) ?? null; + } + const service = + config.workflowServiceForWorkspace?.(ownerWorkspaceId) ?? + (ownerWorkspaceId === config.workspaceId ? config.workflowService : null) ?? + null; + workflowServices.set(ownerWorkspaceId, service); + return service; + }; + const interruptedRunKeys = new Set(); + const interruptOwnedRun = async ( + ownerWorkspaceId: string, + runId: string, + workflowService: ToolWorkflowService + ): Promise => { + const runKey = `${ownerWorkspaceId}\u0000${runId}`; + if (interruptedRunKeys.has(runKey)) { + return null; + } + const outcome = await interruptWorkflowRun(workflowService, ownerWorkspaceId, runId, { + deferTaskSweep: true, + lockAlreadyHeld: true, + onRunInterrupted: (interruptedRunId) => deferredWorkflowRunIds.push(interruptedRunId), + }); + if (outcome.status === "error") { + return outcome.error; + } + if (outcome.status === "not_found") { + return `Workflow run ${runId} disappeared before it could be stopped`; + } + interruptedRunKeys.add(runKey); + if (outcome.status === "stopped") { + deferredWorkflowRunIds.push(runId); + } + return null; + }; + + // A descendant workspace may own an independent background workflow even when that workspace is + // itself workflow-owned. Scan every resolvable descendant session, but do not require workflow + // services for ordinary task trees when dynamic workflows are unavailable. + for (const ownerWorkspaceId of [taskId, ...descendants.map((task) => task.taskId)]) { + const workflowService = resolveWorkflowService(ownerWorkspaceId); + if (workflowService?.listRuns == null || workflowService.interruptRun == null) { + continue; + } + const rawRuns = await workflowService.listRuns({ workspaceId: ownerWorkspaceId }); + for (const rawRun of rawRuns) { + const parsedRun = WorkflowRunRecordSchema.safeParse(rawRun); + if ( + !parsedRun.success || + (!isActiveWorkflowRunStatus(parsedRun.data.status) && + parsedRun.data.status !== "interrupted") || + isNestedWorkflowRun(parsedRun.data) + ) { + continue; + } + const error = await interruptOwnedRun(ownerWorkspaceId, parsedRun.data.id, workflowService); + if (error != null) return error; + } + } + + // Correlate each still-active workflow worker to its exact owning run. A healthy run elsewhere in + // the tree must not mask a missing, unreadable, or already-terminal owner for this worker. + for (const worker of activeWorkflowOwnedDescendants) { + const ownerWorkspaceId = worker.workflowOwnerWorkspaceId; + const runId = worker.workflowRunId; + if (ownerWorkspaceId == null || runId == null) { + return `Active workflow-owned descendant ${worker.taskId} is missing workflow ownership metadata`; + } + const runKey = `${ownerWorkspaceId}\u0000${runId}`; + if (interruptedRunKeys.has(runKey)) { + continue; + } + const workflowService = resolveWorkflowService(ownerWorkspaceId); + if (workflowService?.getRun == null || workflowService.interruptRun == null) { + return `Workflow service not available for active workflow-owned descendant ${worker.taskId}`; + } + const rawRun = await workflowService.getRun({ workspaceId: ownerWorkspaceId, runId }); + const parsedRun = WorkflowRunRecordSchema.safeParse(rawRun); + if (!parsedRun.success) { + return `Owning workflow run ${runId} for active descendant ${worker.taskId} is missing or unreadable`; + } + if ( + !isActiveWorkflowRunStatus(parsedRun.data.status) && + parsedRun.data.status !== "interrupted" + ) { + return `Owning workflow run ${runId} for active descendant ${worker.taskId} is already ${parsedRun.data.status}`; + } + const error = await interruptOwnedRun(ownerWorkspaceId, runId, workflowService); + if (error != null) return error; + } + + return null; +} + +async function sweepDeferredWorkflowRuns( + taskService: TaskService, + taskId: string, + workflowRunIds: string[] +): Promise { + for (const workflowRunId of dedupeStrings(workflowRunIds)) { + try { + await taskService.markWorkflowRunEnded(workflowRunId); + } catch (error: unknown) { + log.warn("task_stop deferred workflow sweep failed", { + taskId, + workflowRunId, + error: getErrorMessage(error), + }); + } + } +} + export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { return tool({ description: TOOL_DEFINITIONS.task_stop.description, @@ -99,7 +282,7 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { const terminationPromise = (async () => { try { if (isWorkflowRunTaskId(taskId)) { - return await interruptWorkflowRun(config, workspaceId, taskId); + return await interruptWorkflowRun(config.workflowService, workspaceId, taskId); } if (isWorkspaceTurnTaskId(taskId)) { @@ -114,6 +297,9 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { } return { status: "error" as const, taskId, error: msg }; } + if (interruptResult.data.alreadyInactive === true) { + return { status: "already_inactive" as const, taskId }; + } return { status: "stopped" as const, taskId, @@ -143,7 +329,14 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { const inScope = proc.workspaceId === workspaceId || (await taskService.isDescendantAgentTask(workspaceId, proc.workspaceId)); - if (!inScope) { + const workflowOwned = + proc.workspaceId !== workspaceId && + ((await taskService.isWorkflowOwnedDescendantAgentTask?.( + workspaceId, + proc.workspaceId + )) ?? + false); + if (!inScope || workflowOwned) { return { status: "invalid_scope" as const, taskId }; } @@ -160,7 +353,24 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { }; } - const stopResult = await taskService.stopDescendantAgentTask(workspaceId, taskId); + const deferredWorkflowRunIds: string[] = []; + const stopResult = await taskService.stopDescendantAgentTask(workspaceId, taskId, { + // Run workflow interruption while TaskService holds the task-tree lifecycle lock. + // Workflow worker creation uses the same lock, so no new workflow-owned branch can + // appear between discovery and the direct user-owned subtree stop. Archive sweeps + // are deferred because WorkspaceService.archive reacquires this non-reentrant lock. + beforeStop: + taskService.listDescendantAgentTasks != null + ? async () => + await interruptWorkflowRunsOwnedByAgentTaskTree( + config, + taskService, + taskId, + deferredWorkflowRunIds + ) + : undefined, + }); + await sweepDeferredWorkflowRuns(taskService, taskId, deferredWorkflowRunIds); if (!stopResult.success) { const msg = stopResult.error; // Exact-match the canonical scope errors: aggregated cleanup failures @@ -195,7 +405,7 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { } void terminationPromise.catch((error: unknown) => { - log.debug("task_terminate cleanup failed after tool returned", { taskId, error }); + log.debug("task_stop cleanup failed after tool returned", { taskId, error }); }); return { status: "error" as const, diff --git a/src/node/services/workflows/WorkflowRunner.ts b/src/node/services/workflows/WorkflowRunner.ts index 291c3efbc87..3771e1debcd 100644 --- a/src/node/services/workflows/WorkflowRunner.ts +++ b/src/node/services/workflows/WorkflowRunner.ts @@ -197,7 +197,7 @@ export interface WorkflowTaskAdapter { spec: WorkflowApplyPatchSpec, options?: { abortSignal?: AbortSignal } ): Promise; - interruptRun?(): Promise; + interruptRun?(options?: { deferTaskSweep?: boolean }): Promise; /** * Called when the run reaches a terminal state. Not called when the run is * backgrounded (the background continuation re-enters run()) or when the @@ -208,6 +208,9 @@ export interface WorkflowTaskAdapter { export interface WorkflowRunnerRunOptions { onLeaseAcquired?: () => void; + shouldDeferRunEnded?: () => boolean; + onRunEndedDeferred?: () => void; + onRunningStatusPersisted?: () => void; abortSignal?: AbortSignal; backgroundOnMessageQueued?: boolean; allowResumeFromInterrupted?: boolean; @@ -401,7 +404,11 @@ export class WorkflowRunner { assert(runId.length > 0, "WorkflowRunner.run: runId is required"); try { const result = await this.runWithLease(runId, options); - await this.taskAdapter.onRunEnded?.(); + if (options?.shouldDeferRunEnded?.() === true) { + options.onRunEndedDeferred?.(); + } else { + await this.taskAdapter.onRunEnded?.(); + } return result; } catch (error) { // Backgrounding is not terminal (the background continuation re-enters @@ -411,7 +418,11 @@ export class WorkflowRunner { error instanceof WorkflowRunBackgroundedError || (error instanceof Error && error.message === `Workflow run is already active: ${runId}`); if (!keepsHold) { - await this.taskAdapter.onRunEnded?.(); + if (options?.shouldDeferRunEnded?.() === true) { + options.onRunEndedDeferred?.(); + } else { + await this.taskAdapter.onRunEnded?.(); + } } throw error; } @@ -525,6 +536,8 @@ export class WorkflowRunner { } ); + options?.onRunningStatusPersisted?.(); + let runtime: IJSRuntime | undefined; try { runtime = await this.runtimeFactory.create(); diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index a573b42a7c8..0a328c0b4ee 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -2,7 +2,7 @@ import * as crypto from "node:crypto"; import * as path from "node:path"; -import { describe, expect, test } from "bun:test"; +import { describe, expect, mock, test } from "bun:test"; import assert from "@/common/utils/assert"; import { WORKFLOW_CHECKPOINT_RETRY_ERROR_MESSAGE } from "@/common/utils/workflowRetryEligibility"; import { ForegroundWaitBackgroundedError } from "@/node/services/taskService"; @@ -28,6 +28,75 @@ function createScript( } describe("WorkflowService", () => { + test("does not create a durable run when the workspace lifecycle lock rejects startup", async () => { + using tmp = new DisposableTempDir("workflow-service-run-creation-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + const withRunStartLock = mock(async () => { + throw new Error("Cannot start workflow work after task_stop"); + }); + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + withRunStartLock, + generateRunId: () => "wfr_rejected_start", + runnerId: "runner-a", + }); + + await expect( + service.startWorkflow({ + script: createScript("export default function workflow() { return 'unused'; }"), + workspaceId: "workspace-1", + projectTrusted: true, + args: {}, + }) + ).rejects.toThrow("Cannot start workflow work after task_stop"); + expect(withRunStartLock).toHaveBeenCalledTimes(1); + expect(await runStore.listRunStatusSnapshots()).toEqual([]); + }); + + test("does not resume a durable run when the workspace lifecycle lock rejects startup", async () => { + using tmp = new DisposableTempDir("workflow-service-resume-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_rejected_resume", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "project", executable: true }, + source: "export default function workflow() { return {}; }\n", + args: {}, + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_rejected_resume", "interrupted", "2026-05-29T00:00:01.000Z"); + const withRunStartLock = mock(async () => { + throw new Error("Cannot start workflow work after task_stop"); + }); + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + withRunStartLock, + runnerId: "runner-a", + }); + + await expect( + service.resumeRunInBackground({ + workspaceId: "workspace-1", + runId: "wfr_rejected_resume", + projectTrusted: true, + }) + ).rejects.toThrow("Cannot start workflow work after task_stop"); + expect(withRunStartLock).toHaveBeenCalledTimes(1); + expect((await runStore.getRun("wfr_rejected_resume")).status).toBe("interrupted"); + }); + test("starts an explicit script workflow and persists the resolved source snapshot", async () => { using tmp = new DisposableTempDir("workflow-service-script-path"); const source = `export default function workflow({ args }) { @@ -294,6 +363,57 @@ export default function workflow() { return { reportMarkdown: "done" }; } }); }); + test("holds the lifecycle lock until a background start is durably running", async () => { + using tmp = new DisposableTempDir("workflow-service-background-start-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + let lockHeld = false; + const events: string[] = []; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + withRunStartLock: async (_workspaceId, operation) => { + lockHeld = true; + events.push("lock:start"); + try { + return await operation(); + } finally { + events.push("lock:end"); + lockHeld = false; + } + }, + generateRunId: () => "wfr_background_lock", + runnerId: "runner-a", + }); + + await service.startWorkflowInBackground({ + script: createScript( + `export default function workflow() { return { reportMarkdown: "done" }; }\n` + ), + workspaceId: "workspace-1", + projectTrusted: true, + args: {}, + onRunCreated: ({ status }) => { + expect(lockHeld).toBe(true); + expect(status).toBe("pending"); + events.push("created:pending"); + }, + onBackgroundRunCreated: ({ status, run }) => { + expect(lockHeld).toBe(true); + expect(status).toBe("running"); + expect(run.status).toBe("running"); + events.push("created:running"); + }, + }); + + expect(lockHeld).toBe(false); + expect(events).toEqual(["lock:start", "created:pending", "created:running", "lock:end"]); + }); + test("foreground workflows that self-background persist notify_on_terminal policy", async () => { using tmp = new DisposableTempDir("workflow-service-self-background-notify"); const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); @@ -409,6 +529,109 @@ export default function workflow() { return { reportMarkdown: "done" }; } }); }); + test("handles foreground resume aborts while the lifecycle lock is still held", async () => { + using tmp = new DisposableTempDir("workflow-service-resume-abort-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_resume_abort_lock", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "project", executable: true }, + source: "export default function workflow() { return {}; }\n", + args: {}, + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_resume_abort_lock", "interrupted", "2026-05-29T00:00:01.000Z"); + const abortController = new AbortController(); + const events: string[] = []; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapterFactory: (runId) => ({ + async runAgent() { + throw new Error("Runner must not start after the resume is aborted"); + }, + interruptRun() { + events.push(`cleanup:${runId}`); + return Promise.resolve(); + }, + onRunEnded() { + events.push(`sweep:${runId}`); + return Promise.resolve(); + }, + }), + withRunStartLock: async (_workspaceId, operation) => { + events.push("lock:start"); + const result = await operation(); + events.push("lock:end"); + return result; + }, + onRunStatusChanged: (event) => { + if (event.runId === "wfr_resume_abort_lock" && event.status === "running") { + abortController.abort(); + } + }, + runnerId: "runner-a", + }); + + await expect( + service.resumeRun({ + workspaceId: "workspace-1", + runId: "wfr_resume_abort_lock", + projectTrusted: true, + abortSignal: abortController.signal, + }) + ).rejects.toThrow("Workflow run interrupted"); + + expect(events).toEqual([ + "lock:start", + "cleanup:wfr_resume_abort_lock", + "lock:end", + "sweep:wfr_resume_abort_lock", + ]); + expect((await runStore.getRun("wfr_resume_abort_lock")).status).toBe("interrupted"); + }); + + test("ignores a terminal race during abort cleanup retry", async () => { + using tmp = new DisposableTempDir("workflow-service-abort-cleanup-terminal-race"); + const service = new WorkflowService({ + runStore: new WorkflowRunStore({ sessionDir: tmp.path }), + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No runner expected"); + }, + }, + runnerId: "runner-a", + }); + const interruptRun = mock(() => Promise.reject(new Error("workflow cleanup transition raced"))); + const getRun = mock() + .mockResolvedValueOnce({ status: "interrupted" }) + .mockResolvedValueOnce({ status: "completed" }); + service.interruptRun = interruptRun; + service.getRun = getRun; + const abortController = new AbortController(); + const internal = service as unknown as { + interruptRunOnAbort: ( + workspaceId: string, + runId: string, + abortSignal: AbortSignal, + runnerAbortController: AbortController | undefined + ) => { remove: () => void; wait: () => Promise }; + }; + const abortInterrupt = internal.interruptRunOnAbort( + "workspace-1", + "wfr_abort_race", + abortController.signal, + undefined + ); + + abortController.abort(); + await abortInterrupt.wait(); + + expect(interruptRun).toHaveBeenCalledTimes(2); + expect(getRun).toHaveBeenCalledTimes(2); + }); + test("does not continue canceled foreground workflows in the background", async () => { using tmp = new DisposableTempDir("workflow-service-canceled-background"); const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); @@ -546,6 +769,54 @@ export default function workflow({ args }) { ).toBe(true); }); + test("serializes workflow interruption and defers task sweeps until lock release", async () => { + using tmp = new DisposableTempDir("workflow-service-interrupt-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_interrupt_lock", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "project", executable: true }, + source: "export default function workflow() { return {}; }\n", + args: {}, + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_interrupt_lock", "running", "2026-05-29T00:00:01.000Z"); + const events: string[] = []; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapterFactory: (runId) => ({ + async runAgent() { + throw new Error("No agent steps expected"); + }, + interruptRun() { + events.push(`cleanup:${runId}`); + return Promise.resolve(); + }, + onRunEnded() { + events.push(`sweep:${runId}`); + return Promise.resolve(); + }, + }), + withRunStartLock: async (_workspaceId, operation) => { + events.push("lock:start"); + const result = await operation(); + events.push("lock:end"); + return result; + }, + runnerId: "runner-a", + }); + + await service.interruptRun({ workspaceId: "workspace-1", runId: "wfr_interrupt_lock" }); + + expect(events).toEqual([ + "lock:start", + "cleanup:wfr_interrupt_lock", + "lock:end", + "sweep:wfr_interrupt_lock", + ]); + }); + test("interrupts active child workflow runs with the parent", async () => { using tmp = new DisposableTempDir("workflow-service-interrupt-nested-run"); const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); @@ -599,7 +870,13 @@ export default function workflow({ args }) { }, }); - await service.interruptRun({ workspaceId: "workspace-1", runId: "wfr_parent_interrupt" }); + const deferredSweepRunIds: string[] = []; + await service.interruptRun({ + workspaceId: "workspace-1", + runId: "wfr_parent_interrupt", + deferTaskSweep: true, + onRunInterrupted: (runId) => deferredSweepRunIds.push(runId), + }); await expect(runStore.getRun("wfr_parent_interrupt")).resolves.toMatchObject({ status: "interrupted", @@ -607,7 +884,22 @@ export default function workflow({ args }) { await expect(runStore.getRun("wfr_child_interrupt")).resolves.toMatchObject({ status: "interrupted", }); - expect(interruptedRunIds).toEqual(["wfr_parent_interrupt", "wfr_child_interrupt"]); + expect(deferredSweepRunIds).toEqual(["wfr_parent_interrupt", "wfr_child_interrupt"]); + const retriedSweepRunIds: string[] = []; + await service.interruptRun({ + workspaceId: "workspace-1", + runId: "wfr_parent_interrupt", + retryTaskCleanup: true, + deferTaskSweep: true, + onRunInterrupted: (runId) => retriedSweepRunIds.push(runId), + }); + expect(retriedSweepRunIds).toEqual(["wfr_parent_interrupt", "wfr_child_interrupt"]); + expect(interruptedRunIds).toEqual([ + "wfr_parent_interrupt", + "wfr_child_interrupt", + "wfr_parent_interrupt", + "wfr_child_interrupt", + ]); }); test("listRuns only loads root runs for the requested workspace", async () => { diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index adf3c26c5de..3baee9a3c67 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -55,6 +55,8 @@ export interface WorkflowServiceOptions { generateRunId?: () => string; // Delayed crash-recovery retries must use current trust, not the value captured when scheduled. getCurrentProjectTrusted?: () => boolean | Promise; + /** Serialize durable run creation with workspace/task lifecycle transitions. */ + withRunStartLock?: (workspaceId: string, operation: () => Promise) => Promise; /** Stable prefix; WorkflowService appends run identity and a nonce for each lease owner. */ runnerId: string; clock?: WorkflowRunnerClock; @@ -129,6 +131,10 @@ export class WorkflowService { private readonly notifyInterruptedBackgroundRunTerminal: boolean; private readonly generateRunId: () => string; private readonly getCurrentProjectTrusted?: () => boolean | Promise; + private readonly withRunStartLock?: ( + workspaceId: string, + operation: () => Promise + ) => Promise; private readonly runnerId: string; private readonly clock?: WorkflowRunnerClock; @@ -151,6 +157,7 @@ export class WorkflowService { options.notifyInterruptedBackgroundRunTerminal === true; this.generateRunId = options.generateRunId ?? generateWorkflowRunId; this.getCurrentProjectTrusted = options.getCurrentProjectTrusted; + this.withRunStartLock = options.withRunStartLock; this.runnerId = options.runnerId; this.clock = options.clock; } @@ -242,12 +249,73 @@ export class WorkflowService { } } - async interruptRun(input: { workspaceId: string; runId: string }): Promise { - return await this.interruptRunTree(input, new Set(), false); + async interruptRun(input: { + workspaceId: string; + runId: string; + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }): Promise { + const interruptedRunIds: string[] = []; + const interrupt = async () => + await this.interruptRunTree( + { + workspaceId: input.workspaceId, + runId: input.runId, + retryTaskCleanup: input.retryTaskCleanup, + onRunInterrupted: (runId) => { + interruptedRunIds.push(runId); + input.onRunInterrupted?.(runId); + }, + }, + new Set(), + false + ); + + let result: WorkflowRunRecord | undefined; + let operationError: unknown; + try { + result = + input.lockAlreadyHeld === true + ? await interrupt() + : await this.withWorkflowRunStartLock(input.workspaceId, interrupt); + } catch (error: unknown) { + operationError = error; + } + + if (input.deferTaskSweep !== true) { + await this.sweepInterruptedRunTasks(interruptedRunIds); + } + if (operationError != null) { + throw operationError instanceof Error + ? operationError + : new Error(getErrorMessage(operationError)); + } + assert(result != null, "WorkflowService.interruptRun: result is required"); + return result; + } + + private async sweepInterruptedRunTasks(runIds: readonly string[]): Promise { + for (const runId of new Set(runIds)) { + try { + await (this.taskAdapterFactory?.(runId) ?? this.requireTaskAdapter()).onRunEnded?.(); + } catch (error: unknown) { + log.warn("WorkflowService: interrupted workflow task sweep failed", { + runId, + error: getErrorMessage(error), + }); + } + } } private async interruptRunTree( - input: { workspaceId: string; runId: string }, + input: { + workspaceId: string; + runId: string; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }, visitedRunIds: Set, skipTerminalRun: boolean ): Promise { @@ -256,6 +324,14 @@ export class WorkflowService { return run; } visitedRunIds.add(input.runId); + if (run.status === "interrupted" && input.retryTaskCleanup === true) { + input.onRunInterrupted?.(input.runId); + await (this.taskAdapterFactory?.(input.runId) ?? this.requireTaskAdapter()).interruptRun?.({ + deferTaskSweep: true, + }); + await this.interruptChildWorkflowRuns(input, visitedRunIds); + return run; + } if (skipTerminalRun && isTerminalWorkflowRunStatus(run.status)) { return run; } @@ -280,8 +356,11 @@ export class WorkflowService { this.clock?.nowIso() ?? new Date().toISOString() ); settleStatusWrite(); + input.onRunInterrupted?.(input.runId); await this.notifyRunStatusChanged(interrupted); - await (this.taskAdapterFactory?.(input.runId) ?? this.requireTaskAdapter()).interruptRun?.(); + await (this.taskAdapterFactory?.(input.runId) ?? this.requireTaskAdapter()).interruptRun?.({ + deferTaskSweep: true, + }); await this.interruptChildWorkflowRuns(input, visitedRunIds); return interrupted; } finally { @@ -293,43 +372,65 @@ export class WorkflowService { } private async interruptChildWorkflowRuns( - input: { workspaceId: string; runId: string }, + input: { + workspaceId: string; + runId: string; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }, visitedRunIds: Set ): Promise { const childRuns = (await this.runStore.listRunStatusSnapshots()).filter( (snapshot) => snapshot.workspaceId === input.workspaceId && snapshot.parentWorkflow?.runId === input.runId && - !isTerminalWorkflowRunStatus(snapshot.status) + (!isTerminalWorkflowRunStatus(snapshot.status) || + (input.retryTaskCleanup === true && snapshot.status === "interrupted")) ); for (const childRun of childRuns) { // Child workflow runs from older workflow scripts are still persisted separately; // interrupting the parent must also stop their run-scoped agents before returning. await this.interruptRunTree( - { workspaceId: input.workspaceId, runId: childRun.id }, + { + workspaceId: input.workspaceId, + runId: childRun.id, + retryTaskCleanup: input.retryTaskCleanup, + onRunInterrupted: input.onRunInterrupted, + }, visitedRunIds, true ); } } + private async withWorkflowRunStartLock( + workspaceId: string, + operation: () => Promise + ): Promise { + return this.withRunStartLock != null + ? await this.withRunStartLock(workspaceId, operation) + : await operation(); + } + async retryRunFromCheckpointInBackground(input: { workspaceId: string; runId: string; projectTrusted: boolean; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanRetryFromCheckpoint(run); - // A checkpoint retry dispatched in the background is non-blocking just like background resume: - // persist notify_on_terminal before starting the background runner. - await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); - await this.runInBackground(input.runId, "Background workflow checkpoint retry failed:", { - allowRetryFromFailedCheckpoint: true, - projectTrusted: input.projectTrusted, + return await this.withWorkflowRunStartLock(input.workspaceId, async () => { + const run = await this.requireRunForWorkspace(input); + assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); + assertWorkflowRunCanRetryFromCheckpoint(run); + // Hold the workspace lifecycle lock until the background runner acquires its lease. A + // concurrent task_stop then either wins before dispatch or sees this run as active. + await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); + await this.runInBackground(input.runId, "Background workflow checkpoint retry failed:", { + allowRetryFromFailedCheckpoint: true, + projectTrusted: input.projectTrusted, + }); + await this.notifyRunStatusChanged(run, "running"); + return { runId: input.runId, status: "running", result: null }; }); - await this.notifyRunStatusChanged(run, "running"); - return { runId: input.runId, status: "running", result: null }; } async resumeRunInBackground(input: { @@ -337,18 +438,76 @@ export class WorkflowService { runId: string; projectTrusted: boolean; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanTransition(run.status, "running"); - // A run resumed in the background becomes non-blocking; persist so future stream-ends do not - // re-force a task_await even if the run was originally started in the foreground. - await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); - await this.runInBackground(input.runId, "Background workflow resume failed:", { - allowResumeFromInterrupted: run.status === "interrupted", - projectTrusted: input.projectTrusted, + return await this.withWorkflowRunStartLock(input.workspaceId, async () => { + const run = await this.requireRunForWorkspace(input); + assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); + assertWorkflowRunCanTransition(run.status, "running"); + // A run resumed in the background becomes non-blocking; persist so future stream-ends do not + // re-force a task_await even if the run was originally started in the foreground. + await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); + await this.runInBackground(input.runId, "Background workflow resume failed:", { + allowResumeFromInterrupted: run.status === "interrupted", + projectTrusted: input.projectTrusted, + }); + await this.notifyRunStatusChanged(run, "running"); + return { runId: input.runId, status: "running", result: null }; + }); + } + + private async dispatchForegroundRunWithStartLock(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + abortSignal?: AbortSignal; + validateRun: (run: WorkflowRunRecord) => void; + runnerOptions: ( + run: WorkflowRunRecord + ) => Pick< + WorkflowRunnerRunOptions, + "allowResumeFromInterrupted" | "allowRetryFromFailedCheckpoint" + >; + backgroundedFailureMessage: string; + }): Promise { + const startLockState = { held: true }; + const deferredInterruptRunIds: string[] = []; + const dispatch = await this.withWorkflowRunStartLock(input.workspaceId, async () => { + try { + const run = await this.requireRunForWorkspace(input); + assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); + input.validateRun(run); + + const started = Promise.withResolvers(); + const resultPromise = this.runForegroundWithAbortInterrupt({ + workspaceId: input.workspaceId, + run, + projectTrusted: input.projectTrusted, + abortSignal: input.abortSignal, + runnerOptions: input.runnerOptions(run), + backgroundedFailureMessage: input.backgroundedFailureMessage, + onRunningStatusPersisted: started.resolve, + startLockState: { + isHeld: () => startLockState.held, + onRunInterrupted: (runId) => deferredInterruptRunIds.push(runId), + }, + }); + // Do not hold the task-tree lock for the full foreground workflow. Release once the runner + // persists running status, or once an early failure settles the dispatch. + await Promise.race([ + started.promise, + resultPromise.then( + () => undefined, + () => undefined + ), + ]); + return { resultPromise }; + } finally { + // Set this before the mutex callback returns: an abort racing with release can safely wait + // for the lock, while an earlier abort avoids self-deadlock by using the held-lock path. + startLockState.held = false; + } }); - await this.notifyRunStatusChanged(run, "running"); - return { runId: input.runId, status: "running", result: null }; + await this.sweepInterruptedRunTasks(deferredInterruptRunIds); + return await dispatch.resultPromise; } async resumeRun(input: { @@ -357,15 +516,12 @@ export class WorkflowService { projectTrusted: boolean; abortSignal?: AbortSignal; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanTransition(run.status, "running"); - return await this.runForegroundWithAbortInterrupt({ - workspaceId: input.workspaceId, - run, - projectTrusted: input.projectTrusted, - abortSignal: input.abortSignal, - runnerOptions: { allowResumeFromInterrupted: run.status === "interrupted" }, + return await this.dispatchForegroundRunWithStartLock({ + ...input, + validateRun: (run) => assertWorkflowRunCanTransition(run.status, "running"), + runnerOptions: (run) => ({ + allowResumeFromInterrupted: run.status === "interrupted", + }), backgroundedFailureMessage: "Backgrounded workflow resume failed:", }); } @@ -376,15 +532,10 @@ export class WorkflowService { projectTrusted: boolean; abortSignal?: AbortSignal; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanRetryFromCheckpoint(run); - return await this.runForegroundWithAbortInterrupt({ - workspaceId: input.workspaceId, - run, - projectTrusted: input.projectTrusted, - abortSignal: input.abortSignal, - runnerOptions: { allowRetryFromFailedCheckpoint: true }, + return await this.dispatchForegroundRunWithStartLock({ + ...input, + validateRun: assertWorkflowRunCanRetryFromCheckpoint, + runnerOptions: () => ({ allowRetryFromFailedCheckpoint: true }), backgroundedFailureMessage: "Backgrounded workflow checkpoint retry failed:", }); } @@ -403,6 +554,11 @@ export class WorkflowService { "allowResumeFromInterrupted" | "allowRetryFromFailedCheckpoint" >; backgroundedFailureMessage: string; + startLockState?: { + isHeld: () => boolean; + onRunInterrupted: (runId: string) => void; + }; + onRunningStatusPersisted?: () => void; }): Promise { const runId = input.run.id; if (isAbortSignalAborted(input.abortSignal)) { @@ -411,17 +567,22 @@ export class WorkflowService { throw new Error(`Workflow run interrupted: ${runId}`); } - await this.notifyRunStatusChanged(input.run, "running"); - const runnerAbortController = new AbortController(); let unregisterRunnerAbort: () => void = () => undefined; const abortInterrupt = this.interruptRunOnAbort( input.workspaceId, runId, input.abortSignal, - runnerAbortController + runnerAbortController, + input.startLockState ); try { + await this.notifyRunStatusChanged(input.run, "running"); + if (input.abortSignal?.aborted === true) { + await abortInterrupt.wait(); + throw new Error(`Workflow run interrupted: ${runId}`); + } + const runner = await this.createRunner(runId); const result = await runner.run(runId, { abortSignal: runnerAbortController.signal, @@ -431,6 +592,13 @@ export class WorkflowService { runnerAbortController ); }, + ...(input.startLockState != null + ? { + shouldDeferRunEnded: input.startLockState.isHeld, + onRunEndedDeferred: () => input.startLockState?.onRunInterrupted(runId), + } + : {}), + onRunningStatusPersisted: input.onRunningStatusPersisted, ...input.runnerOptions, }); await this.notifyRunStatusChanged(input.run, "completed"); @@ -468,24 +636,26 @@ export class WorkflowService { } async startWorkflowInBackground(input: StartWorkflowInput): Promise { - const createdRun = await this.createWorkflowRun({ - ...input, - attentionPolicy: "notify_on_terminal", + return await this.withWorkflowRunStartLock(input.workspaceId, async () => { + const createdRun = await this.createWorkflowRunUnlocked({ + ...input, + attentionPolicy: "notify_on_terminal", + }); + const runId = createdRun.id; + await this.notifyRunStatusChanged(createdRun); + await input.onRunCreated?.({ runId, status: "pending", result: null, run: createdRun }); + const run = await this.runStore.appendStatus( + runId, + "running", + this.clock?.nowIso() ?? new Date().toISOString() + ); + await this.notifyRunStatusChanged(run); + await input.onBackgroundRunCreated?.({ runId, status: "running", result: null, run }); + void this.runInBackground(runId, "Background workflow run failed:", { + projectTrusted: input.projectTrusted, + }).catch(() => undefined); + return { runId, status: "running", result: null }; }); - const runId = createdRun.id; - await this.notifyRunStatusChanged(createdRun); - await input.onRunCreated?.({ runId, status: "pending", result: null, run: createdRun }); - const run = await this.runStore.appendStatus( - runId, - "running", - this.clock?.nowIso() ?? new Date().toISOString() - ); - await this.notifyRunStatusChanged(run); - await input.onBackgroundRunCreated?.({ runId, status: "running", result: null, run }); - void this.runInBackground(runId, "Background workflow run failed:", { - projectTrusted: input.projectTrusted, - }).catch(() => undefined); - return { runId, status: "running", result: null }; } async startWorkflow(input: StartWorkflowInput): Promise { @@ -654,24 +824,68 @@ export class WorkflowService { workspaceId: string, runId: string, abortSignal: AbortSignal | undefined, - runnerAbortController: AbortController | undefined + runnerAbortController: AbortController | undefined, + startLockState?: { + isHeld: () => boolean; + onRunInterrupted: (runId: string) => void; + } ): { remove: () => void; wait: () => Promise } { if (abortSignal == null) { return { remove: () => undefined, wait: () => Promise.resolve() }; } let interruptPromise: Promise | null = null; + let interruptStarted = false; + const getLockOptions = () => + startLockState?.isHeld() === true + ? { + lockAlreadyHeld: true as const, + deferTaskSweep: true as const, + onRunInterrupted: startLockState.onRunInterrupted, + } + : {}; const interrupt = () => { + if (interruptStarted) { + return; + } + interruptStarted = true; // Cancel the coordinator before interrupt side effects can block on task cleanup or disk I/O. runnerAbortController?.abort(); interruptPromise = (async () => { try { - await this.interruptRun({ workspaceId, runId }); - } catch { - // The run may have completed or failed before the abort event was delivered. + await this.interruptRun({ workspaceId, runId, ...getLockOptions() }); + } catch (error: unknown) { + const run = await this.getRun({ workspaceId, runId }); + if (run?.status === "completed" || run?.status === "failed") { + return; + } + if (run?.status === "interrupted") { + try { + await this.interruptRun({ + workspaceId, + runId, + retryTaskCleanup: true, + ...getLockOptions(), + }); + return; + } catch (retryError: unknown) { + // A resume can win between the interrupted re-read and cleanup retry. Ignore only + // terminal completion/failure; if the run remains interrupted, preserve the cleanup + // failure so callers know workflow-owned workers may still need repair. + const latestRun = await this.getRun({ workspaceId, runId }); + if (latestRun?.status === "completed" || latestRun?.status === "failed") { + return; + } + throw retryError; + } + } + throw error; } })(); }; abortSignal.addEventListener("abort", interrupt, { once: true }); + if (abortSignal.aborted) { + interrupt(); + } return { remove: () => abortSignal.removeEventListener("abort", interrupt), wait: async () => { @@ -681,6 +895,12 @@ export class WorkflowService { } private async createWorkflowRun(input: StartWorkflowInput): Promise { + return await this.withWorkflowRunStartLock(input.workspaceId, async () => + this.createWorkflowRunUnlocked(input) + ); + } + + private async createWorkflowRunUnlocked(input: StartWorkflowInput): Promise { assert( input.workspaceId.length > 0, "WorkflowService.createWorkflowRun: workspaceId is required" @@ -744,16 +964,18 @@ export class WorkflowService { runId, runnerAbortController ); - markStarted(); }; const runPromise = runner .run(runId, { abortSignal: runnerAbortController.signal, onLeaseAcquired: markLeaseAcquired, + onRunningStatusPersisted: markStarted, backgroundOnMessageQueued: false, ...runnerOptions, }) .then(async (result) => { + // Completed runs can return before another running-status append; unblock startup waiters. + markStarted(); await this.notifyRunStatusChanged(runStatus, "completed"); await this.notifyBackgroundRunTerminal(runId, result); }) diff --git a/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts b/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts index 184f628b542..d38e8affc65 100644 --- a/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts +++ b/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts @@ -631,6 +631,57 @@ describe("WorkflowTaskServiceAdapter", () => { }); }); + test("can defer the workflow task sweep when the caller already holds the tree lock", async () => { + const create = mock(async () => + Ok({ taskId: "task_1", kind: "agent" as const, status: "running" as const }) + ); + const waitForAgentReport = mock(async () => ({ reportMarkdown: "unused" })); + const terminateAllDescendantAgentTasks = mock(async () => ["task_1"]); + const adapter = new WorkflowTaskServiceAdapter({ + taskService: { create, waitForAgentReport, terminateAllDescendantAgentTasks }, + parentWorkspaceId: "parent_1", + workflowRunId: "wfr_123", + defaultAgentId: "explore", + }); + + await adapter.interruptRun({ deferTaskSweep: true }); + + expect(terminateAllDescendantAgentTasks).toHaveBeenCalledWith("parent_1", { + workflowRunId: "wfr_123", + deferWorkflowSweep: true, + }); + }); + + test("cleans background processes for workflow-owned task workspaces during interruption", async () => { + const create = mock(async () => + Ok({ taskId: "task_1", kind: "agent" as const, status: "running" as const }) + ); + const waitForAgentReport = mock(async () => ({ reportMarkdown: "unused" })); + const cleanupWorkspaceBackgroundProcesses = mock(async () => undefined); + const terminateAllDescendantAgentTasks = mock( + async ( + _workspaceId: string, + options?: { + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; + } + ) => { + await options?.cleanupWorkspaceBackgroundProcesses?.("workflow-task"); + return ["workflow-task"]; + } + ); + const adapter = new WorkflowTaskServiceAdapter({ + taskService: { create, waitForAgentReport, terminateAllDescendantAgentTasks }, + parentWorkspaceId: "parent_1", + workflowRunId: "wfr_123", + defaultAgentId: "explore", + cleanupWorkspaceBackgroundProcesses, + }); + + await adapter.interruptRun(); + + expect(cleanupWorkspaceBackgroundProcesses).toHaveBeenCalledWith("workflow-task"); + }); + test("fails fast when task creation fails", async () => { const create = mock(async () => ({ success: false as const, error: "no runnable agent" })); const waitForAgentReport = mock(async () => ({ reportMarkdown: "should not wait" })); diff --git a/src/node/services/workflows/WorkflowTaskServiceAdapter.ts b/src/node/services/workflows/WorkflowTaskServiceAdapter.ts index 076597dfabc..8d9244dccda 100644 --- a/src/node/services/workflows/WorkflowTaskServiceAdapter.ts +++ b/src/node/services/workflows/WorkflowTaskServiceAdapter.ts @@ -102,7 +102,11 @@ interface WorkflowTaskServiceLike { ): Promise; terminateAllDescendantAgentTasks?( workspaceId: string, - options?: { workflowRunId?: string } + options?: { + workflowRunId?: string; + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; + deferWorkflowSweep?: boolean; + } ): Promise; markWorkflowRunEnded?(workflowRunId: string): Promise; } @@ -129,6 +133,7 @@ export interface WorkflowTaskServiceAdapterOptions { patchToolConfig?: TaskApplyGitPatchConfiguration; applyPatchArtifact?: WorkflowPatchArtifactApplier; getProjectTrusted?: () => boolean | Promise; + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; } export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { @@ -140,6 +145,7 @@ export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { private readonly patchToolConfig?: TaskApplyGitPatchConfiguration; private readonly applyPatchArtifact?: WorkflowPatchArtifactApplier; private readonly getProjectTrusted?: () => boolean | Promise; + private readonly cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; private readonly patchApplyMutex = new AsyncMutex(); private readonly experiments?: WorkflowTaskExperiments; private readonly modelString?: string; @@ -166,6 +172,7 @@ export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { this.patchToolConfig = options.patchToolConfig; this.applyPatchArtifact = options.applyPatchArtifact; this.getProjectTrusted = options.getProjectTrusted; + this.cleanupWorkspaceBackgroundProcesses = options.cleanupWorkspaceBackgroundProcesses; this.experiments = options.experiments; this.modelString = options.modelString; this.thinkingLevel = options.thinkingLevel; @@ -338,9 +345,13 @@ export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { return undefined; } - async interruptRun(): Promise { + async interruptRun(options?: { deferTaskSweep?: boolean }): Promise { await this.taskService.terminateAllDescendantAgentTasks?.(this.parentWorkspaceId, { workflowRunId: this.workflowRunId, + ...(this.cleanupWorkspaceBackgroundProcesses != null + ? { cleanupWorkspaceBackgroundProcesses: this.cleanupWorkspaceBackgroundProcesses } + : {}), + ...(options?.deferTaskSweep === true ? { deferWorkflowSweep: true } : {}), }); }