From 157ac6360797a68732d7602a2d12cd495a567941 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 20:18:28 -0500 Subject: [PATCH 01/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20align=20task=20life?= =?UTF-8?q?cycle=20tool=20contracts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align task ID prompting with follow-up tool schemas, preserve precise await statuses, and enforce workflow-owned task isolation across lifecycle operations. Serialize workflow starts, resumes, retries, and interrupts with task-tree lifecycle locks while keeping cleanup retryable and deadlock-safe. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$87.15`_ --- docs/agents/system-prompt.mdx | 2 +- docs/hooks/tools.mdx | 42 +-- src/cli/workflow.ts | 4 + src/common/utils/tools/toolDefinitions.ts | 45 +-- src/common/utils/tools/tools.ts | 11 +- src/node/builtinSkills/orchestrate.md | 2 +- src/node/orpc/router.ts | 4 + .../builtInSkillContent.generated.ts | 46 +-- src/node/services/aiService.ts | 8 + src/node/services/systemMessage.ts | 2 +- src/node/services/taskService.test.ts | 153 +++++++- src/node/services/taskService.ts | 137 ++++++- src/node/services/tools/task.bash.test.ts | 33 +- src/node/services/tools/task_await.test.ts | 66 ++-- src/node/services/tools/task_await.ts | 33 +- src/node/services/tools/task_remove.ts | 1 - src/node/services/tools/task_stop.test.ts | 221 ++++++++++- src/node/services/tools/task_stop.ts | 180 ++++++++- src/node/services/workflows/WorkflowRunner.ts | 19 +- .../workflows/WorkflowService.test.ts | 247 ++++++++++++- .../services/workflows/WorkflowService.ts | 346 ++++++++++++++---- .../WorkflowTaskServiceAdapter.test.ts | 51 +++ .../workflows/WorkflowTaskServiceAdapter.ts | 15 +- 23 files changed, 1406 insertions(+), 262 deletions(-) diff --git a/docs/agents/system-prompt.mdx b/docs/agents/system-prompt.mdx index ed2a73bd0de..b2454ba533c 100644 --- a/docs/agents/system-prompt.mdx +++ b/docs/agents/system-prompt.mdx @@ -55,7 +55,7 @@ When the user asks for "best of n" work, assume they want the \`task\` tool's \` Before spawning the batch, do a small amount of preliminary analysis to capture shared context, constraints, or evaluation criteria that would otherwise be repeated by every child. Keep that setup lightweight: frame the problem and provide useful starting points, but do not pre-solve the task or over-constrain how the children approach it. Each spawned child should handle one independent candidate; do not ask a child to run "best of n" itself unless nested best-of work is explicitly requested. -Picking the best candidate requires every report, so await the full batch (pass \`task_await\` \`min_completed\` equal to the batch size, or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. +Picking the best candidate requires every report, so await the full batch with \`task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })\` (or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. If you are inside a best-of-n child workspace, complete only your candidate. diff --git a/docs/hooks/tools.mdx b/docs/hooks/tools.mdx index 2c3af99d6e9..b343a26317c 100644 --- a/docs/hooks/tools.mdx +++ b/docs/hooks/tools.mdx @@ -333,17 +333,17 @@ If a value is too large for the environment, it may be omitted (not set). Mux al
bash (9) -| Env var | JSON path | Type | Description | -| --------------------------------------- | ------------------------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. | -| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. | -| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. | -| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. | -| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. | -| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await (returns only new output since last check). Stop with task_stop using the taskId. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the returned taskId first. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. | -| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute | -| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive | +| Env var | JSON path | Type | Description | +| --------------------------------------- | ------------------------ | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. | +| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. | +| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. | +| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. | +| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. | +| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await({ task_ids: [result.taskId] }) (returns only new output since last check). Stop with task_stop({ task_ids: [result.taskId] }). List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. | +| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute | +| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |
@@ -693,8 +693,8 @@ If a value is too large for the environment, it may be omitted (not set). Mux al | `MUX_TOOL_INPUT_FILTER` | `filter` | string | Optional regex to filter bash task output lines. By default, only matching lines are returned. When filter_exclude is true, matching lines are excluded instead. Non-matching lines are discarded and cannot be retrieved later. | | `MUX_TOOL_INPUT_FILTER_EXCLUDE` | `filter_exclude` | boolean | When true, lines matching 'filter' are excluded instead of kept. Requires 'filter' to be set. | | `MUX_TOOL_INPUT_MIN_COMPLETED` | `min_completed` | number | Number of awaited tasks that must complete before this call returns. Defaults to 1, so by default task_await returns as soon as the FIRST awaited task completes, letting you act on it while the rest keep running. The result still includes every task complete at that moment plus current status (running/queued) for the rest. Tasks that have not yet completed keep running and remain re-awaitable on a later task_await call. Raise this (e.g. set it to the total number of awaited tasks) when you genuinely need more before proceeding — for example best-of-N synthesis that must compare every candidate. Clamped to the number of awaited tasks; values above that behave like 'wait for all'. | -| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. | -| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) | +| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. | +| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) | | `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Maximum time to wait in seconds for each task. For bash tasks, this waits for NEW output (or process exit). If exceeded, the result returns status=queued\|starting\|running\|awaiting_report (task is still active). Defaults to 600 seconds (10 minutes) if not specified. Set to 0 for a non-blocking status check. | @@ -737,7 +737,7 @@ If a value is too large for the environment, it may be omitted (not set). Mux al | ------------------------------------ | --------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- | | `MUX_TOOL_INPUT_MESSAGE` | `message` | string | Updated guidance to send to the sub-agent. | | `MUX_TOOL_INPUT_QUEUE_DISPATCH_MODE` | `queue_dispatch_mode` | enum | When the child is busy, dispatch the guidance at "tool-end" after its next tool call (default) or at "turn-end" after its current turn. | -| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active descendant sub-agent task ID returned by task or task_list. | +| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active or inactive persistent descendant sub-agent task ID returned by task or task_list. | @@ -798,19 +798,19 @@ If a value is too large for the environment, it may be omitted (not set). Mux al | ---------------------------------- | ------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `MUX_TOOL_INPUT_MODE` | `mode` | enum | Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. Use 'retry_from_checkpoint' only for failed runs; it re-executes work after the last checkpoint and is rejected when unsafe. | | `MUX_TOOL_INPUT_RUN_ID` | `run_id` | string | Workflow run ID (wfr\_...) to resume. Must belong to the current workspace. | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await the runId with task_await when you need the result. | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with task_await({ task_ids: [result.runId] }) when you need the result. |
workflow_run (4) -| Env var | JSON path | Type | Description | -| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using the result. | -| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. | -| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. | +| Env var | JSON path | Type | Description | +| ---------------------------------- | ------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result. | +| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. | +| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |
diff --git a/src/cli/workflow.ts b/src/cli/workflow.ts index 7ae4a933260..c04c0c503d1 100644 --- a/src/cli/workflow.ts +++ b/src/cli/workflow.ts @@ -436,6 +436,8 @@ function createWorkflowService(input: { return new WorkflowService({ runStore: new WorkflowRunStore({ sessionDir: workspaceSessionDir }), runtimeFactory: new QuickJSRuntimeFactory(), + withRunStartLock: (ownerWorkspaceId, operation) => + input.ctx.services.taskService.withWorkspaceOwnedWorkStartLock(ownerWorkspaceId, operation), taskAdapterFactory: (runId) => new WorkflowTaskServiceAdapter({ taskService: input.ctx.services.taskService, @@ -445,6 +447,8 @@ function createWorkflowService(input: { experiments, modelString: input.model, thinkingLevel: input.thinkingLevel, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + input.ctx.services.backgroundProcessManager.cleanup(taskWorkspaceId), getProjectTrusted: () => input.ctx.projectTrusted, patchToolConfig: { workspaceId: input.ctx.workspaceId, diff --git a/src/common/utils/tools/toolDefinitions.ts b/src/common/utils/tools/toolDefinitions.ts index 6ab3ac78dd8..12935d2d512 100644 --- a/src/common/utils/tools/toolDefinitions.ts +++ b/src/common/utils/tools/toolDefinitions.ts @@ -320,7 +320,7 @@ export function buildTaskToolDescription(runtimeMode: RuntimeMode | undefined): "\n\nWhen the user explicitly asks for best-of-n work, the parent should begin with light preliminary analysis to extract shared context, constraints, or evaluation criteria that would otherwise be duplicated across children. " + "Keep that pre-work lightweight: frame the task and provide useful starting points, but do not pre-solve the problem or over-constrain how the children reason about it. Then delegate the substantive analysis to the spawned sub-agents. " + "Do not also do a full parallel analysis in the parent. Call task_await when you are ready to act on child output; do not await reflexively just because tasks are running. " + - "task_await returns as soon as the first awaited task completes by default (min_completed), so you can start dependent work on each result as it lands instead of blocking on the whole batch; for best-of-N synthesis that must compare every candidate, pass min_completed equal to the batch size (or use a foreground grouped spawn, below). " + + "task_await returns as soon as the first awaited task completes by default (min_completed), so you can start dependent work on each result as it lands instead of blocking on the whole batch. Pass returned camelCase IDs through task_await's snake_case input: task_await({ task_ids: [result.taskId] }) for one handle, or task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length }) when every grouped result is required (or use a foreground grouped spawn, below). " + "\n\nWhen delegating, include a compact task brief (Task / Background / Scope / Starting points / Acceptance / Deliverables / Constraints). " + "For now, persisted sub-agent goals are not supported; pass sub-agent objectives, success criteria, and deliverables directly in the prompt. " + "Sub-agents observe the same system instructions as the parent (project/global AGENTS.md and custom instructions), so do not restate that shared context in the prompt; spend the prompt on task-specific information the sub-agent cannot infer from those instructions. " + @@ -617,8 +617,8 @@ export const TaskAwaitToolArgsSchema = z .nullish() .describe( "List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. " + - "task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. " + - "When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs." + "task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. " + + "When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs." ), filter: z .string() @@ -725,7 +725,7 @@ const TaskAwaitToolArtifactsSchema = z * and can be re-fetched by ID after context compaction instead of re-running the work. */ export const COMPLETED_REPORT_REFETCH_NOTE = - 'Report persisted on disk; re-fetch anytime (even after context compaction) with task_await(task_ids: [""], timeout_secs: 0).'; + 'Report persisted on disk; re-fetch anytime (even after context compaction) with task_await({ task_ids: [""], timeout_secs: 0 }).'; export const TaskAwaitToolCompletedResultSchema = z .object({ @@ -946,7 +946,9 @@ export const TaskSendMessageToolArgsSchema = z task_id: z .string() .min(1) - .describe("Active descendant sub-agent task ID returned by task or task_list."), + .describe( + "Active or inactive persistent descendant sub-agent task ID returned by task or task_list." + ), message: z.string().trim().min(1).describe("Updated guidance to send to the sub-agent."), queue_dispatch_mode: z .enum(["tool-end", "turn-end"]) @@ -1131,7 +1133,6 @@ export const TaskRemoveToolResultSchema = z TaskRemoveToolBaseResultSchema.extend({ status: z.literal("removed") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("already_removed") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("active") }).strict(), - TaskRemoveToolBaseResultSchema.extend({ status: z.literal("not_found") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("invalid_scope") }).strict(), TaskRemoveToolBaseResultSchema.extend({ status: z.literal("error") }).strict(), ]) @@ -1140,7 +1141,7 @@ export const TaskRemoveToolResultSchema = z .strict(); // ----------------------------------------------------------------------------- -// task_terminate (terminate sub-agent/bash tasks, interrupt workflow runs) +// task_terminate (legacy schema retained to render historical tool calls) // ----------------------------------------------------------------------------- export const TaskTerminateToolArgsSchema = z .object({ @@ -1256,7 +1257,7 @@ export const TaskWorkspaceLifecycleToolArgsSchema = z .boolean() .nullish() .describe( - "When true, interrupt active workspace turns for the target before performing an otherwise-eligible lifecycle action. Active sub-agents must be discarded with task_terminate instead. Defaults to false." + "When true, interrupt active workspace turns for the target before performing an otherwise-eligible lifecycle action. Active sub-agents must be stopped with task_stop instead. Defaults to false." ), force: z .boolean() @@ -1405,7 +1406,7 @@ export const WorkflowRunToolArgsSchema = z .default(false) .describe( "Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. " + - "Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using the result." + "Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result." ), }) .strict() @@ -1448,7 +1449,7 @@ export const WorkflowResumeToolArgsSchema = z .default(false) .describe( "Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. " + - "Set true to resume in the background and continue other work; await the runId with task_await when you need the result." + "Set true to resume in the background and continue other work; await it with task_await({ task_ids: [result.runId] }) when you need the result." ), mode: WorkflowResumeModeSchema.nullish().describe( "Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. " + @@ -1681,15 +1682,15 @@ export const TOOL_DEFINITIONS = { "Do NOT use for quick commands (<5s), interactive processes (no stdin support), " + "or processes requiring real-time output (use foreground with larger timeout instead). " + "Returns immediately with a taskId (bash:) and backgroundProcessId. " + - "Read output with task_await (returns only new output since last check). " + - "Stop with task_stop using the taskId. " + + "Read output with task_await({ task_ids: [result.taskId] }) (returns only new output since last check). " + + "Stop with task_stop({ task_ids: [result.taskId] }). " + "List active tasks with task_list. " + "Process persists until timeout_secs expires, terminated, or workspace is removed." + "\\n\\nFor long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. " + "Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. " + "With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. " + "Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. " + - "Do not call task_await in the same parallel tool-call batch; wait for the returned taskId first. " + + "Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. " + "When you actually need the output, read it with task_await; do not poll task_await just because the process is still running." ), monitor: BashMonitorSchema.nullish().describe( @@ -2243,8 +2244,8 @@ export const TOOL_DEFINITIONS = { "When a terminal wake-up says a sub-agent report or failure is already injected into context, integrate it directly — do NOT call task_await for it. When a wake-up asks you to retrieve a workspace turn's terminal output, call task_await with the listed IDs and timeout_secs: 0 (a one-shot retrieval, not a wait). " + "\n\nIMPORTANT: Do not call task_await in the same parallel tool-call batch as task, bash, or workflow_run — " + "the taskId/runId is not available until the spawning tool returns. " + - "Always wait for the task/bash/workflow_run tool result first, then call task_await in a subsequent step. " + - "When omitting task_ids to await active tasks/workflows, ensure at least one background task or workflow was already spawned in a prior step. Omitted task_ids discover top-level workflow runs only and exclude workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. " + + "Always wait for the task/bash/workflow_run tool result first, then call task_await in a subsequent step. Map returned result fields explicitly: task_await({ task_ids: [result.taskId] }) or task_await({ task_ids: result.taskIds }). " + + "When omitting task_ids to await active tasks/workflows, ensure at least one background task or workflow was already spawned in a prior step. Omitted task_ids discover active in-scope descendant agent tasks, public workspace turns, background bash tasks, and top-level workflow runs, while excluding workflow-owned internal workers because their results are consumed through parent workflow runs. " + "\n\nAgent tasks and workflow runs return reports when completed. " + "Completed reports are persisted on disk and survive context compaction: calling task_await on an already-completed task/workflow run ID (timeout_secs: 0 for non-blocking) re-fetches the full report instead of re-running the work. " + "Bash tasks return incremental output while running and a final reportMarkdown when they exit. " + @@ -2263,8 +2264,8 @@ export const TOOL_DEFINITIONS = { }, task_send_message: { description: - "Send guidance to a descendant sub-agent. Queued/running work is interrupted or queued at the requested boundary so the child can incorporate the update. An inactive child is reawakened in the same persistent workspace under a fresh internal execution. " + - "The stable sub-agent task ID and durable role title remain unchanged. If the new assignment changes the child's reusable responsibility, call task_retitle as well; do not retitle it for ordinary one-off assignments. Best-of children retain candidate metadata, so reawaken them only to continue that same candidate; use a standalone specialist for unrelated work. This tool does not target bash tasks, workflow runs, or workspace-turn handles.", + "Send guidance to a user-owned descendant sub-agent. Queued/running work is interrupted or queued at the requested boundary so the child can incorporate the update. An inactive child is reawakened in the same persistent workspace under a fresh internal execution. " + + "The stable sub-agent task ID and durable role title remain unchanged. If the new assignment changes the child's reusable responsibility, call task_retitle as well; do not retitle it for ordinary one-off assignments. Best-of children retain candidate metadata, so reawaken them only to continue that same candidate; use a standalone specialist for unrelated work. This tool does not target workflow-owned internal workers, bash tasks, workflow runs, or workspace-turn handles.", schema: TaskSendMessageToolArgsSchema, }, task_retitle: { @@ -2274,18 +2275,18 @@ export const TOOL_DEFINITIONS = { }, task_stop: { description: - "Stop one or more tasks without removing persistent child workspaces. Sub-agent trees are stopped leaf-first and unfinished children become interrupted; workspace turns and workflow runs are interrupted; bash processes are terminated. Use this to cancel or abandon work, not to mark useful progress as completed—ask a child to finalize with task_send_message and await its report instead. Stopping an already-inactive task is idempotent.", + "Stop one or more user-owned tasks without removing persistent child workspaces. Sub-agent trees are stopped leaf-first and unfinished children become interrupted; workspace turns and workflow runs are interrupted; bash processes are terminated. Workflow-owned internal workers are controlled through their owning workflow run and return invalid_scope when targeted directly. Use this to cancel or abandon work, not to mark useful progress as completed—ask a child to finalize with task_send_message and await its report instead. Stopping any already-terminal handle is idempotent.", schema: TaskStopToolArgsSchema, }, task_remove: { description: - "Irreversibly remove inactive child task workspaces owned by the current workspace. Removed sub-agents cannot be restored or reawakened. Active targets are rejected; descendants must be removed first, so nested batches are processed deepest-first.", + "Irreversibly remove inactive user-owned child task workspaces. Workflow-owned internal workers are controlled through their owning workflow run and return invalid_scope when targeted directly. Removed sub-agents cannot be restored or reawakened. Active targets are rejected; descendants must be removed first, so nested batches are processed deepest-first.", schema: TaskRemoveToolArgsSchema, }, task_list: { description: "List descendant tasks for the current workspace, including status + metadata. " + - "This includes sub-agent tasks, background bash tasks, and top-level workflow runs, but omits workflow-owned sub-agents/background bash tasks whose reports are consumed through parent workflow runs. " + + "This includes sub-agent tasks, public workspace turns, background bash tasks, and top-level workflow runs, but omits workflow-owned sub-agents/background bash tasks whose reports are consumed through parent workflow runs. " + "Use this after compaction, interruptions, workflow_run errors/aborts, or an app restart to rediscover active tasks, inactive persistent sub-agents, and resumable workflow runs. The default statuses find unfinished work; request `reported` explicitly for completed persistent sub-agents. " + "When recovering an uncertain workflow_run, omit statuses first or include pending/running/backgrounded as well as interrupted/failed/completed; terminal-only filters can hide unfinished workflow runs. Pending runs may need workflow_resume because no runner may be active yet. " + "Workflow rows may include compact `workflowProgress` so callers can see the latest phase before deciding whether to await, resume, or leave the run alone. " + @@ -2302,7 +2303,7 @@ export const TOOL_DEFINITIONS = { "the conductor follows the documented phases more faithfully and gains durable checkpoints, resume, and fresh delegated context per phase. " + "Use agent_skill_read / agent_skill_read_file to discover and inspect skill-packaged workflows; non-skill workflow files must be addressed by an explicit known path and can be inspected with normal file tools. " + "Prefer the default foreground mode (`run_in_background` omitted or false) so completed workflows return their result without an extra task_await round-trip. " + - "If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using or reporting the workflow output. " + + "If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using or reporting the workflow output. " + "After a previous workflow_run error, abort, timeout, or uncertain result, do not start a fresh run until you rediscover existing workflow runs: either omit task_list statuses first, or query pending/running/backgrounded/interrupted/failed/completed together. " + "Use task_await for running/backgrounded runs, workflow_resume for pending/interrupted runs, workflow_resume({ mode: 'retry_from_checkpoint' }) only for eligible failed runs, and inspect/refetch completed results instead of rerunning. " + "Use background mode only when you intend to start another workflow/task or do independent work while the workflow runs; a background run is non-blocking and Mux wakes this workspace with the terminal workflow result, so call task_await only when the current request depends on the output before you can answer.", @@ -2316,7 +2317,7 @@ export const TOOL_DEFINITIONS = { "For failed runs, pass mode='retry_from_checkpoint' explicitly; it re-executes work after the last checkpoint, so only use it when that is acceptable, and start a fresh workflow_run when it is rejected as unsafe. " + "Calling this on a completed run returns its existing result without re-running anything. " + "Prefer foreground mode (run_in_background omitted or false) to get the final result directly; " + - "if the returned status is running or backgrounded, await the runId with task_await before using the result.", + "if the returned status is running or backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result.", schema: WorkflowResumeToolArgsSchema, }, agent_report: { diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index ef41a0b22ac..66b712494cc 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -231,7 +231,14 @@ export interface ToolConfiguration { run: unknown; }) => Promise | void; }): Promise<{ runId: string; status: string; result: unknown }>; - interruptRun?(input: { workspaceId: string; runId: string }): Promise; + interruptRun?(input: { + workspaceId: string; + runId: string; + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }): Promise; resumeRun?(input: { workspaceId: string; runId: string; @@ -765,7 +772,7 @@ export async function getToolsForModel( task_remove: wrap(createTaskRemoveTool(config)), task_list: wrap(createTaskListTool(config)), - // Bash execution (foreground/background). Manage background output via task_await/task_list/task_terminate. + // Bash execution (foreground/background). Manage background output via task_await/task_list/task_stop. bash: wrap(createBashTool(config)), // Legacy bash process tools (deprecated) diff --git a/src/node/builtinSkills/orchestrate.md b/src/node/builtinSkills/orchestrate.md index eb07ac90811..f785b051c24 100644 --- a/src/node/builtinSkills/orchestrate.md +++ b/src/node/builtinSkills/orchestrate.md @@ -138,7 +138,7 @@ In a workflow, the verifier becomes `agent(prompt, { id, schema, onRefusal: "fai ## Sequential protocol (only for dependency chains) 1. Spawn the prerequisite `exec` implementation task with `run_in_background: false`. -2. If step 1 returns `queued`/`running` without a completed report, call `task_await` with the returned `taskId` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application. +2. If step 1 returns `queued`/`running` without a completed report, call `task_await({ task_ids: [result.taskId] })` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application. 3. Dry-run apply its patch (`dry_run: true`); then apply for real (`dry_run: false`). If either step fails, follow the conflict playbook above (including `git am --abort` only when a real apply leaves a git-am session in progress). 4. Only then spawn the dependent task. diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index a94fcada3d1..ff284f9fe00 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -457,6 +457,8 @@ export async function resolveWorkflowContext( options.notifyInterruptedBackgroundRunTerminal === true, runStore: new WorkflowRunStore({ sessionDir: context.config.getSessionDir(workspaceId) }), runtimeFactory: context.workflowRuntimeFactory, + withRunStartLock: (ownerWorkspaceId, operation) => + context.taskService.withWorkspaceOwnedWorkStartLock(ownerWorkspaceId, operation), taskAdapterFactory: (runId, workflowName) => new WorkflowTaskServiceAdapter({ taskService: context.taskService, @@ -472,6 +474,8 @@ export async function resolveWorkflowContext( workspaceSessionDir: context.config.getSessionDir(workspaceId), trusted: projectTrusted, }, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + context.aiService.cleanupWorkspaceBackgroundProcesses(taskWorkspaceId), getProjectTrusted: resolveWorkflowProjectTrusted, experiments: { dynamicWorkflows: true, diff --git a/src/node/services/agentSkills/builtInSkillContent.generated.ts b/src/node/services/agentSkills/builtInSkillContent.generated.ts index 119658ede5f..dd55ad84355 100644 --- a/src/node/services/agentSkills/builtInSkillContent.generated.ts +++ b/src/node/services/agentSkills/builtInSkillContent.generated.ts @@ -2701,7 +2701,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "Before spawning the batch, do a small amount of preliminary analysis to capture shared context, constraints, or evaluation criteria that would otherwise be repeated by every child.", "Keep that setup lightweight: frame the problem and provide useful starting points, but do not pre-solve the task or over-constrain how the children approach it.", 'Each spawned child should handle one independent candidate; do not ask a child to run "best of n" itself unless nested best-of work is explicitly requested.', - "Picking the best candidate requires every report, so await the full batch (pass \\`task_await\\` \\`min_completed\\` equal to the batch size, or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands.", + "Picking the best candidate requires every report, so await the full batch with \\`task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })\\` (or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands.", "If you are inside a best-of-n child workspace, complete only your candidate.", "", "", @@ -5229,17 +5229,17 @@ export const BUILTIN_SKILL_FILES: Record> = { "
", "bash (9)", "", - "| Env var | JSON path | Type | Description |", - "| --------------------------------------- | ------------------------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", - "| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. |", - "| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. |", - "| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. |", - "| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. |", - "| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. |", - "| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await (returns only new output since last check). Stop with task_stop using the taskId. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the returned taskId first. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. |", - "| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute |", - "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |", + "| Env var | JSON path | Type | Description |", + "| --------------------------------------- | ------------------------ | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |", + "| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. |", + "| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. |", + "| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. |", + "| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. |", + "| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. |", + "| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await({ task_ids: [result.taskId] }) (returns only new output since last check). Stop with task_stop({ task_ids: [result.taskId] }). List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. |", + "| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute |", + "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |", "", "
", "", @@ -5589,8 +5589,8 @@ export const BUILTIN_SKILL_FILES: Record> = { "| `MUX_TOOL_INPUT_FILTER` | `filter` | string | Optional regex to filter bash task output lines. By default, only matching lines are returned. When filter_exclude is true, matching lines are excluded instead. Non-matching lines are discarded and cannot be retrieved later. |", "| `MUX_TOOL_INPUT_FILTER_EXCLUDE` | `filter_exclude` | boolean | When true, lines matching 'filter' are excluded instead of kept. Requires 'filter' to be set. |", "| `MUX_TOOL_INPUT_MIN_COMPLETED` | `min_completed` | number | Number of awaited tasks that must complete before this call returns. Defaults to 1, so by default task_await returns as soon as the FIRST awaited task completes, letting you act on it while the rest keep running. The result still includes every task complete at that moment plus current status (running/queued) for the rest. Tasks that have not yet completed keep running and remain re-awaitable on a later task_await call. Raise this (e.g. set it to the total number of awaited tasks) when you genuinely need more before proceeding — for example best-of-N synthesis that must compare every candidate. Clamped to the number of awaited tasks; values above that behave like 'wait for all'. |", - "| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. |", - "| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent/background bash IDs, but top-level workflow run rediscovery is done by omitting task_ids. When omitted, waits for active descendant tasks and top-level workflow runs of the current workspace, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) |", + "| `MUX_TOOL_INPUT_TASK_IDS_` | `task_ids[]` | string | List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs. |", + "| `MUX_TOOL_INPUT_TASK_IDS_COUNT` | `task_ids.length` | number | Number of elements in task_ids (List of task IDs or workflow run IDs to await — use only real IDs returned by prior task, bash, or workflow_run results; never fabricate an ID. task_list can rediscover sub-agent, background bash, workspace-turn, and top-level workflow run IDs. When omitted, waits for all active in-scope handles of those kinds, excluding workflow-owned sub-agents/background bash tasks because those results are consumed through parent workflow runs.) |", "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Maximum time to wait in seconds for each task. For bash tasks, this waits for NEW output (or process exit). If exceeded, the result returns status=queued\\|starting\\|running\\|awaiting_report (task is still active). Defaults to 600 seconds (10 minutes) if not specified. Set to 0 for a non-blocking status check. |", "", "", @@ -5633,7 +5633,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "| ------------------------------------ | --------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- |", "| `MUX_TOOL_INPUT_MESSAGE` | `message` | string | Updated guidance to send to the sub-agent. |", '| `MUX_TOOL_INPUT_QUEUE_DISPATCH_MODE` | `queue_dispatch_mode` | enum | When the child is busy, dispatch the guidance at "tool-end" after its next tool call (default) or at "turn-end" after its current turn. |', - "| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active descendant sub-agent task ID returned by task or task_list. |", + "| `MUX_TOOL_INPUT_TASK_ID` | `task_id` | string | Active or inactive persistent descendant sub-agent task ID returned by task or task_list. |", "", "", "", @@ -5694,19 +5694,19 @@ export const BUILTIN_SKILL_FILES: Record> = { "| ---------------------------------- | ------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", "| `MUX_TOOL_INPUT_MODE` | `mode` | enum | Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. Use 'retry_from_checkpoint' only for failed runs; it re-executes work after the last checkpoint and is rejected when unsafe. |", "| `MUX_TOOL_INPUT_RUN_ID` | `run_id` | string | Workflow run ID (wfr\\_...) to resume. Must belong to the current workspace. |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await the runId with task_await when you need the result. |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with task_await({ task_ids: [result.runId] }) when you need the result. |", "", "", "", "
", "workflow_run (4)", "", - "| Env var | JSON path | Type | Description |", - "| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |", - "| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await the returned runId with task_await before using the result. |", - '| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. |', - "| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |", + "| Env var | JSON path | Type | Description |", + "| ---------------------------------- | ------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", + "| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result. |", + '| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. |', + "| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |", "", "
", "", @@ -7880,7 +7880,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "## Sequential protocol (only for dependency chains)", "", "1. Spawn the prerequisite `exec` implementation task with `run_in_background: false`.", - "2. If step 1 returns `queued`/`running` without a completed report, call `task_await` with the returned `taskId` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application.", + "2. If step 1 returns `queued`/`running` without a completed report, call `task_await({ task_ids: [result.taskId] })` before attempting any patch apply. If step 1 returns `status: completed` inline, that same `taskId` still requires patch application.", "3. Dry-run apply its patch (`dry_run: true`); then apply for real (`dry_run: false`). If either step fails, follow the conflict playbook above (including `git am --abort` only when a real apply leaves a git-am session in progress).", "4. Only then spawn the dependent task.", "", diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index af1eccb4697..bc73b126a33 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -557,6 +557,10 @@ export class AIService extends EventEmitter { private analyticsService?: { executeRawQuery(sql: string): Promise }; private desktopSessionManager?: DesktopSessionManager; + async cleanupWorkspaceBackgroundProcesses(workspaceId: string): Promise { + await this.backgroundProcessManager?.cleanup(workspaceId); + } + constructor( config: Config, historyService: HistoryService, @@ -2202,6 +2206,8 @@ export class AIService extends EventEmitter { await this.onWorkflowRunStatusChanged?.(event); }, runtimeFactory: new QuickJSRuntimeFactory(), + withRunStartLock: (ownerWorkspaceId, operation) => + this.taskService!.withWorkspaceOwnedWorkStartLock(ownerWorkspaceId, operation), taskAdapterFactory: (runId, workflowName) => new WorkflowTaskServiceAdapter({ taskService: this.taskService!, @@ -2217,6 +2223,8 @@ export class AIService extends EventEmitter { workspaceSessionDir: this.config.getSessionDir(workspaceId), trusted: getWorkflowProjectTrusted(), }, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + this.cleanupWorkspaceBackgroundProcesses(taskWorkspaceId), getProjectTrusted: getWorkflowProjectTrusted, experiments: { ...experiments, diff --git a/src/node/services/systemMessage.ts b/src/node/services/systemMessage.ts index f4bc8f41431..dd349605a83 100644 --- a/src/node/services/systemMessage.ts +++ b/src/node/services/systemMessage.ts @@ -91,7 +91,7 @@ When the user asks for "best of n" work, assume they want the \`task\` tool's \` Before spawning the batch, do a small amount of preliminary analysis to capture shared context, constraints, or evaluation criteria that would otherwise be repeated by every child. Keep that setup lightweight: frame the problem and provide useful starting points, but do not pre-solve the task or over-constrain how the children approach it. Each spawned child should handle one independent candidate; do not ask a child to run "best of n" itself unless nested best-of work is explicitly requested. -Picking the best candidate requires every report, so await the full batch (pass \`task_await\` \`min_completed\` equal to the batch size, or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. +Picking the best candidate requires every report, so await the full batch with \`task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })\` (or use a foreground grouped spawn) before selecting — but you may start setup-only work (e.g. preparing the evaluation rubric or integration scaffolding) as soon as the first candidate lands. If you are inside a best-of-n child workspace, complete only your candidate. diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4ad1017e040..119abc271fb 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -2121,6 +2121,18 @@ describe("TaskService", () => { expect(aiMocks.stopStream).not.toHaveBeenCalled(); }); + test("interruptWorkspaceTurn is idempotent after interruption", async () => { + const { parentId, taskService, aiMocks } = await startWorkspaceTurnForTest(); + + expect(await taskService.interruptWorkspaceTurn(parentId, "wst_handle")).toEqual( + Ok({ workspaceId: "childworkspace" }) + ); + expect(await taskService.interruptWorkspaceTurn(parentId, "wst_handle")).toEqual( + Ok({ workspaceId: "childworkspace", alreadyInactive: true }) + ); + expect(aiMocks.stopStream).toHaveBeenCalledTimes(1); + }); + test("createWorkspaceTurn reserves a slot before queueing a manually busy existing workspace", async () => { const config = await createTestConfig(rootDir); stubStableIds(config, ["queuedhandle", "queuedturn"]); @@ -5300,7 +5312,7 @@ describe("TaskService", () => { test("late correlated stream-end does not resettle an explicitly interrupted workspace turn", async () => { const { config, parentId, taskService } = await startWorkspaceTurnForTest(); - // Explicit interrupt (user Esc / task_terminate): status interrupted WITHOUT the + // Explicit interrupt (user Esc / task_stop): status interrupted WITHOUT the // stale-restart marker. An in-flight stream-end completing after the cancel must not // make the canceled turn appear completed. await new TaskHandleStore(config).upsertWorkspaceTurn({ @@ -11852,6 +11864,97 @@ describe("TaskService", () => { expect(updateTitle).not.toHaveBeenCalled(); }); + test("direct lifecycle operations reject workflow-owned descendants", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-workflow-owned-lifecycle"; + const workflowChildId = "workflow-owned-lifecycle-root"; + const messageChildId = "workflow-owned-message-child"; + const stopChildId = "workflow-owned-stop-child"; + const removeChildId = "workflow-owned-remove-child"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "workflow-child", workflowChildId, { + parentWorkspaceId, + taskStatus: "running", + workflowTask: { runId: "wfr_lifecycle", stepId: "step" }, + }), + projectWorkspace(projectPath, "message-child", messageChildId, { + parentWorkspaceId: workflowChildId, + taskStatus: "queued", + taskPrompt: "Original workflow-owned assignment", + }), + projectWorkspace(projectPath, "stop-child", stopChildId, { + parentWorkspaceId: workflowChildId, + taskStatus: "running", + }), + projectWorkspace(projectPath, "remove-child", removeChildId, { + parentWorkspaceId: workflowChildId, + taskStatus: "reported", + }), + ], + testTaskSettings() + ); + const { taskService } = createTaskServiceHarness(config); + + expect( + await taskService.sendMessageToDescendantAgentTask( + parentWorkspaceId, + messageChildId, + "Bypass the owning workflow", + "tool-end" + ) + ).toEqual(Err({ code: "invalid_scope" })); + expect(await taskService.stopDescendantAgentTask(parentWorkspaceId, stopChildId)).toEqual( + Err("Task is not a descendant of this workspace") + ); + expect( + await taskService.removeInactiveDescendantAgentTask(parentWorkspaceId, removeChildId) + ).toMatchObject({ success: true, data: { status: "invalid_scope" } }); + }); + + test("stopDescendantAgentTask leaves workflow-owned descendant branches to WorkflowService", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-stop-workflow-branch"; + const childTaskId = "user-owned-stop-root"; + const workflowChildId = "workflow-owned-stop-descendant"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "child", childTaskId, { + parentWorkspaceId, + taskStatus: "running", + }), + projectWorkspace(projectPath, "workflow-child", workflowChildId, { + parentWorkspaceId: childTaskId, + taskStatus: "running", + workflowTask: { runId: "wfr_stop_branch", stepId: "step" }, + }), + ], + testTaskSettings() + ); + const stopStream = mock((): Promise> => Promise.resolve(Ok(undefined))); + const { aiService } = createAIServiceMocks(config, { + isStreaming: mock(() => true), + stopStream, + }); + const { taskService } = createTaskServiceHarness(config, { aiService }); + + expect(await taskService.stopDescendantAgentTask(parentWorkspaceId, childTaskId)).toEqual( + Ok({ stoppedTaskIds: [childTaskId] }) + ); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(stopStream).toHaveBeenCalledWith(childTaskId, { abandonPartial: false }); + expect(findWorkspaceInConfig(config, childTaskId)?.taskStatus).toBe("interrupted"); + expect(findWorkspaceInConfig(config, workflowChildId)?.taskStatus).toBe("running"); + }); + test("retitleDescendantAgentTask surfaces title update failures", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -12901,13 +13004,30 @@ describe("TaskService", () => { await stopStarted; const creation = createAgentTask(taskService, childTaskId, "Spawn after stop"); + const createWorkflowRun = mock(() => Promise.resolve("created")); + const workflowCreation = taskService.withWorkspaceOwnedWorkStartLock( + childTaskId, + createWorkflowRun + ); await Promise.resolve(); expect(create).not.toHaveBeenCalled(); + expect(createWorkflowRun).not.toHaveBeenCalled(); releaseStop?.(); expect(await stopping).toEqual(Ok({ stoppedTaskIds: [childTaskId] })); expect(await creation).toEqual(Err("Task.create: cannot spawn new tasks after task_stop")); + let workflowCreationError: unknown; + try { + await workflowCreation; + } catch (error: unknown) { + workflowCreationError = error; + } + expect(workflowCreationError).toBeInstanceOf(Error); + expect((workflowCreationError as Error).message).toBe( + "Cannot start workflow work after task_stop" + ); expect(create).not.toHaveBeenCalled(); + expect(createWorkflowRun).not.toHaveBeenCalled(); }); test("bulk task creation waits for task stop and rejects the interrupted parent", async () => { @@ -13863,12 +13983,23 @@ describe("TaskService", () => { const { aiService } = createAIServiceMocks(config); const { workspaceService } = createWorkspaceServiceMocks(); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + const cleanupWorkspaceBackgroundProcesses = mock( + (workspaceId: string): Promise => + workspaceId === workflowChildTaskId + ? Promise.reject(new Error("background cleanup failed")) + : Promise.resolve() + ); const interruptedTaskIds = await taskService.terminateAllDescendantAgentTasks(rootWorkspaceId, { workflowRunId: "wfr_target", + cleanupWorkspaceBackgroundProcesses, }); expect(interruptedTaskIds).toEqual([workflowChildTaskId, workflowTaskId]); + expect(cleanupWorkspaceBackgroundProcesses.mock.calls.map((call) => call[0])).toEqual([ + workflowChildTaskId, + workflowTaskId, + ]); const saved = config.loadConfigOrDefault(); const tasks = saved.projects.get(projectPath)?.workspaces ?? []; expect(tasks.find((workspace) => workspace.id === workflowTaskId)?.taskStatus).toBe( @@ -14774,7 +14905,7 @@ describe("TaskService", () => { expect(userInterrupted?.taskStatus).toBe("interrupted"); }); - test("terminateAllDescendantAgentTasks archives run-scoped interrupted children immediately", async () => { + test("terminateAllDescendantAgentTasks can defer run-scoped archive sweeps", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -14814,14 +14945,18 @@ describe("TaskService", () => { const { workspaceService } = createWorkspaceServiceMocks({ archive }); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - // Run-scoped interrupt (WorkflowService.interruptRun path): the sweep archives the - // freshly interrupted workflow child even if the runner's onRunEnded hook already - // fired before the children were interrupted. - await taskService.terminateAllDescendantAgentTasks(rootId, { workflowRunId }); + // task_stop can already hold the non-reentrant tree lock while interrupting the owning run. + // Defer archive work until that outer lock releases, then run the normal idempotent sweep. + await taskService.terminateAllDescendantAgentTasks(rootId, { + workflowRunId, + deferWorkflowSweep: true, + }); - const workflowChild = findWorkspaceInConfig(config, workflowChildId); - expect(workflowChild?.taskStatus).toBe("interrupted"); - expect(workflowChild?.archivedAt).toBeString(); + expect(findWorkspaceInConfig(config, workflowChildId)?.taskStatus).toBe("interrupted"); + expect(findWorkspaceInConfig(config, workflowChildId)?.archivedAt).toBeUndefined(); + + await taskService.markWorkflowRunEnded(workflowRunId); + expect(findWorkspaceInConfig(config, workflowChildId)?.archivedAt).toBeString(); // The run-scoped filter leaves the user-spawned sibling running and unarchived. const userChild = findWorkspaceInConfig(config, userChildId); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 539aa61ffac..299f5ae51c9 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -192,6 +192,7 @@ export interface AgentTaskTimestamps { } type WorkspaceLifecycleResult = z.infer; +type RemoveInactiveAgentTaskResult = Exclude; export interface TaskCreateArgs { parentWorkspaceId: string; @@ -687,7 +688,7 @@ const WORKSPACE_TURN_STALE_RESTART_ERROR = "Workspace turn interrupted after res * Settled workspace-turn records eligible for self-heal correction (resettle from a * correlated stream-end, or read-time repair/revive): transient stream-error settlements * (status "error") and stale restart-recovery interrupts. Explicit interrupts — user Esc, - * task_terminate, cancel reasons — must stay terminal even if a late correlated stream-end + * task_stop, cancel reasons — must stay terminal even if a late correlated stream-end * or same-turn retry evidence arrives, so canceled work never resurfaces as completed. */ function isSelfHealEligibleSettledWorkspaceTurn( @@ -2095,6 +2096,38 @@ export class TaskService { return await acquire(0); } + async withWorkspaceOwnedWorkStartLock( + workspaceId: string, + operation: () => Promise + ): Promise { + assert(workspaceId.length > 0, "withWorkspaceOwnedWorkStartLock requires workspaceId"); + return await this.withTaskTreeLifecycleLock(workspaceId, async () => { + { + await using _lock = await this.mutex.acquire(); + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + if ( + entry != null && + isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt) + ) { + throw new Error("Cannot start workflow work from an archived workspace"); + } + if ( + entry?.workspace.taskStatus === "interrupted" && + !isActiveWorkspaceTurnTaskStatus(entry.workspace.taskExecutionStatus) + ) { + throw new Error("Cannot start workflow work after task_stop"); + } + if ( + entry?.workspace.taskStatus === "reported" && + !isActiveWorkspaceTurnTaskStatus(entry.workspace.taskExecutionStatus) + ) { + throw new Error("Cannot start workflow work after agent_report"); + } + } + return await operation(); + }); + } + private async editWorkspaceEntry( workspaceId: string, updater: (workspace: WorkspaceConfigEntry) => void, @@ -4444,7 +4477,10 @@ export class TaskService { taskIndex.parentById, ancestorWorkspaceId, taskId - ) + ) || + // Workflow-owned workers are private implementation details; their lifecycle is consumed + // through the owning workflow run rather than direct parent task tools. + this.isWorkflowOwnedTaskUsingIndex(taskIndex, taskId) ) { return Err({ code: "invalid_scope" as const }); } @@ -4487,7 +4523,8 @@ export class TaskService { taskIndex.parentById, ancestorWorkspaceId, taskId - ) + ) || + this.isWorkflowOwnedTaskUsingIndex(taskIndex, taskId) ) { return Err({ code: "invalid_scope" as const }); } @@ -4656,14 +4693,46 @@ export class TaskService { async stopDescendantAgentTask( ancestorWorkspaceId: string, - taskId: string + taskId: string, + options?: { beforeStop?: () => Promise } ): Promise> { assert(ancestorWorkspaceId.length > 0, "stopDescendantAgentTask: ancestorWorkspaceId required"); assert(taskId.length > 0, "stopDescendantAgentTask: taskId required"); - return await this.withTaskTreeLifecycleLock(taskId, () => - this.stopDescendantAgentTaskUnderLifecycleLock(ancestorWorkspaceId, taskId) - ); + return await this.withTaskTreeLifecycleLock(taskId, async () => { + const scopeResult = await this.validateDirectDescendantAgentTaskScope( + ancestorWorkspaceId, + taskId + ); + if (!scopeResult.success) { + return scopeResult; + } + const beforeStopError = await options?.beforeStop?.(); + if (beforeStopError != null) { + return Err(beforeStopError); + } + return await this.stopDescendantAgentTaskUnderLifecycleLock(ancestorWorkspaceId, taskId); + }); + } + + private async validateDirectDescendantAgentTaskScope( + ancestorWorkspaceId: string, + taskId: string + ): Promise> { + await using _lock = await this.mutex.acquire(); + const cfg = this.config.loadConfigOrDefault(); + const entry = findWorkspaceEntry(cfg, taskId); + if (!entry?.workspace.parentWorkspaceId) { + return Err("Task not found"); + } + const index = this.buildAgentTaskIndex(cfg); + if ( + !this.isDescendantAgentTaskUsingParentById(index.parentById, ancestorWorkspaceId, taskId) || + this.isWorkflowOwnedTaskUsingIndex(index, taskId) + ) { + return Err("Task is not a descendant of this workspace"); + } + return Ok(undefined); } private async stopDescendantAgentTaskUnderLifecycleLock( @@ -4682,12 +4751,20 @@ export class TaskService { } const index = this.buildAgentTaskIndex(cfg); if ( - !this.isDescendantAgentTaskUsingParentById(index.parentById, ancestorWorkspaceId, taskId) + !this.isDescendantAgentTaskUsingParentById(index.parentById, ancestorWorkspaceId, taskId) || + this.isWorkflowOwnedTaskUsingIndex(index, taskId) ) { return Err("Task is not a descendant of this workspace"); } - const taskIds = [taskId, ...this.listDescendantAgentTaskIdsFromIndex(index, taskId)]; + const taskIds = [ + taskId, + ...this.listDescendantAgentTaskIdsFromIndex(index, taskId).filter( + // Workflow-owned branches are stopped through WorkflowService so the durable run and + // its workers transition together; direct tree stopping must not mutate them behind it. + (descendantTaskId) => !this.isWorkflowOwnedTaskUsingIndex(index, descendantTaskId) + ), + ]; taskIds.sort( (left, right) => this.getTaskDepthFromParentById(index.parentById, right) - @@ -5139,7 +5216,11 @@ export class TaskService { */ async terminateAllDescendantAgentTasks( workspaceId: string, - options?: { workflowRunId?: string } + options?: { + workflowRunId?: string; + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; + deferWorkflowSweep?: boolean; + } ): Promise { assert( workspaceId.length > 0, @@ -5173,6 +5254,18 @@ export class TaskService { const interruptionError = new Error("Parent workspace interrupted"); for (const id of descendants) { + // Workflow workers can leave long-running bash processes behind even after their AI stream + // stops. Cleanup is best-effort: a disposal failure must not leave an interrupted workflow + // with agent streams and task statuses still active. + try { + await options?.cleanupWorkspaceBackgroundProcesses?.(id); + } catch (error: unknown) { + log.warn("terminateAllDescendantAgentTasks: background cleanup failed", { + taskId: id, + error: getErrorMessage(error), + }); + } + // Best-effort: clear queue first. AgentSession stream-end cleanup auto-flushes // queued messages, so descendants must not keep pending input after a hard interrupt. try { @@ -5245,7 +5338,7 @@ export class TaskService { await this.emitWorkspaceMetadata(taskId); } - if (options?.workflowRunId != null) { + if (options?.workflowRunId != null && options.deferWorkflowSweep !== true) { // Run-scoped interrupts arrive after the owning run's terminal status write // (WorkflowService.interruptRun aborts the runner, persists "interrupted", THEN // terminates descendants), so the children just interrupted above can be archived @@ -8313,7 +8406,7 @@ export class TaskService { async interruptWorkspaceTurn( ownerWorkspaceId: string, handleId: string - ): Promise> { + ): Promise> { let workspaceId: string | undefined; let shouldClearQueuedPrompt = false; let shouldStopStream = false; @@ -8324,8 +8417,12 @@ export class TaskService { if (record == null) { return Err("Workspace turn not found or out of scope"); } - if (record.status === "completed" || record.status === "error") { - return Err(`Workspace turn is already ${record.status} and cannot be interrupted.`); + if ( + record.status === "completed" || + record.status === "error" || + record.status === "interrupted" + ) { + return Ok({ workspaceId: record.workspaceId, alreadyInactive: true }); } workspaceId = record.workspaceId; @@ -8357,7 +8454,10 @@ export class TaskService { return Ok({ workspaceId: record.workspaceId }); }); - if (!result.success) { + if ( + !result.success || + ("alreadyInactive" in result.data && result.data.alreadyInactive === true) + ) { return result; } @@ -8429,7 +8529,7 @@ export class TaskService { async removeInactiveDescendantAgentTask( ownerWorkspaceId: string, taskId: string - ): Promise> { + ): Promise> { assert(ownerWorkspaceId.length > 0, "removeInactiveDescendantAgentTask requires owner"); assert(taskId.length > 0, "removeInactiveDescendantAgentTask requires taskId"); @@ -8448,7 +8548,10 @@ export class TaskService { } const index = this.buildAgentTaskIndex(config); - if (!this.isDescendantAgentTaskUsingParentById(index.parentById, ownerWorkspaceId, taskId)) { + if ( + !this.isDescendantAgentTaskUsingParentById(index.parentById, ownerWorkspaceId, taskId) || + this.isWorkflowOwnedTaskUsingIndex(index, taskId) + ) { return Ok({ status: "invalid_scope", action: "remove", taskId }); } diff --git a/src/node/services/tools/task.bash.test.ts b/src/node/services/tools/task.bash.test.ts index ca5a94a2ac6..46b79e4c9bc 100644 --- a/src/node/services/tools/task.bash.test.ts +++ b/src/node/services/tools/task.bash.test.ts @@ -306,8 +306,37 @@ describe("bash + task_* (background bash tasks)", () => { }); }); - it("task_terminate can terminate bash tasks", async () => { - using tempDir = new TestTempDir("test-task-terminate-bash"); + it("task_stop rejects bash tasks from workflow-owned descendants", async () => { + using tempDir = new TestTempDir("test-task-stop-bash-workflow-owned"); + + const getProcess = mock(() => ({ id: "workflow-proc", workspaceId: "workflow-task" })); + const terminate = mock(() => ({ success: true as const })); + const backgroundProcessManager = { + getProcess, + terminate, + } as unknown as BackgroundProcessManager; + const taskService = { + isDescendantAgentTask: mock(() => Promise.resolve(true)), + isWorkflowOwnedDescendantAgentTask: mock(() => Promise.resolve(true)), + } as unknown as TaskService; + const tool = createTaskStopTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "ws-1" }), + backgroundProcessManager, + taskService, + }); + + expect( + await Promise.resolve( + tool.execute!({ task_ids: ["bash:workflow-proc"] }, mockToolCallOptions) + ) + ).toEqual({ + results: [{ status: "invalid_scope", taskId: "bash:workflow-proc" }], + }); + expect(terminate).not.toHaveBeenCalled(); + }); + + it("task_stop can stop bash tasks", async () => { + using tempDir = new TestTempDir("test-task-stop-bash"); const getProcess = mock(() => ({ id: "proc-1", workspaceId: "ws-1" })); const terminate = mock(() => ({ success: true as const })); diff --git a/src/node/services/tools/task_await.test.ts b/src/node/services/tools/task_await.test.ts index 73ed99ab8fb..c52f1fd649b 100644 --- a/src/node/services/tools/task_await.test.ts +++ b/src/node/services/tools/task_await.test.ts @@ -933,7 +933,7 @@ describe("task_await tool", () => { expect(waitForAgentReport).toHaveBeenCalledTimes(1); }); - it("returns an error with descendant task suggestions for hallucinated IDs", async () => { + it("preserves not_found while suggesting active descendant task IDs", async () => { using tempDir = new TestTempDir("test-task-await-tool-hallucinated-descendant-suggestions"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "parent-workspace" }); @@ -954,27 +954,22 @@ describe("task_await tool", () => { const tool = createTaskAwaitTool({ ...baseConfig, taskService }); - const result = (await Promise.resolve( - tool.execute!({ task_ids: ["hallucinated"] }, mockToolCallOptions) - )) as { results: Array<{ status: string; taskId: string; error?: string }> }; - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - status: "error", - taskId: "hallucinated", + expect( + await Promise.resolve(tool.execute!({ task_ids: ["hallucinated"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "not_found", + taskId: "hallucinated", + activeTaskIds: ["real-child"], + }, + ], }); - const descendantSuggestionError = result.results[0]?.error; - expect(typeof descendantSuggestionError).toBe("string"); - if (typeof descendantSuggestionError !== "string") { - throw new Error("Expected hallucinated descendant result to include an error message"); - } - expect(descendantSuggestionError).toContain("same parallel tool-call batch"); - expect(descendantSuggestionError).toContain("real-child"); expect(getAgentTaskStatuses).toHaveBeenCalledTimes(1); expect(waitForAgentReport).toHaveBeenCalledTimes(0); }); - it("returns an error with bash task suggestions for out-of-scope IDs", async () => { + it("preserves invalid_scope while suggesting active bash task IDs", async () => { using tempDir = new TestTempDir("test-task-await-tool-hallucinated-bash-suggestions"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "parent-workspace" }); @@ -1010,22 +1005,17 @@ describe("task_await tool", () => { taskService, }); - const result = (await Promise.resolve( - tool.execute!({ task_ids: ["other-workspace"] }, mockToolCallOptions) - )) as { results: Array<{ status: string; taskId: string; error?: string }> }; - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - status: "error", - taskId: "other-workspace", + expect( + await Promise.resolve(tool.execute!({ task_ids: ["other-workspace"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "invalid_scope", + taskId: "other-workspace", + activeTaskIds: ["bash:proc-1"], + }, + ], }); - const bashSuggestionError = result.results[0]?.error; - expect(typeof bashSuggestionError).toBe("string"); - if (typeof bashSuggestionError !== "string") { - throw new Error("Expected out-of-scope bash suggestion result to include an error message"); - } - expect(bashSuggestionError).toContain("same parallel tool-call batch"); - expect(bashSuggestionError).toContain("bash:proc-1"); expect(getAgentTaskStatuses).toHaveBeenCalledTimes(1); expect(waitForAgentReport).toHaveBeenCalledTimes(0); }); @@ -1062,6 +1052,7 @@ describe("task_await tool", () => { reportMarkdown?: string; title?: string; note?: string; + activeTaskIds?: string[]; }>; }; @@ -1073,16 +1064,11 @@ describe("task_await tool", () => { title: undefined, note: COMPLETED_REPORT_REFETCH_NOTE, }); - expect(result.results[1]).toMatchObject({ - status: "error", + expect(result.results[1]).toEqual({ + status: "not_found", taskId: "hallucinated", + activeTaskIds: ["real-child"], }); - const mixedResultError = result.results[1]?.error; - expect(typeof mixedResultError).toBe("string"); - if (typeof mixedResultError !== "string") { - throw new Error("Expected mixed-result hallucinated task to include an error message"); - } - expect(mixedResultError).toContain("real-child"); expect(waitForAgentReport).toHaveBeenCalledTimes(1); expect(waitForAgentReport).toHaveBeenCalledWith("real-child", expect.any(Object)); expect(getAgentTaskStatuses).toHaveBeenCalledTimes(1); diff --git a/src/node/services/tools/task_await.ts b/src/node/services/tools/task_await.ts index 7cb92785033..5bd1a24fd89 100644 --- a/src/node/services/tools/task_await.ts +++ b/src/node/services/tools/task_await.ts @@ -93,17 +93,6 @@ function withElapsedMs(elapsedMs: number | undefined): { elapsed_ms?: number } { return elapsedMs == null ? {} : { elapsed_ms: elapsedMs }; } -function buildTaskAwaitSequencingError(taskId: string, suggestedTaskIds: string[]) { - return { - status: "error" as const, - taskId, - error: - "Do not call task_await in the same parallel tool-call batch as task or bash. " + - "Wait for the spawning tool result first, then call task_await in a later step. " + - `Use one of these returned task IDs instead: ${suggestedTaskIds.join(", ")}.`, - }; -} - function parseWorkflowRun(value: unknown): WorkflowRunRecord { return WorkflowRunRecordSchema.parse(value); } @@ -770,19 +759,15 @@ export const createTaskAwaitTool: ToolFactory = (config: ToolConfiguration) => { if (!descendantAgentTaskIdSet.has(taskId)) { const lookup = rejectedAgentTaskStatuses.get(taskId); - const activeTaskIds = - activeDescendantAgentTaskIds.length > 0 ? activeDescendantAgentTaskIds : undefined; - if (requestedIds) { - const suggestedTaskIds = dedupeStrings([ - ...activeDescendantAgentTaskIds, - ...(await listInScopeWorkspaceTurnTaskIds().catch(() => [])), - ...(await getSuggestionBashTaskIds()), - ...(await getSuggestionWorkflowRunIds()), - ]); - if (suggestedTaskIds.length > 0) { - return buildTaskAwaitSequencingError(taskId, suggestedTaskIds); - } - } + const suggestedTaskIds = requestedIds + ? dedupeStrings([ + ...activeDescendantAgentTaskIds, + ...(await listInScopeWorkspaceTurnTaskIds().catch(() => [])), + ...(await getSuggestionBashTaskIds()), + ...(await getSuggestionWorkflowRunIds()), + ]) + : activeDescendantAgentTaskIds; + const activeTaskIds = suggestedTaskIds.length > 0 ? suggestedTaskIds : undefined; if (!lookup?.exists) { return { status: "not_found" as const, taskId, activeTaskIds }; } diff --git a/src/node/services/tools/task_remove.ts b/src/node/services/tools/task_remove.ts index 0e53f77dbc8..c2de5ccf3a4 100644 --- a/src/node/services/tools/task_remove.ts +++ b/src/node/services/tools/task_remove.ts @@ -43,7 +43,6 @@ export const createTaskRemoveTool: ToolFactory = (config: ToolConfiguration) => case "removed": case "already_removed": case "active": - case "not_found": case "invalid_scope": results.push({ status: data.status, diff --git a/src/node/services/tools/task_stop.test.ts b/src/node/services/tools/task_stop.test.ts index eff272effe2..6598d2ea4ac 100644 --- a/src/node/services/tools/task_stop.test.ts +++ b/src/node/services/tools/task_stop.test.ts @@ -41,15 +41,23 @@ describe("task_stop tool", () => { using tempDir = new TestTempDir("test-task-terminate-invalid-scope"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const listRuns = mock(() => Promise.resolve([])); + const stopDescendantAgentTask = mock( + (): Promise> => + Promise.resolve(Err("Task is not a descendant of this workspace")) + ); const taskService = { listActiveDescendantAgentTaskIds: mock(() => ["child-task"]), - stopDescendantAgentTask: mock( - (): Promise> => - Promise.resolve(Err("Task is not a descendant of this workspace")) - ), + listDescendantAgentTasks: mock(() => []), + isDescendantAgentTask: mock(() => Promise.resolve(false)), + stopDescendantAgentTask, } as unknown as TaskService; - const tool = createTaskStopTool({ ...baseConfig, taskService }); + const tool = createTaskStopTool({ + ...baseConfig, + taskService, + workflowService: { listRuns }, + }); const result: unknown = await Promise.resolve( tool.execute!({ task_ids: ["other-task"] }, mockToolCallOptions) @@ -58,6 +66,8 @@ describe("task_stop tool", () => { expect(result).toEqual({ results: [{ status: "invalid_scope", taskId: "other-task" }], }); + expect(listRuns).not.toHaveBeenCalled(); + expect(stopDescendantAgentTask).toHaveBeenCalledTimes(1); }); it("reports aggregated cleanup failures as error, not invalid_scope", async () => { @@ -119,6 +129,7 @@ describe("task_stop tool", () => { const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); const controller = new AbortController(); + const finished = Promise.withResolvers(); const taskService = { stopDescendantAgentTask: mock( ( @@ -128,6 +139,7 @@ describe("task_stop tool", () => { if (taskId === "stuck-task") { return new Promise(() => undefined); } + finished.resolve(); return Promise.resolve(Ok({ stoppedTaskIds: [taskId] })); } ), @@ -140,8 +152,11 @@ describe("task_stop tool", () => { { ...mockToolCallOptions, abortSignal: controller.signal } ) ); - await Promise.resolve(); - await Promise.resolve(); + await finished.promise; + // Let the completed branch propagate through the tool's abort race before aborting the stuck one. + for (let i = 0; i < 10; i += 1) { + await Promise.resolve(); + } controller.abort(); expect(await resultPromise).toEqual({ @@ -193,6 +208,25 @@ describe("task_stop tool", () => { }); }); + it("treats stopping a terminal workspace turn as idempotent", async () => { + using tempDir = new TestTempDir("test-task-stop-terminal-workspace-turn"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const interruptWorkspaceTurn = mock( + (): Promise> => + Promise.resolve(Ok({ workspaceId: "child-workspace", alreadyInactive: true })) + ); + const tool = createTaskStopTool({ + ...baseConfig, + taskService: { interruptWorkspaceTurn } as unknown as TaskService, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wst_turn"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "already_inactive", taskId: "wst_turn" }], + }); + }); + const buildWorkflowRun = (status: string) => ({ id: "wfr_run_1", workspaceId: "root-workspace", @@ -217,7 +251,10 @@ describe("task_stop tool", () => { const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); const getRun = mock(() => Promise.resolve(buildWorkflowRun("running"))); - const interruptRun = mock(() => Promise.resolve(buildWorkflowRun("interrupted"))); + const interruptRun = mock((input: { onRunInterrupted?: (runId: string) => void }) => { + input.onRunInterrupted?.("wfr_run_1"); + return Promise.resolve(buildWorkflowRun("interrupted")); + }); const taskService = { stopDescendantAgentTask: mock(() => { throw new Error("workflow IDs must not reach agent task termination"); @@ -241,6 +278,7 @@ describe("task_stop tool", () => { expect(interruptRun).toHaveBeenCalledWith({ workspaceId: "root-workspace", runId: "wfr_run_1", + onRunInterrupted: expect.any(Function), }); expect(result).toEqual({ results: [ @@ -253,6 +291,70 @@ describe("task_stop tool", () => { }); }); + it("interrupts workflow runs owned by an agent subtree before stopping user-owned tasks", async () => { + using tempDir = new TestTempDir("test-task-stop-agent-workflows-first"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const events: string[] = []; + const workflowRun = { ...buildWorkflowRun("running"), workspaceId: "child-task" }; + const taskService = { + listDescendantAgentTasks: mock( + (_taskId: string, options?: { excludeWorkflowTasks?: boolean }) => + options?.excludeWorkflowTasks === true + ? [] + : [{ taskId: "workflow-worker", status: "running" as const }] + ), + isDescendantAgentTask: mock(() => Promise.resolve(true)), + isWorkflowOwnedDescendantAgentTask: mock(() => Promise.resolve(false)), + stopDescendantAgentTask: mock( + async ( + _workspaceId: string, + _taskId: string, + options?: { beforeStop?: () => Promise } + ): Promise> => { + const beforeStopError = await options?.beforeStop?.(); + if (beforeStopError != null) { + return Err(beforeStopError); + } + events.push("task"); + return Ok({ stoppedTaskIds: ["child-task"] }); + } + ), + markWorkflowRunEnded: mock((workflowRunId: string) => { + events.push(`sweep:${workflowRunId}`); + return Promise.resolve(); + }), + } as unknown as TaskService; + const tool = createTaskStopTool({ + ...baseConfig, + taskService, + workflowService: { + listRuns: mock(() => Promise.resolve([workflowRun])), + getRun: mock(() => Promise.resolve(workflowRun)), + interruptRun: mock( + (input: { + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + onRunInterrupted?: (runId: string) => void; + }) => { + expect(input.deferTaskSweep).toBe(true); + expect(input.lockAlreadyHeld).toBe(true); + input.onRunInterrupted?.("wfr_run_1"); + input.onRunInterrupted?.("wfr_nested"); + events.push("workflow"); + return Promise.resolve({ ...workflowRun, status: "interrupted" }); + } + ), + }, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["child-task"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "stopped", taskId: "child-task", stoppedTaskIds: ["child-task"] }], + }); + expect(events).toEqual(["workflow", "task", "sweep:wfr_run_1", "sweep:wfr_nested"]); + }); + it("does not start termination when the signal is already aborted", async () => { using tempDir = new TestTempDir("test-task-terminate-preaborted"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); @@ -313,7 +415,13 @@ describe("task_stop tool", () => { using tempDir = new TestTempDir("test-task-terminate-workflow-idempotent"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); - const interruptRun = mock(() => Promise.reject(new Error("must not re-interrupt"))); + const interruptRun = mock( + (input: { retryTaskCleanup?: boolean; onRunInterrupted?: (runId: string) => void }) => { + expect(input.retryTaskCleanup).toBe(true); + input.onRunInterrupted?.("wfr_run_1"); + return Promise.resolve(buildWorkflowRun("interrupted")); + } + ); const tool = createTaskStopTool({ ...baseConfig, taskService: {} as unknown as TaskService, @@ -327,7 +435,7 @@ describe("task_stop tool", () => { tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions) ); - expect(interruptRun).not.toHaveBeenCalled(); + expect(interruptRun).toHaveBeenCalledTimes(1); expect(result).toEqual({ results: [ { @@ -338,8 +446,8 @@ describe("task_stop tool", () => { }); }); - it("rejects interrupting terminal workflow runs", async () => { - using tempDir = new TestTempDir("test-task-terminate-workflow-terminal"); + it("treats stopping terminal workflow runs as idempotent", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-terminal"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); const interruptRun = mock(() => Promise.reject(new Error("must not interrupt terminal runs"))); @@ -358,14 +466,101 @@ describe("task_stop tool", () => { expect(interruptRun).not.toHaveBeenCalled(); expect(result).toEqual({ + results: [{ status: "already_inactive", taskId: "wfr_run_1" }], + }); + }); + + it("treats a workflow that settles during interruption as already inactive", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-settlement-race"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const runs = [buildWorkflowRun("running"), buildWorkflowRun("completed")]; + const tool = createTaskStopTool({ + ...baseConfig, + taskService: {} as unknown as TaskService, + workflowService: { + getRun: mock(() => Promise.resolve(runs.shift() ?? buildWorkflowRun("completed"))), + interruptRun: mock(() => Promise.reject(new Error("invalid workflow transition"))), + }, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "already_inactive", taskId: "wfr_run_1" }], + }); + }); + + it("surfaces cleanup failure after this interruption persisted terminal state", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-post-persist-failure"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const getRun = mock(() => Promise.resolve(buildWorkflowRun("running"))); + const tool = createTaskStopTool({ + ...baseConfig, + taskService: {} as unknown as TaskService, + workflowService: { + getRun, + interruptRun: mock( + (input: { onRunInterrupted?: (runId: string) => void }): Promise => { + input.onRunInterrupted?.("wfr_run_1"); + return Promise.reject(new Error("workflow worker cleanup failed")); + } + ), + }, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "error", + taskId: "wfr_run_1", + error: "workflow worker cleanup failed", + }, + ], + }); + expect(getRun).toHaveBeenCalledTimes(1); + }); + + it("retries workflow task cleanup after a post-persistence failure", async () => { + using tempDir = new TestTempDir("test-task-stop-workflow-cleanup-retry"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const runs = [buildWorkflowRun("running"), buildWorkflowRun("interrupted")]; + let interruptCalls = 0; + const tool = createTaskStopTool({ + ...baseConfig, + taskService: {} as unknown as TaskService, + workflowService: { + getRun: mock(() => Promise.resolve(runs.shift() ?? buildWorkflowRun("interrupted"))), + interruptRun: mock( + (input: { retryTaskCleanup?: boolean; onRunInterrupted?: (runId: string) => void }) => { + interruptCalls += 1; + input.onRunInterrupted?.("wfr_run_1"); + return interruptCalls === 1 + ? Promise.reject(new Error("workflow worker cleanup failed")) + : Promise.resolve(buildWorkflowRun("interrupted")); + } + ), + }, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ results: [ { status: "error", taskId: "wfr_run_1", - error: expect.stringContaining("already completed"), + error: "workflow worker cleanup failed", }, ], }); + expect( + await Promise.resolve(tool.execute!({ task_ids: ["wfr_run_1"] }, mockToolCallOptions)) + ).toEqual({ + results: [{ status: "already_inactive", taskId: "wfr_run_1" }], + }); + expect(interruptCalls).toBe(2); }); it("reports workflow runs outside this workspace as not found", async () => { diff --git a/src/node/services/tools/task_stop.ts b/src/node/services/tools/task_stop.ts index c819c68a56d..a3546982701 100644 --- a/src/node/services/tools/task_stop.ts +++ b/src/node/services/tools/task_stop.ts @@ -3,10 +3,16 @@ import { tool } from "ai"; import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; +import { + isActiveWorkflowRunStatus, + isNestedWorkflowRun, + isTerminalWorkflowRunStatus, +} from "@/common/types/workflow"; import { TaskStopToolResultSchema, TOOL_DEFINITIONS } from "@/common/utils/tools/toolDefinitions"; import { TASK_TERMINATION_TOOL_TIMEOUT_MS } from "@/constants/terminationTimeouts"; import { log } from "@/node/services/log"; +import type { TaskService } from "@/node/services/taskService"; import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { isWorkspaceTurnTaskId } from "@/node/services/taskHandleStore"; import { fromBashTaskId, isWorkflowRunTaskId } from "./taskId"; @@ -28,7 +34,12 @@ const WORKFLOW_STOPPED_NOTE = async function interruptWorkflowRun( config: ToolConfiguration, workspaceId: string, - taskId: string + taskId: string, + options?: { + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + onRunInterrupted?: (runId: string) => void; + } ) { const workflowService = config.workflowService; if (workflowService?.getRun == null || workflowService.interruptRun == null) { @@ -56,26 +67,142 @@ async function interruptWorkflowRun( } const run = parsedRun.data; - if (run.status === "interrupted") { - // Idempotent: re-interrupting an interrupted run is a no-op success. + if (run.status === "completed" || run.status === "failed") { return { status: "already_inactive" as const, taskId }; } - if (run.status === "completed" || run.status === "failed") { - return { - status: "error" as const, - taskId, - error: `Workflow run is already ${run.status} and cannot be interrupted.`, - }; + + if (run.status === "interrupted") { + try { + await workflowService.interruptRun({ + workspaceId, + runId: taskId, + retryTaskCleanup: true, + ...(options?.deferTaskSweep === true ? { deferTaskSweep: true } : {}), + ...(options?.lockAlreadyHeld === true ? { lockAlreadyHeld: true } : {}), + onRunInterrupted: options?.onRunInterrupted, + }); + return { status: "already_inactive" as const, taskId }; + } catch (error: unknown) { + return { status: "error" as const, taskId, error: getErrorMessage(error) }; + } } + let persistedByThisCall = false; try { - await workflowService.interruptRun({ workspaceId, runId: taskId }); + await workflowService.interruptRun({ + workspaceId, + runId: taskId, + ...(options?.deferTaskSweep === true ? { deferTaskSweep: true } : {}), + ...(options?.lockAlreadyHeld === true ? { lockAlreadyHeld: true } : {}), + onRunInterrupted: (runId) => { + persistedByThisCall ||= runId === taskId; + options?.onRunInterrupted?.(runId); + }, + }); } catch (error: unknown) { + // A terminal re-read means idempotent success only when another actor won the transition race. + // If this invocation persisted interrupted and cleanup then failed, surface that failure so the + // caller does not report success while workflow-owned workers may still be active. + if (!persistedByThisCall) { + const latestRawRun = await workflowService + .getRun({ workspaceId, runId: taskId }) + .catch(() => null); + const latestRun = WorkflowRunRecordSchema.safeParse(latestRawRun); + if (latestRun.success && isTerminalWorkflowRunStatus(latestRun.data.status)) { + return { status: "already_inactive" as const, taskId }; + } + } return { status: "error" as const, taskId, error: getErrorMessage(error) }; } return { status: "stopped" as const, taskId, note: WORKFLOW_STOPPED_NOTE }; } +async function interruptWorkflowRunsOwnedByAgentTaskTree( + config: ToolConfiguration, + taskService: TaskService, + taskId: string, + deferredWorkflowRunIds: string[] +): Promise { + const listDescendants = taskService.listDescendantAgentTasks?.bind(taskService); + if (listDescendants == null) { + return null; + } + + const descendants = listDescendants(taskId); + const userOwnedDescendants = listDescendants(taskId, { excludeWorkflowTasks: true }); + const userOwnedTaskIds = new Set(userOwnedDescendants.map((task) => task.taskId)); + const activeWorkflowOwnedDescendants = descendants.filter( + (task) => + !userOwnedTaskIds.has(task.taskId) && + (task.status === "queued" || + task.status === "starting" || + task.status === "running" || + task.status === "awaiting_report") + ); + + const workflowService = config.workflowService; + if (workflowService?.listRuns == null || workflowService.interruptRun == null) { + return activeWorkflowOwnedDescendants.length > 0 + ? "Workflow service not available to stop workflow-owned descendants" + : null; + } + + let activeRunCount = 0; + for (const ownerWorkspaceId of [taskId, ...userOwnedTaskIds]) { + const rawRuns = await workflowService.listRuns({ workspaceId: ownerWorkspaceId }); + for (const rawRun of rawRuns) { + const parsedRun = WorkflowRunRecordSchema.safeParse(rawRun); + if ( + !parsedRun.success || + (!isActiveWorkflowRunStatus(parsedRun.data.status) && + parsedRun.data.status !== "interrupted") || + isNestedWorkflowRun(parsedRun.data) + ) { + continue; + } + + activeRunCount += 1; + const outcome = await interruptWorkflowRun(config, ownerWorkspaceId, parsedRun.data.id, { + deferTaskSweep: true, + lockAlreadyHeld: true, + onRunInterrupted: (runId) => deferredWorkflowRunIds.push(runId), + }); + if (outcome.status === "stopped") { + deferredWorkflowRunIds.push(parsedRun.data.id); + } + if (outcome.status === "error") { + return outcome.error; + } + if (outcome.status === "not_found") { + return `Workflow run ${parsedRun.data.id} disappeared before it could be stopped`; + } + } + } + + if (activeWorkflowOwnedDescendants.length > 0 && activeRunCount === 0) { + return "Active workflow-owned descendants have no active owning workflow run"; + } + return null; +} + +async function sweepDeferredWorkflowRuns( + taskService: TaskService, + taskId: string, + workflowRunIds: string[] +): Promise { + for (const workflowRunId of dedupeStrings(workflowRunIds)) { + try { + await taskService.markWorkflowRunEnded(workflowRunId); + } catch (error: unknown) { + log.warn("task_stop deferred workflow sweep failed", { + taskId, + workflowRunId, + error: getErrorMessage(error), + }); + } + } +} + export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { return tool({ description: TOOL_DEFINITIONS.task_stop.description, @@ -114,6 +241,9 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { } return { status: "error" as const, taskId, error: msg }; } + if (interruptResult.data.alreadyInactive === true) { + return { status: "already_inactive" as const, taskId }; + } return { status: "stopped" as const, taskId, @@ -143,7 +273,14 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { const inScope = proc.workspaceId === workspaceId || (await taskService.isDescendantAgentTask(workspaceId, proc.workspaceId)); - if (!inScope) { + const workflowOwned = + proc.workspaceId !== workspaceId && + ((await taskService.isWorkflowOwnedDescendantAgentTask?.( + workspaceId, + proc.workspaceId + )) ?? + false); + if (!inScope || workflowOwned) { return { status: "invalid_scope" as const, taskId }; } @@ -160,7 +297,24 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { }; } - const stopResult = await taskService.stopDescendantAgentTask(workspaceId, taskId); + const deferredWorkflowRunIds: string[] = []; + const stopResult = await taskService.stopDescendantAgentTask(workspaceId, taskId, { + // Run workflow interruption while TaskService holds the task-tree lifecycle lock. + // Workflow worker creation uses the same lock, so no new workflow-owned branch can + // appear between discovery and the direct user-owned subtree stop. Archive sweeps + // are deferred because WorkspaceService.archive reacquires this non-reentrant lock. + beforeStop: + taskService.listDescendantAgentTasks != null + ? async () => + await interruptWorkflowRunsOwnedByAgentTaskTree( + config, + taskService, + taskId, + deferredWorkflowRunIds + ) + : undefined, + }); + await sweepDeferredWorkflowRuns(taskService, taskId, deferredWorkflowRunIds); if (!stopResult.success) { const msg = stopResult.error; // Exact-match the canonical scope errors: aggregated cleanup failures @@ -195,7 +349,7 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { } void terminationPromise.catch((error: unknown) => { - log.debug("task_terminate cleanup failed after tool returned", { taskId, error }); + log.debug("task_stop cleanup failed after tool returned", { taskId, error }); }); return { status: "error" as const, diff --git a/src/node/services/workflows/WorkflowRunner.ts b/src/node/services/workflows/WorkflowRunner.ts index 291c3efbc87..3771e1debcd 100644 --- a/src/node/services/workflows/WorkflowRunner.ts +++ b/src/node/services/workflows/WorkflowRunner.ts @@ -197,7 +197,7 @@ export interface WorkflowTaskAdapter { spec: WorkflowApplyPatchSpec, options?: { abortSignal?: AbortSignal } ): Promise; - interruptRun?(): Promise; + interruptRun?(options?: { deferTaskSweep?: boolean }): Promise; /** * Called when the run reaches a terminal state. Not called when the run is * backgrounded (the background continuation re-enters run()) or when the @@ -208,6 +208,9 @@ export interface WorkflowTaskAdapter { export interface WorkflowRunnerRunOptions { onLeaseAcquired?: () => void; + shouldDeferRunEnded?: () => boolean; + onRunEndedDeferred?: () => void; + onRunningStatusPersisted?: () => void; abortSignal?: AbortSignal; backgroundOnMessageQueued?: boolean; allowResumeFromInterrupted?: boolean; @@ -401,7 +404,11 @@ export class WorkflowRunner { assert(runId.length > 0, "WorkflowRunner.run: runId is required"); try { const result = await this.runWithLease(runId, options); - await this.taskAdapter.onRunEnded?.(); + if (options?.shouldDeferRunEnded?.() === true) { + options.onRunEndedDeferred?.(); + } else { + await this.taskAdapter.onRunEnded?.(); + } return result; } catch (error) { // Backgrounding is not terminal (the background continuation re-enters @@ -411,7 +418,11 @@ export class WorkflowRunner { error instanceof WorkflowRunBackgroundedError || (error instanceof Error && error.message === `Workflow run is already active: ${runId}`); if (!keepsHold) { - await this.taskAdapter.onRunEnded?.(); + if (options?.shouldDeferRunEnded?.() === true) { + options.onRunEndedDeferred?.(); + } else { + await this.taskAdapter.onRunEnded?.(); + } } throw error; } @@ -525,6 +536,8 @@ export class WorkflowRunner { } ); + options?.onRunningStatusPersisted?.(); + let runtime: IJSRuntime | undefined; try { runtime = await this.runtimeFactory.create(); diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index a573b42a7c8..4396d7a98e9 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -2,7 +2,7 @@ import * as crypto from "node:crypto"; import * as path from "node:path"; -import { describe, expect, test } from "bun:test"; +import { describe, expect, mock, test } from "bun:test"; import assert from "@/common/utils/assert"; import { WORKFLOW_CHECKPOINT_RETRY_ERROR_MESSAGE } from "@/common/utils/workflowRetryEligibility"; import { ForegroundWaitBackgroundedError } from "@/node/services/taskService"; @@ -28,6 +28,75 @@ function createScript( } describe("WorkflowService", () => { + test("does not create a durable run when the workspace lifecycle lock rejects startup", async () => { + using tmp = new DisposableTempDir("workflow-service-run-creation-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + const withRunStartLock = mock(async () => { + throw new Error("Cannot start workflow work after task_stop"); + }); + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + withRunStartLock, + generateRunId: () => "wfr_rejected_start", + runnerId: "runner-a", + }); + + await expect( + service.startWorkflow({ + script: createScript("export default function workflow() { return 'unused'; }"), + workspaceId: "workspace-1", + projectTrusted: true, + args: {}, + }) + ).rejects.toThrow("Cannot start workflow work after task_stop"); + expect(withRunStartLock).toHaveBeenCalledTimes(1); + expect(await runStore.listRunStatusSnapshots()).toEqual([]); + }); + + test("does not resume a durable run when the workspace lifecycle lock rejects startup", async () => { + using tmp = new DisposableTempDir("workflow-service-resume-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_rejected_resume", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "project", executable: true }, + source: "export default function workflow() { return {}; }\n", + args: {}, + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_rejected_resume", "interrupted", "2026-05-29T00:00:01.000Z"); + const withRunStartLock = mock(async () => { + throw new Error("Cannot start workflow work after task_stop"); + }); + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + withRunStartLock, + runnerId: "runner-a", + }); + + await expect( + service.resumeRunInBackground({ + workspaceId: "workspace-1", + runId: "wfr_rejected_resume", + projectTrusted: true, + }) + ).rejects.toThrow("Cannot start workflow work after task_stop"); + expect(withRunStartLock).toHaveBeenCalledTimes(1); + expect((await runStore.getRun("wfr_rejected_resume")).status).toBe("interrupted"); + }); + test("starts an explicit script workflow and persists the resolved source snapshot", async () => { using tmp = new DisposableTempDir("workflow-service-script-path"); const source = `export default function workflow({ args }) { @@ -409,6 +478,109 @@ export default function workflow() { return { reportMarkdown: "done" }; } }); }); + test("handles foreground resume aborts while the lifecycle lock is still held", async () => { + using tmp = new DisposableTempDir("workflow-service-resume-abort-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_resume_abort_lock", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "project", executable: true }, + source: "export default function workflow() { return {}; }\n", + args: {}, + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_resume_abort_lock", "interrupted", "2026-05-29T00:00:01.000Z"); + const abortController = new AbortController(); + const events: string[] = []; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapterFactory: (runId) => ({ + async runAgent() { + throw new Error("Runner must not start after the resume is aborted"); + }, + interruptRun() { + events.push(`cleanup:${runId}`); + return Promise.resolve(); + }, + onRunEnded() { + events.push(`sweep:${runId}`); + return Promise.resolve(); + }, + }), + withRunStartLock: async (_workspaceId, operation) => { + events.push("lock:start"); + const result = await operation(); + events.push("lock:end"); + return result; + }, + onRunStatusChanged: (event) => { + if (event.runId === "wfr_resume_abort_lock" && event.status === "running") { + abortController.abort(); + } + }, + runnerId: "runner-a", + }); + + await expect( + service.resumeRun({ + workspaceId: "workspace-1", + runId: "wfr_resume_abort_lock", + projectTrusted: true, + abortSignal: abortController.signal, + }) + ).rejects.toThrow("Workflow run interrupted"); + + expect(events).toEqual([ + "lock:start", + "cleanup:wfr_resume_abort_lock", + "lock:end", + "sweep:wfr_resume_abort_lock", + ]); + expect((await runStore.getRun("wfr_resume_abort_lock")).status).toBe("interrupted"); + }); + + test("ignores a terminal race during abort cleanup retry", async () => { + using tmp = new DisposableTempDir("workflow-service-abort-cleanup-terminal-race"); + const service = new WorkflowService({ + runStore: new WorkflowRunStore({ sessionDir: tmp.path }), + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No runner expected"); + }, + }, + runnerId: "runner-a", + }); + const interruptRun = mock(() => Promise.reject(new Error("workflow cleanup transition raced"))); + const getRun = mock() + .mockResolvedValueOnce({ status: "interrupted" }) + .mockResolvedValueOnce({ status: "completed" }); + service.interruptRun = interruptRun; + service.getRun = getRun; + const abortController = new AbortController(); + const internal = service as unknown as { + interruptRunOnAbort: ( + workspaceId: string, + runId: string, + abortSignal: AbortSignal, + runnerAbortController: AbortController | undefined + ) => { remove: () => void; wait: () => Promise }; + }; + const abortInterrupt = internal.interruptRunOnAbort( + "workspace-1", + "wfr_abort_race", + abortController.signal, + undefined + ); + + abortController.abort(); + await abortInterrupt.wait(); + + expect(interruptRun).toHaveBeenCalledTimes(2); + expect(getRun).toHaveBeenCalledTimes(2); + }); + test("does not continue canceled foreground workflows in the background", async () => { using tmp = new DisposableTempDir("workflow-service-canceled-background"); const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); @@ -546,6 +718,54 @@ export default function workflow({ args }) { ).toBe(true); }); + test("serializes workflow interruption and defers task sweeps until lock release", async () => { + using tmp = new DisposableTempDir("workflow-service-interrupt-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_interrupt_lock", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "project", executable: true }, + source: "export default function workflow() { return {}; }\n", + args: {}, + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_interrupt_lock", "running", "2026-05-29T00:00:01.000Z"); + const events: string[] = []; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapterFactory: (runId) => ({ + async runAgent() { + throw new Error("No agent steps expected"); + }, + interruptRun() { + events.push(`cleanup:${runId}`); + return Promise.resolve(); + }, + onRunEnded() { + events.push(`sweep:${runId}`); + return Promise.resolve(); + }, + }), + withRunStartLock: async (_workspaceId, operation) => { + events.push("lock:start"); + const result = await operation(); + events.push("lock:end"); + return result; + }, + runnerId: "runner-a", + }); + + await service.interruptRun({ workspaceId: "workspace-1", runId: "wfr_interrupt_lock" }); + + expect(events).toEqual([ + "lock:start", + "cleanup:wfr_interrupt_lock", + "lock:end", + "sweep:wfr_interrupt_lock", + ]); + }); + test("interrupts active child workflow runs with the parent", async () => { using tmp = new DisposableTempDir("workflow-service-interrupt-nested-run"); const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); @@ -599,7 +819,13 @@ export default function workflow({ args }) { }, }); - await service.interruptRun({ workspaceId: "workspace-1", runId: "wfr_parent_interrupt" }); + const deferredSweepRunIds: string[] = []; + await service.interruptRun({ + workspaceId: "workspace-1", + runId: "wfr_parent_interrupt", + deferTaskSweep: true, + onRunInterrupted: (runId) => deferredSweepRunIds.push(runId), + }); await expect(runStore.getRun("wfr_parent_interrupt")).resolves.toMatchObject({ status: "interrupted", @@ -607,7 +833,22 @@ export default function workflow({ args }) { await expect(runStore.getRun("wfr_child_interrupt")).resolves.toMatchObject({ status: "interrupted", }); - expect(interruptedRunIds).toEqual(["wfr_parent_interrupt", "wfr_child_interrupt"]); + expect(deferredSweepRunIds).toEqual(["wfr_parent_interrupt", "wfr_child_interrupt"]); + const retriedSweepRunIds: string[] = []; + await service.interruptRun({ + workspaceId: "workspace-1", + runId: "wfr_parent_interrupt", + retryTaskCleanup: true, + deferTaskSweep: true, + onRunInterrupted: (runId) => retriedSweepRunIds.push(runId), + }); + expect(retriedSweepRunIds).toEqual(["wfr_parent_interrupt", "wfr_child_interrupt"]); + expect(interruptedRunIds).toEqual([ + "wfr_parent_interrupt", + "wfr_child_interrupt", + "wfr_parent_interrupt", + "wfr_child_interrupt", + ]); }); test("listRuns only loads root runs for the requested workspace", async () => { diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index adf3c26c5de..8ca3ab50733 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -55,6 +55,8 @@ export interface WorkflowServiceOptions { generateRunId?: () => string; // Delayed crash-recovery retries must use current trust, not the value captured when scheduled. getCurrentProjectTrusted?: () => boolean | Promise; + /** Serialize durable run creation with workspace/task lifecycle transitions. */ + withRunStartLock?: (workspaceId: string, operation: () => Promise) => Promise; /** Stable prefix; WorkflowService appends run identity and a nonce for each lease owner. */ runnerId: string; clock?: WorkflowRunnerClock; @@ -129,6 +131,10 @@ export class WorkflowService { private readonly notifyInterruptedBackgroundRunTerminal: boolean; private readonly generateRunId: () => string; private readonly getCurrentProjectTrusted?: () => boolean | Promise; + private readonly withRunStartLock?: ( + workspaceId: string, + operation: () => Promise + ) => Promise; private readonly runnerId: string; private readonly clock?: WorkflowRunnerClock; @@ -151,6 +157,7 @@ export class WorkflowService { options.notifyInterruptedBackgroundRunTerminal === true; this.generateRunId = options.generateRunId ?? generateWorkflowRunId; this.getCurrentProjectTrusted = options.getCurrentProjectTrusted; + this.withRunStartLock = options.withRunStartLock; this.runnerId = options.runnerId; this.clock = options.clock; } @@ -242,12 +249,73 @@ export class WorkflowService { } } - async interruptRun(input: { workspaceId: string; runId: string }): Promise { - return await this.interruptRunTree(input, new Set(), false); + async interruptRun(input: { + workspaceId: string; + runId: string; + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }): Promise { + const interruptedRunIds: string[] = []; + const interrupt = async () => + await this.interruptRunTree( + { + workspaceId: input.workspaceId, + runId: input.runId, + retryTaskCleanup: input.retryTaskCleanup, + onRunInterrupted: (runId) => { + interruptedRunIds.push(runId); + input.onRunInterrupted?.(runId); + }, + }, + new Set(), + false + ); + + let result: WorkflowRunRecord | undefined; + let operationError: unknown; + try { + result = + input.lockAlreadyHeld === true + ? await interrupt() + : await this.withWorkflowRunStartLock(input.workspaceId, interrupt); + } catch (error: unknown) { + operationError = error; + } + + if (input.deferTaskSweep !== true) { + await this.sweepInterruptedRunTasks(interruptedRunIds); + } + if (operationError != null) { + throw operationError instanceof Error + ? operationError + : new Error(getErrorMessage(operationError)); + } + assert(result != null, "WorkflowService.interruptRun: result is required"); + return result; + } + + private async sweepInterruptedRunTasks(runIds: readonly string[]): Promise { + for (const runId of new Set(runIds)) { + try { + await (this.taskAdapterFactory?.(runId) ?? this.requireTaskAdapter()).onRunEnded?.(); + } catch (error: unknown) { + log.warn("WorkflowService: interrupted workflow task sweep failed", { + runId, + error: getErrorMessage(error), + }); + } + } } private async interruptRunTree( - input: { workspaceId: string; runId: string }, + input: { + workspaceId: string; + runId: string; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }, visitedRunIds: Set, skipTerminalRun: boolean ): Promise { @@ -256,6 +324,14 @@ export class WorkflowService { return run; } visitedRunIds.add(input.runId); + if (run.status === "interrupted" && input.retryTaskCleanup === true) { + input.onRunInterrupted?.(input.runId); + await (this.taskAdapterFactory?.(input.runId) ?? this.requireTaskAdapter()).interruptRun?.({ + deferTaskSweep: true, + }); + await this.interruptChildWorkflowRuns(input, visitedRunIds); + return run; + } if (skipTerminalRun && isTerminalWorkflowRunStatus(run.status)) { return run; } @@ -280,8 +356,11 @@ export class WorkflowService { this.clock?.nowIso() ?? new Date().toISOString() ); settleStatusWrite(); + input.onRunInterrupted?.(input.runId); await this.notifyRunStatusChanged(interrupted); - await (this.taskAdapterFactory?.(input.runId) ?? this.requireTaskAdapter()).interruptRun?.(); + await (this.taskAdapterFactory?.(input.runId) ?? this.requireTaskAdapter()).interruptRun?.({ + deferTaskSweep: true, + }); await this.interruptChildWorkflowRuns(input, visitedRunIds); return interrupted; } finally { @@ -293,43 +372,65 @@ export class WorkflowService { } private async interruptChildWorkflowRuns( - input: { workspaceId: string; runId: string }, + input: { + workspaceId: string; + runId: string; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }, visitedRunIds: Set ): Promise { const childRuns = (await this.runStore.listRunStatusSnapshots()).filter( (snapshot) => snapshot.workspaceId === input.workspaceId && snapshot.parentWorkflow?.runId === input.runId && - !isTerminalWorkflowRunStatus(snapshot.status) + (!isTerminalWorkflowRunStatus(snapshot.status) || + (input.retryTaskCleanup === true && snapshot.status === "interrupted")) ); for (const childRun of childRuns) { // Child workflow runs from older workflow scripts are still persisted separately; // interrupting the parent must also stop their run-scoped agents before returning. await this.interruptRunTree( - { workspaceId: input.workspaceId, runId: childRun.id }, + { + workspaceId: input.workspaceId, + runId: childRun.id, + retryTaskCleanup: input.retryTaskCleanup, + onRunInterrupted: input.onRunInterrupted, + }, visitedRunIds, true ); } } + private async withWorkflowRunStartLock( + workspaceId: string, + operation: () => Promise + ): Promise { + return this.withRunStartLock != null + ? await this.withRunStartLock(workspaceId, operation) + : await operation(); + } + async retryRunFromCheckpointInBackground(input: { workspaceId: string; runId: string; projectTrusted: boolean; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanRetryFromCheckpoint(run); - // A checkpoint retry dispatched in the background is non-blocking just like background resume: - // persist notify_on_terminal before starting the background runner. - await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); - await this.runInBackground(input.runId, "Background workflow checkpoint retry failed:", { - allowRetryFromFailedCheckpoint: true, - projectTrusted: input.projectTrusted, + return await this.withWorkflowRunStartLock(input.workspaceId, async () => { + const run = await this.requireRunForWorkspace(input); + assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); + assertWorkflowRunCanRetryFromCheckpoint(run); + // Hold the workspace lifecycle lock until the background runner acquires its lease. A + // concurrent task_stop then either wins before dispatch or sees this run as active. + await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); + await this.runInBackground(input.runId, "Background workflow checkpoint retry failed:", { + allowRetryFromFailedCheckpoint: true, + projectTrusted: input.projectTrusted, + }); + await this.notifyRunStatusChanged(run, "running"); + return { runId: input.runId, status: "running", result: null }; }); - await this.notifyRunStatusChanged(run, "running"); - return { runId: input.runId, status: "running", result: null }; } async resumeRunInBackground(input: { @@ -337,18 +438,76 @@ export class WorkflowService { runId: string; projectTrusted: boolean; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanTransition(run.status, "running"); - // A run resumed in the background becomes non-blocking; persist so future stream-ends do not - // re-force a task_await even if the run was originally started in the foreground. - await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); - await this.runInBackground(input.runId, "Background workflow resume failed:", { - allowResumeFromInterrupted: run.status === "interrupted", - projectTrusted: input.projectTrusted, + return await this.withWorkflowRunStartLock(input.workspaceId, async () => { + const run = await this.requireRunForWorkspace(input); + assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); + assertWorkflowRunCanTransition(run.status, "running"); + // A run resumed in the background becomes non-blocking; persist so future stream-ends do not + // re-force a task_await even if the run was originally started in the foreground. + await this.runStore.setAttentionPolicy(input.runId, "notify_on_terminal"); + await this.runInBackground(input.runId, "Background workflow resume failed:", { + allowResumeFromInterrupted: run.status === "interrupted", + projectTrusted: input.projectTrusted, + }); + await this.notifyRunStatusChanged(run, "running"); + return { runId: input.runId, status: "running", result: null }; }); - await this.notifyRunStatusChanged(run, "running"); - return { runId: input.runId, status: "running", result: null }; + } + + private async dispatchForegroundRunWithStartLock(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + abortSignal?: AbortSignal; + validateRun: (run: WorkflowRunRecord) => void; + runnerOptions: ( + run: WorkflowRunRecord + ) => Pick< + WorkflowRunnerRunOptions, + "allowResumeFromInterrupted" | "allowRetryFromFailedCheckpoint" + >; + backgroundedFailureMessage: string; + }): Promise { + const startLockState = { held: true }; + const deferredInterruptRunIds: string[] = []; + const dispatch = await this.withWorkflowRunStartLock(input.workspaceId, async () => { + try { + const run = await this.requireRunForWorkspace(input); + assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); + input.validateRun(run); + + const started = Promise.withResolvers(); + const resultPromise = this.runForegroundWithAbortInterrupt({ + workspaceId: input.workspaceId, + run, + projectTrusted: input.projectTrusted, + abortSignal: input.abortSignal, + runnerOptions: input.runnerOptions(run), + backgroundedFailureMessage: input.backgroundedFailureMessage, + onRunningStatusPersisted: started.resolve, + startLockState: { + isHeld: () => startLockState.held, + onRunInterrupted: (runId) => deferredInterruptRunIds.push(runId), + }, + }); + // Do not hold the task-tree lock for the full foreground workflow. Release once the runner + // persists running status, or once an early failure settles the dispatch. + await Promise.race([ + started.promise, + resultPromise.then( + () => undefined, + () => undefined + ), + ]); + return { resultPromise }; + } finally { + // Set this before the mutex callback returns: an abort racing with release can safely wait + // for the lock, while an earlier abort avoids self-deadlock by using the held-lock path. + startLockState.held = false; + } + }); + await this.sweepInterruptedRunTasks(deferredInterruptRunIds); + return await dispatch.resultPromise; } async resumeRun(input: { @@ -357,15 +516,12 @@ export class WorkflowService { projectTrusted: boolean; abortSignal?: AbortSignal; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanTransition(run.status, "running"); - return await this.runForegroundWithAbortInterrupt({ - workspaceId: input.workspaceId, - run, - projectTrusted: input.projectTrusted, - abortSignal: input.abortSignal, - runnerOptions: { allowResumeFromInterrupted: run.status === "interrupted" }, + return await this.dispatchForegroundRunWithStartLock({ + ...input, + validateRun: (run) => assertWorkflowRunCanTransition(run.status, "running"), + runnerOptions: (run) => ({ + allowResumeFromInterrupted: run.status === "interrupted", + }), backgroundedFailureMessage: "Backgrounded workflow resume failed:", }); } @@ -376,15 +532,10 @@ export class WorkflowService { projectTrusted: boolean; abortSignal?: AbortSignal; }): Promise { - const run = await this.requireRunForWorkspace(input); - assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); - assertWorkflowRunCanRetryFromCheckpoint(run); - return await this.runForegroundWithAbortInterrupt({ - workspaceId: input.workspaceId, - run, - projectTrusted: input.projectTrusted, - abortSignal: input.abortSignal, - runnerOptions: { allowRetryFromFailedCheckpoint: true }, + return await this.dispatchForegroundRunWithStartLock({ + ...input, + validateRun: assertWorkflowRunCanRetryFromCheckpoint, + runnerOptions: () => ({ allowRetryFromFailedCheckpoint: true }), backgroundedFailureMessage: "Backgrounded workflow checkpoint retry failed:", }); } @@ -403,6 +554,11 @@ export class WorkflowService { "allowResumeFromInterrupted" | "allowRetryFromFailedCheckpoint" >; backgroundedFailureMessage: string; + startLockState?: { + isHeld: () => boolean; + onRunInterrupted: (runId: string) => void; + }; + onRunningStatusPersisted?: () => void; }): Promise { const runId = input.run.id; if (isAbortSignalAborted(input.abortSignal)) { @@ -411,17 +567,22 @@ export class WorkflowService { throw new Error(`Workflow run interrupted: ${runId}`); } - await this.notifyRunStatusChanged(input.run, "running"); - const runnerAbortController = new AbortController(); let unregisterRunnerAbort: () => void = () => undefined; const abortInterrupt = this.interruptRunOnAbort( input.workspaceId, runId, input.abortSignal, - runnerAbortController + runnerAbortController, + input.startLockState ); try { + await this.notifyRunStatusChanged(input.run, "running"); + if (input.abortSignal?.aborted === true) { + await abortInterrupt.wait(); + throw new Error(`Workflow run interrupted: ${runId}`); + } + const runner = await this.createRunner(runId); const result = await runner.run(runId, { abortSignal: runnerAbortController.signal, @@ -431,6 +592,13 @@ export class WorkflowService { runnerAbortController ); }, + ...(input.startLockState != null + ? { + shouldDeferRunEnded: input.startLockState.isHeld, + onRunEndedDeferred: () => input.startLockState?.onRunInterrupted(runId), + } + : {}), + onRunningStatusPersisted: input.onRunningStatusPersisted, ...input.runnerOptions, }); await this.notifyRunStatusChanged(input.run, "completed"); @@ -654,24 +822,68 @@ export class WorkflowService { workspaceId: string, runId: string, abortSignal: AbortSignal | undefined, - runnerAbortController: AbortController | undefined + runnerAbortController: AbortController | undefined, + startLockState?: { + isHeld: () => boolean; + onRunInterrupted: (runId: string) => void; + } ): { remove: () => void; wait: () => Promise } { if (abortSignal == null) { return { remove: () => undefined, wait: () => Promise.resolve() }; } let interruptPromise: Promise | null = null; + let interruptStarted = false; + const getLockOptions = () => + startLockState?.isHeld() === true + ? { + lockAlreadyHeld: true as const, + deferTaskSweep: true as const, + onRunInterrupted: startLockState.onRunInterrupted, + } + : {}; const interrupt = () => { + if (interruptStarted) { + return; + } + interruptStarted = true; // Cancel the coordinator before interrupt side effects can block on task cleanup or disk I/O. runnerAbortController?.abort(); interruptPromise = (async () => { try { - await this.interruptRun({ workspaceId, runId }); - } catch { - // The run may have completed or failed before the abort event was delivered. + await this.interruptRun({ workspaceId, runId, ...getLockOptions() }); + } catch (error: unknown) { + const run = await this.getRun({ workspaceId, runId }); + if (run?.status === "completed" || run?.status === "failed") { + return; + } + if (run?.status === "interrupted") { + try { + await this.interruptRun({ + workspaceId, + runId, + retryTaskCleanup: true, + ...getLockOptions(), + }); + return; + } catch (retryError: unknown) { + // A resume can win between the interrupted re-read and cleanup retry. Ignore only + // terminal completion/failure; if the run remains interrupted, preserve the cleanup + // failure so callers know workflow-owned workers may still need repair. + const latestRun = await this.getRun({ workspaceId, runId }); + if (latestRun?.status === "completed" || latestRun?.status === "failed") { + return; + } + throw retryError; + } + } + throw error; } })(); }; abortSignal.addEventListener("abort", interrupt, { once: true }); + if (abortSignal.aborted) { + interrupt(); + } return { remove: () => abortSignal.removeEventListener("abort", interrupt), wait: async () => { @@ -691,15 +903,19 @@ export class WorkflowService { const normalized = normalizeWorkflowArgsForSource(input.script.source, input.args, { defaultArgs: input.defaultArgs, }); - return await this.runStore.createRun({ - id: runId, - workspaceId: input.workspaceId, - workflow: buildWorkflowScriptDescriptor(input.script), - source: input.script.source, - args: normalized.args, - ...(input.attentionPolicy != null ? { attentionPolicy: input.attentionPolicy } : {}), - now: this.clock?.nowIso() ?? new Date().toISOString(), - }); + const createRun = async () => + await this.runStore.createRun({ + id: runId, + workspaceId: input.workspaceId, + workflow: buildWorkflowScriptDescriptor(input.script), + source: input.script.source, + args: normalized.args, + ...(input.attentionPolicy != null ? { attentionPolicy: input.attentionPolicy } : {}), + now: this.clock?.nowIso() ?? new Date().toISOString(), + }); + return this.withRunStartLock != null + ? await this.withRunStartLock(input.workspaceId, createRun) + : await createRun(); } private async runInBackground( @@ -744,16 +960,18 @@ export class WorkflowService { runId, runnerAbortController ); - markStarted(); }; const runPromise = runner .run(runId, { abortSignal: runnerAbortController.signal, onLeaseAcquired: markLeaseAcquired, + onRunningStatusPersisted: markStarted, backgroundOnMessageQueued: false, ...runnerOptions, }) .then(async (result) => { + // Completed runs can return before another running-status append; unblock startup waiters. + markStarted(); await this.notifyRunStatusChanged(runStatus, "completed"); await this.notifyBackgroundRunTerminal(runId, result); }) diff --git a/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts b/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts index 184f628b542..d38e8affc65 100644 --- a/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts +++ b/src/node/services/workflows/WorkflowTaskServiceAdapter.test.ts @@ -631,6 +631,57 @@ describe("WorkflowTaskServiceAdapter", () => { }); }); + test("can defer the workflow task sweep when the caller already holds the tree lock", async () => { + const create = mock(async () => + Ok({ taskId: "task_1", kind: "agent" as const, status: "running" as const }) + ); + const waitForAgentReport = mock(async () => ({ reportMarkdown: "unused" })); + const terminateAllDescendantAgentTasks = mock(async () => ["task_1"]); + const adapter = new WorkflowTaskServiceAdapter({ + taskService: { create, waitForAgentReport, terminateAllDescendantAgentTasks }, + parentWorkspaceId: "parent_1", + workflowRunId: "wfr_123", + defaultAgentId: "explore", + }); + + await adapter.interruptRun({ deferTaskSweep: true }); + + expect(terminateAllDescendantAgentTasks).toHaveBeenCalledWith("parent_1", { + workflowRunId: "wfr_123", + deferWorkflowSweep: true, + }); + }); + + test("cleans background processes for workflow-owned task workspaces during interruption", async () => { + const create = mock(async () => + Ok({ taskId: "task_1", kind: "agent" as const, status: "running" as const }) + ); + const waitForAgentReport = mock(async () => ({ reportMarkdown: "unused" })); + const cleanupWorkspaceBackgroundProcesses = mock(async () => undefined); + const terminateAllDescendantAgentTasks = mock( + async ( + _workspaceId: string, + options?: { + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; + } + ) => { + await options?.cleanupWorkspaceBackgroundProcesses?.("workflow-task"); + return ["workflow-task"]; + } + ); + const adapter = new WorkflowTaskServiceAdapter({ + taskService: { create, waitForAgentReport, terminateAllDescendantAgentTasks }, + parentWorkspaceId: "parent_1", + workflowRunId: "wfr_123", + defaultAgentId: "explore", + cleanupWorkspaceBackgroundProcesses, + }); + + await adapter.interruptRun(); + + expect(cleanupWorkspaceBackgroundProcesses).toHaveBeenCalledWith("workflow-task"); + }); + test("fails fast when task creation fails", async () => { const create = mock(async () => ({ success: false as const, error: "no runnable agent" })); const waitForAgentReport = mock(async () => ({ reportMarkdown: "should not wait" })); diff --git a/src/node/services/workflows/WorkflowTaskServiceAdapter.ts b/src/node/services/workflows/WorkflowTaskServiceAdapter.ts index 076597dfabc..8d9244dccda 100644 --- a/src/node/services/workflows/WorkflowTaskServiceAdapter.ts +++ b/src/node/services/workflows/WorkflowTaskServiceAdapter.ts @@ -102,7 +102,11 @@ interface WorkflowTaskServiceLike { ): Promise; terminateAllDescendantAgentTasks?( workspaceId: string, - options?: { workflowRunId?: string } + options?: { + workflowRunId?: string; + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; + deferWorkflowSweep?: boolean; + } ): Promise; markWorkflowRunEnded?(workflowRunId: string): Promise; } @@ -129,6 +133,7 @@ export interface WorkflowTaskServiceAdapterOptions { patchToolConfig?: TaskApplyGitPatchConfiguration; applyPatchArtifact?: WorkflowPatchArtifactApplier; getProjectTrusted?: () => boolean | Promise; + cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; } export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { @@ -140,6 +145,7 @@ export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { private readonly patchToolConfig?: TaskApplyGitPatchConfiguration; private readonly applyPatchArtifact?: WorkflowPatchArtifactApplier; private readonly getProjectTrusted?: () => boolean | Promise; + private readonly cleanupWorkspaceBackgroundProcesses?: (workspaceId: string) => Promise; private readonly patchApplyMutex = new AsyncMutex(); private readonly experiments?: WorkflowTaskExperiments; private readonly modelString?: string; @@ -166,6 +172,7 @@ export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { this.patchToolConfig = options.patchToolConfig; this.applyPatchArtifact = options.applyPatchArtifact; this.getProjectTrusted = options.getProjectTrusted; + this.cleanupWorkspaceBackgroundProcesses = options.cleanupWorkspaceBackgroundProcesses; this.experiments = options.experiments; this.modelString = options.modelString; this.thinkingLevel = options.thinkingLevel; @@ -338,9 +345,13 @@ export class WorkflowTaskServiceAdapter implements WorkflowTaskAdapter { return undefined; } - async interruptRun(): Promise { + async interruptRun(options?: { deferTaskSweep?: boolean }): Promise { await this.taskService.terminateAllDescendantAgentTasks?.(this.parentWorkspaceId, { workflowRunId: this.workflowRunId, + ...(this.cleanupWorkspaceBackgroundProcesses != null + ? { cleanupWorkspaceBackgroundProcesses: this.cleanupWorkspaceBackgroundProcesses } + : {}), + ...(options?.deferTaskSweep === true ? { deferWorkflowSweep: true } : {}), }); } From 8b8961116578aea62965233ec98b320f2093f793 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 20:34:17 -0500 Subject: [PATCH 02/10] =?UTF-8?q?=F0=9F=A4=96=20tests:=20update=20workflow?= =?UTF-8?q?=20router=20service=20mocks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide the lifecycle and cleanup methods required by the hardened WorkflowService in router unit-test contexts. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- src/node/orpc/router.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index 51b1d5f7214..ab42995698e 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -128,6 +128,7 @@ describe("router workflow routes", () => { config, aiService: { waitForInit: mock(async () => undefined), + cleanupWorkspaceBackgroundProcesses: mock(async () => undefined), getWorkspaceMetadata: mock(async () => ({ success: true, data: { @@ -149,7 +150,13 @@ describe("router workflow routes", () => { getWorkflowContinuationSendOptions: mock(() => null), sendMessage: mock(async () => ({ success: true, data: undefined })), }, - taskService: {}, + taskService: { + withWorkspaceOwnedWorkStartLock: mock( + async (_workspaceId: string, operation: () => Promise) => await operation() + ), + terminateAllDescendantAgentTasks: mock(async () => []), + markWorkflowRunEnded: mock(async () => undefined), + }, experimentsService: { isExperimentEnabled: mock(() => options.enabled), }, @@ -616,6 +623,9 @@ export default function workflow() { return { reportMarkdown: "should not run" } let waitCalls = 0; context.taskService = { + withWorkspaceOwnedWorkStartLock: mock( + async (_workspaceId: string, operation: () => Promise) => await operation() + ), create: mock(async () => ({ success: true, data: { taskId: "task_slow" } })), waitForAgentReport: mock(async () => { waitCalls += 1; @@ -624,6 +634,8 @@ export default function workflow() { return { reportMarkdown: "should not run" } } return { reportMarkdown: "done", structuredOutput: {} }; }), + terminateAllDescendantAgentTasks: mock(async () => []), + markWorkflowRunEnded: mock(async () => undefined), } as unknown as ORPCContext["taskService"]; const client = createRouterClient(router(), { context }); From 6cd78039e1008bbc00ed26502c9ae8bb1c406008 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 20:46:35 -0500 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20resolve=20child-own?= =?UTF-8?q?ed=20workflow=20sessions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve workflow lifecycle services from each descendant owner's session before listing or interrupting child-owned runs. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- src/common/utils/tools/tools.ts | 124 +++++++++++----------- src/node/services/aiService.ts | 42 ++++++++ src/node/services/tools/task_stop.test.ts | 43 +++++--- src/node/services/tools/task_stop.ts | 39 ++++--- 4 files changed, 156 insertions(+), 92 deletions(-) diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index 66b712494cc..deb0fe85fa0 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -140,6 +140,67 @@ export interface WorkflowServiceScriptInput { sourceKind: "skill" | "workspace-file" | "inline"; } +export interface ToolWorkflowService { + getRun?(input: { workspaceId: string; runId: string }): Promise; + listRuns?(input: { workspaceId: string }): Promise; + startWorkflowInBackground?(input: { + script: WorkflowServiceScriptInput; + workspaceId: string; + projectTrusted: boolean; + args: unknown; + attentionPolicy?: BackgroundWorkAttentionPolicy; + onRunCreated?: (event: { + runId: string; + status: "pending"; + result: null; + run: unknown; + }) => Promise | void; + }): Promise<{ runId: string; status: string; result: unknown }>; + startWorkflow?(input: { + script: WorkflowServiceScriptInput; + workspaceId: string; + projectTrusted: boolean; + args: unknown; + abortSignal?: AbortSignal; + onRunCreated?: (event: { + runId: string; + status: "pending"; + result: null; + run: unknown; + }) => Promise | void; + }): Promise<{ runId: string; status: string; result: unknown }>; + interruptRun?(input: { + workspaceId: string; + runId: string; + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + retryTaskCleanup?: boolean; + onRunInterrupted?: (runId: string) => void; + }): Promise; + resumeRun?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + abortSignal?: AbortSignal; + }): Promise<{ runId: string; status: string; result: unknown }>; + resumeRunInBackground?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + }): Promise<{ runId: string; status: string; result: unknown }>; + retryRunFromCheckpoint?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + abortSignal?: AbortSignal; + }): Promise<{ runId: string; status: string; result: unknown }>; + retryRunFromCheckpointInBackground?(input: { + workspaceId: string; + runId: string; + projectTrusted: boolean; + }): Promise<{ runId: string; status: string; result: unknown }>; +} + export interface ToolConfiguration { /** Working directory for command execution - actual path in runtime's context (local or remote) */ cwd: string; @@ -202,66 +263,9 @@ export interface ToolConfiguration { /** Task orchestration for sub-agent tasks */ taskService?: TaskService; /** Durable workflow lifecycle service for dynamic workflow tools. */ - workflowService?: { - getRun?(input: { workspaceId: string; runId: string }): Promise; - listRuns?(input: { workspaceId: string }): Promise; - startWorkflowInBackground?(input: { - script: WorkflowServiceScriptInput; - workspaceId: string; - projectTrusted: boolean; - args: unknown; - attentionPolicy?: BackgroundWorkAttentionPolicy; - onRunCreated?: (event: { - runId: string; - status: "pending"; - result: null; - run: unknown; - }) => Promise | void; - }): Promise<{ runId: string; status: string; result: unknown }>; - startWorkflow?(input: { - script: WorkflowServiceScriptInput; - workspaceId: string; - projectTrusted: boolean; - args: unknown; - abortSignal?: AbortSignal; - onRunCreated?: (event: { - runId: string; - status: "pending"; - result: null; - run: unknown; - }) => Promise | void; - }): Promise<{ runId: string; status: string; result: unknown }>; - interruptRun?(input: { - workspaceId: string; - runId: string; - deferTaskSweep?: boolean; - lockAlreadyHeld?: boolean; - retryTaskCleanup?: boolean; - onRunInterrupted?: (runId: string) => void; - }): Promise; - resumeRun?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - abortSignal?: AbortSignal; - }): Promise<{ runId: string; status: string; result: unknown }>; - resumeRunInBackground?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - }): Promise<{ runId: string; status: string; result: unknown }>; - retryRunFromCheckpoint?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - abortSignal?: AbortSignal; - }): Promise<{ runId: string; status: string; result: unknown }>; - retryRunFromCheckpointInBackground?(input: { - workspaceId: string; - runId: string; - projectTrusted: boolean; - }): Promise<{ runId: string; status: string; result: unknown }>; - }; + workflowService?: ToolWorkflowService; + /** Resolve the workflow lifecycle service for a descendant workspace's session. */ + workflowServiceForWorkspace?: (workspaceId: string) => ToolWorkflowService | null; /** Workspace heartbeat settings service for model-facing heartbeat configuration. */ workspaceHeartbeatService?: WorkspaceHeartbeatToolService; /** Workspace goal lifecycle service for model-facing goal tools. */ diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index bc73b126a33..7e1bb436d37 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -2190,6 +2190,41 @@ export class AIService extends EventEmitter { }); const getWorkflowProjectTrusted = () => isWorkspaceProjectTrusted(this.config, metadata); + const createWorkflowLifecycleService = (ownerWorkspaceId: string) => + new WorkflowService({ + runStore: new WorkflowRunStore({ + sessionDir: this.config.getSessionDir(ownerWorkspaceId), + }), + onRunStatusChanged: async (event) => { + if (!isTerminalWorkflowRunStatus(event.status)) { + await this.taskService?.resetWorkflowRunTerminalAttention({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } + await this.onWorkflowRunStatusChanged?.(event); + }, + runtimeFactory: new QuickJSRuntimeFactory(), + withRunStartLock: (lockedWorkspaceId, operation) => + this.taskService!.withWorkspaceOwnedWorkStartLock(lockedWorkspaceId, operation), + taskAdapterFactory: (runId, workflowName) => + new WorkflowTaskServiceAdapter({ + taskService: this.taskService!, + parentWorkspaceId: ownerWorkspaceId, + workflowRunId: runId, + workflowName, + defaultAgentId: DEFAULT_WORKFLOW_AGENT_ID, + cleanupWorkspaceBackgroundProcesses: (taskWorkspaceId) => + this.cleanupWorkspaceBackgroundProcesses(taskWorkspaceId), + experiments: { + ...experiments, + dynamicWorkflows: dynamicWorkflowsExperimentEnabled, + workspaceHeartbeats: workspaceHeartbeatsExperimentEnabled, + }, + }), + runnerId: `workflow-runner:${ownerWorkspaceId}`, + }); + const workflowService = dynamicWorkflowsExperimentEnabled && this.taskService != null ? new WorkflowService({ @@ -2503,6 +2538,13 @@ export class AIService extends EventEmitter { muxScope, timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, + workflowServiceForWorkspace: + workflowService != null + ? (ownerWorkspaceId) => + ownerWorkspaceId === workspaceId + ? workflowService + : createWorkflowLifecycleService(ownerWorkspaceId) + : undefined, workflowService, goalService: workspaceGoalService, goalDefaults: effectiveGoalDefaults, diff --git a/src/node/services/tools/task_stop.test.ts b/src/node/services/tools/task_stop.test.ts index 6598d2ea4ac..26c2aac10eb 100644 --- a/src/node/services/tools/task_stop.test.ts +++ b/src/node/services/tools/task_stop.test.ts @@ -324,27 +324,37 @@ describe("task_stop tool", () => { return Promise.resolve(); }), } as unknown as TaskService; + const childWorkflowService = { + listRuns: mock(() => Promise.resolve([workflowRun])), + getRun: mock(() => Promise.resolve(workflowRun)), + interruptRun: mock( + (input: { + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + onRunInterrupted?: (runId: string) => void; + }) => { + expect(input.deferTaskSweep).toBe(true); + expect(input.lockAlreadyHeld).toBe(true); + input.onRunInterrupted?.("wfr_run_1"); + input.onRunInterrupted?.("wfr_nested"); + events.push("workflow"); + return Promise.resolve({ ...workflowRun, status: "interrupted" }); + } + ), + }; + const workflowServiceForWorkspace = mock((ownerWorkspaceId: string) => { + expect(ownerWorkspaceId).toBe("child-task"); + return childWorkflowService; + }); const tool = createTaskStopTool({ ...baseConfig, taskService, workflowService: { - listRuns: mock(() => Promise.resolve([workflowRun])), - getRun: mock(() => Promise.resolve(workflowRun)), - interruptRun: mock( - (input: { - deferTaskSweep?: boolean; - lockAlreadyHeld?: boolean; - onRunInterrupted?: (runId: string) => void; - }) => { - expect(input.deferTaskSweep).toBe(true); - expect(input.lockAlreadyHeld).toBe(true); - input.onRunInterrupted?.("wfr_run_1"); - input.onRunInterrupted?.("wfr_nested"); - events.push("workflow"); - return Promise.resolve({ ...workflowRun, status: "interrupted" }); - } - ), + listRuns: mock(() => { + throw new Error("Parent workflow store must not be used for child-owned runs"); + }), }, + workflowServiceForWorkspace, }); expect( @@ -352,6 +362,7 @@ describe("task_stop tool", () => { ).toEqual({ results: [{ status: "stopped", taskId: "child-task", stoppedTaskIds: ["child-task"] }], }); + expect(workflowServiceForWorkspace).toHaveBeenCalledWith("child-task"); expect(events).toEqual(["workflow", "task", "sweep:wfr_run_1", "sweep:wfr_nested"]); }); diff --git a/src/node/services/tools/task_stop.ts b/src/node/services/tools/task_stop.ts index a3546982701..e7478a11497 100644 --- a/src/node/services/tools/task_stop.ts +++ b/src/node/services/tools/task_stop.ts @@ -1,7 +1,11 @@ import { tool } from "ai"; import { getErrorMessage } from "@/common/utils/errors"; -import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; +import type { + ToolConfiguration, + ToolFactory, + ToolWorkflowService, +} from "@/common/utils/tools/tools"; import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import { isActiveWorkflowRunStatus, @@ -32,7 +36,7 @@ const WORKFLOW_STOPPED_NOTE = * (whose contract says in-progress work is discarded). */ async function interruptWorkflowRun( - config: ToolConfiguration, + workflowService: ToolWorkflowService | null | undefined, workspaceId: string, taskId: string, options?: { @@ -41,7 +45,6 @@ async function interruptWorkflowRun( onRunInterrupted?: (runId: string) => void; } ) { - const workflowService = config.workflowService; if (workflowService?.getRun == null || workflowService.interruptRun == null) { return { status: "error" as const, @@ -140,15 +143,14 @@ async function interruptWorkflowRunsOwnedByAgentTaskTree( task.status === "awaiting_report") ); - const workflowService = config.workflowService; - if (workflowService?.listRuns == null || workflowService.interruptRun == null) { - return activeWorkflowOwnedDescendants.length > 0 - ? "Workflow service not available to stop workflow-owned descendants" - : null; - } - let activeRunCount = 0; for (const ownerWorkspaceId of [taskId, ...userOwnedTaskIds]) { + const workflowService = + config.workflowServiceForWorkspace?.(ownerWorkspaceId) ?? + (ownerWorkspaceId === config.workspaceId ? config.workflowService : null); + if (workflowService?.listRuns == null || workflowService.interruptRun == null) { + return `Workflow service not available for descendant workspace ${ownerWorkspaceId}`; + } const rawRuns = await workflowService.listRuns({ workspaceId: ownerWorkspaceId }); for (const rawRun of rawRuns) { const parsedRun = WorkflowRunRecordSchema.safeParse(rawRun); @@ -162,11 +164,16 @@ async function interruptWorkflowRunsOwnedByAgentTaskTree( } activeRunCount += 1; - const outcome = await interruptWorkflowRun(config, ownerWorkspaceId, parsedRun.data.id, { - deferTaskSweep: true, - lockAlreadyHeld: true, - onRunInterrupted: (runId) => deferredWorkflowRunIds.push(runId), - }); + const outcome = await interruptWorkflowRun( + workflowService, + ownerWorkspaceId, + parsedRun.data.id, + { + deferTaskSweep: true, + lockAlreadyHeld: true, + onRunInterrupted: (runId) => deferredWorkflowRunIds.push(runId), + } + ); if (outcome.status === "stopped") { deferredWorkflowRunIds.push(parsedRun.data.id); } @@ -226,7 +233,7 @@ export const createTaskStopTool: ToolFactory = (config: ToolConfiguration) => { const terminationPromise = (async () => { try { if (isWorkflowRunTaskId(taskId)) { - return await interruptWorkflowRun(config, workspaceId, taskId); + return await interruptWorkflowRun(config.workflowService, workspaceId, taskId); } if (isWorkspaceTurnTaskId(taskId)) { From 3bf1b1bdc4c2735f734e0f566145dda88f879d21 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 20:52:14 -0500 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20clean=20workflow=20?= =?UTF-8?q?processes=20after=20stream=20stop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop workflow worker streams before taking the final background-process snapshot so concurrently registering bash tasks are included in cleanup. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- src/node/services/taskService.test.ts | 25 ++++++++++++++++++------- src/node/services/taskService.ts | 24 ++++++++++++------------ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 119abc271fb..359c2d59d54 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -13980,15 +13980,20 @@ describe("TaskService", () => { testTaskSettings() ); - const { aiService } = createAIServiceMocks(config); + const lifecycleEvents: string[] = []; + const stopStream = mock((workspaceId: string): Promise> => { + lifecycleEvents.push(`stop:${workspaceId}`); + return Promise.resolve(Ok(undefined)); + }); + const { aiService } = createAIServiceMocks(config, { stopStream }); const { workspaceService } = createWorkspaceServiceMocks(); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - const cleanupWorkspaceBackgroundProcesses = mock( - (workspaceId: string): Promise => - workspaceId === workflowChildTaskId - ? Promise.reject(new Error("background cleanup failed")) - : Promise.resolve() - ); + const cleanupWorkspaceBackgroundProcesses = mock((workspaceId: string): Promise => { + lifecycleEvents.push(`cleanup:${workspaceId}`); + return workspaceId === workflowChildTaskId + ? Promise.reject(new Error("background cleanup failed")) + : Promise.resolve(); + }); const interruptedTaskIds = await taskService.terminateAllDescendantAgentTasks(rootWorkspaceId, { workflowRunId: "wfr_target", @@ -13996,6 +14001,12 @@ describe("TaskService", () => { }); expect(interruptedTaskIds).toEqual([workflowChildTaskId, workflowTaskId]); + expect(lifecycleEvents).toEqual([ + `stop:${workflowChildTaskId}`, + `cleanup:${workflowChildTaskId}`, + `stop:${workflowTaskId}`, + `cleanup:${workflowTaskId}`, + ]); expect(cleanupWorkspaceBackgroundProcesses.mock.calls.map((call) => call[0])).toEqual([ workflowChildTaskId, workflowTaskId, diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 299f5ae51c9..44a967a0a92 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -5254,18 +5254,6 @@ export class TaskService { const interruptionError = new Error("Parent workspace interrupted"); for (const id of descendants) { - // Workflow workers can leave long-running bash processes behind even after their AI stream - // stops. Cleanup is best-effort: a disposal failure must not leave an interrupted workflow - // with agent streams and task statuses still active. - try { - await options?.cleanupWorkspaceBackgroundProcesses?.(id); - } catch (error: unknown) { - log.warn("terminateAllDescendantAgentTasks: background cleanup failed", { - taskId: id, - error: getErrorMessage(error), - }); - } - // Best-effort: clear queue first. AgentSession stream-end cleanup auto-flushes // queued messages, so descendants must not keep pending input after a hard interrupt. try { @@ -5291,6 +5279,18 @@ export class TaskService { log.debug("terminateAllDescendantAgentTasks: stopStream threw", { taskId: id, error }); } + // Stop the worker stream before taking the final process snapshot: a background bash tool + // can register its process while the stream is still winding down. Cleanup remains + // best-effort so a disposal failure cannot block task status teardown. + try { + await options?.cleanupWorkspaceBackgroundProcesses?.(id); + } catch (error: unknown) { + log.warn("terminateAllDescendantAgentTasks: background cleanup failed", { + taskId: id, + error: getErrorMessage(error), + }); + } + let preservedCompletedDescendant = false; let transitionedToInterrupted = false; let parentWorkspaceId: string | undefined; From f423a54425309d68c58c50e295dac733c1a61e26 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 20:59:49 -0500 Subject: [PATCH 05/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20clean=20nested=20wo?= =?UTF-8?q?rkflow=20task=20trees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interrupt workflows owned by every descendant session and remove archived workflow-owned workers when deleting their user-owned parent subtree. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- src/node/services/taskService.test.ts | 39 +++++++++++++ src/node/services/taskService.ts | 67 +++++++++++++++++++++++ src/node/services/tools/task_stop.test.ts | 56 +++++++++++++------ src/node/services/tools/task_stop.ts | 2 +- 4 files changed, 145 insertions(+), 19 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 359c2d59d54..7c69bc41b4f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -13380,6 +13380,45 @@ describe("TaskService", () => { ).toEqual(Err({ code: "not_found" })); }); + test("removeInactiveDescendantAgentTask removes archived workflow-owned descendants with their user-owned parent", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const ownerWorkspaceId = "owner-remove-archived-workflow"; + const parentTaskId = "parent-remove-archived-workflow"; + const workflowTaskId = "workflow-remove-archived-child"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "owner", ownerWorkspaceId), + projectWorkspace(projectPath, "parent", parentTaskId, { + parentWorkspaceId: ownerWorkspaceId, + taskStatus: "reported", + }), + projectWorkspace(projectPath, "workflow-child", workflowTaskId, { + parentWorkspaceId: parentTaskId, + taskStatus: "interrupted", + workflowTask: { runId: "wfr_remove_archived", stepId: "worker" }, + archivedAt: "2026-08-16T00:00:00.000Z", + }), + ], + testTaskSettings() + ); + const remove = mock(async (workspaceId: string): Promise> => { + await removeWorkspaceFromTestConfig(config, workspaceId); + return Ok(undefined); + }); + const { workspaceService } = createWorkspaceServiceMocks({ remove }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + expect( + await taskService.removeInactiveDescendantAgentTask(ownerWorkspaceId, parentTaskId) + ).toMatchObject({ success: true, data: { status: "removed", taskId: parentTaskId } }); + expect(remove.mock.calls.map((call) => call[0])).toEqual([workflowTaskId, parentTaskId]); + expect(findWorkspaceInConfig(config, workflowTaskId)).toBeUndefined(); + expect(findWorkspaceInConfig(config, parentTaskId)).toBeUndefined(); + }); + test("requestAgentFinalReportForTimeout records finalization token only after prompt send succeeds", async () => { const config = await createTestConfig(rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 44a967a0a92..fda6556f1d7 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8526,6 +8526,60 @@ export class TaskService { return Ok(didUnarchive); } + private async removeArchivedWorkflowOwnedDescendantsUnderLifecycleLock( + taskId: string + ): Promise> { + const config = this.config.loadConfigOrDefault(); + const index = this.buildAgentTaskIndex(config); + const candidateTaskIds = this.listDescendantAgentTaskIdsFromIndex(index, taskId).filter( + (descendantTaskId) => { + const descendant = index.byId.get(descendantTaskId); + return ( + descendant != null && + this.isWorkflowOwnedTaskUsingIndex(index, descendantTaskId) && + isWorkspaceArchived(descendant.archivedAt, descendant.unarchivedAt) + ); + } + ); + candidateTaskIds.sort( + (left, right) => + this.getTaskDepthFromParentById(index.parentById, right) - + this.getTaskDepthFromParentById(index.parentById, left) + ); + + for (const descendantTaskId of candidateTaskIds) { + const freshConfig = this.config.loadConfigOrDefault(); + const freshIndex = this.buildAgentTaskIndex(freshConfig); + const descendant = freshIndex.byId.get(descendantTaskId); + if ( + descendant == null || + !this.isWorkflowOwnedTaskUsingIndex(freshIndex, descendantTaskId) || + !isWorkspaceArchived(descendant.archivedAt, descendant.unarchivedAt) + ) { + continue; + } + if (this.isActiveAgentTaskEntry(descendant) || this.aiService.isStreaming(descendantTaskId)) { + return Err(`Archived workflow-owned descendant ${descendantTaskId} is still active.`); + } + if ((freshIndex.childrenByParent.get(descendantTaskId) ?? []).length > 0) { + continue; + } + + const tombstoneResult = await this.persistRemovedAgentTaskTombstones(descendantTaskId); + if (!tombstoneResult.success) { + return tombstoneResult; + } + const removeResult = await this.workspaceService.removeWhileTaskTreeLocked( + descendantTaskId, + true + ); + if (!removeResult.success) { + return Err(removeResult.error); + } + } + return Ok(undefined); + } + async removeInactiveDescendantAgentTask( ownerWorkspaceId: string, taskId: string @@ -8561,6 +8615,19 @@ export class TaskService { workspaceId: taskId, ...(displayName != null ? { displayName } : {}), }; + // Workflow-owned workers are hidden from public lifecycle tools, but their archived + // workspaces must not permanently block removal of the user-owned parent subtree. + const workflowCleanupResult = + await this.removeArchivedWorkflowOwnedDescendantsUnderLifecycleLock(taskId); + if (!workflowCleanupResult.success) { + return Ok({ + status: "error", + action: "remove", + ...target, + error: workflowCleanupResult.error, + }); + } + const descendantTaskIds = this.listDescendantAgentTasks(taskId).map((task) => task.taskId); if (descendantTaskIds.length > 0) { return Ok({ diff --git a/src/node/services/tools/task_stop.test.ts b/src/node/services/tools/task_stop.test.ts index 26c2aac10eb..d6cd1e0442e 100644 --- a/src/node/services/tools/task_stop.test.ts +++ b/src/node/services/tools/task_stop.test.ts @@ -296,6 +296,11 @@ describe("task_stop tool", () => { const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); const events: string[] = []; const workflowRun = { ...buildWorkflowRun("running"), workspaceId: "child-task" }; + const workerWorkflowRun = { + ...buildWorkflowRun("running"), + id: "wfr_worker", + workspaceId: "workflow-worker", + }; const taskService = { listDescendantAgentTasks: mock( (_taskId: string, options?: { excludeWorkflowTasks?: boolean }) => @@ -324,27 +329,33 @@ describe("task_stop tool", () => { return Promise.resolve(); }), } as unknown as TaskService; + const interruptRun = + (runId: string, run: ReturnType, event: string) => + (input: { + deferTaskSweep?: boolean; + lockAlreadyHeld?: boolean; + onRunInterrupted?: (interruptedRunId: string) => void; + }) => { + expect(input.deferTaskSweep).toBe(true); + expect(input.lockAlreadyHeld).toBe(true); + input.onRunInterrupted?.(runId); + events.push(event); + return Promise.resolve({ ...run, status: "interrupted" }); + }; const childWorkflowService = { listRuns: mock(() => Promise.resolve([workflowRun])), getRun: mock(() => Promise.resolve(workflowRun)), - interruptRun: mock( - (input: { - deferTaskSweep?: boolean; - lockAlreadyHeld?: boolean; - onRunInterrupted?: (runId: string) => void; - }) => { - expect(input.deferTaskSweep).toBe(true); - expect(input.lockAlreadyHeld).toBe(true); - input.onRunInterrupted?.("wfr_run_1"); - input.onRunInterrupted?.("wfr_nested"); - events.push("workflow"); - return Promise.resolve({ ...workflowRun, status: "interrupted" }); - } - ), + interruptRun: mock(interruptRun("wfr_run_1", workflowRun, "workflow:child")), + }; + const workerWorkflowService = { + listRuns: mock(() => Promise.resolve([workerWorkflowRun])), + getRun: mock(() => Promise.resolve(workerWorkflowRun)), + interruptRun: mock(interruptRun("wfr_worker", workerWorkflowRun, "workflow:workflow-worker")), }; const workflowServiceForWorkspace = mock((ownerWorkspaceId: string) => { - expect(ownerWorkspaceId).toBe("child-task"); - return childWorkflowService; + if (ownerWorkspaceId === "child-task") return childWorkflowService; + if (ownerWorkspaceId === "workflow-worker") return workerWorkflowService; + throw new Error(`Unexpected workflow owner ${ownerWorkspaceId}`); }); const tool = createTaskStopTool({ ...baseConfig, @@ -362,8 +373,17 @@ describe("task_stop tool", () => { ).toEqual({ results: [{ status: "stopped", taskId: "child-task", stoppedTaskIds: ["child-task"] }], }); - expect(workflowServiceForWorkspace).toHaveBeenCalledWith("child-task"); - expect(events).toEqual(["workflow", "task", "sweep:wfr_run_1", "sweep:wfr_nested"]); + expect(workflowServiceForWorkspace.mock.calls.map((call) => call[0])).toEqual([ + "child-task", + "workflow-worker", + ]); + expect(events).toEqual([ + "workflow:child", + "workflow:workflow-worker", + "task", + "sweep:wfr_run_1", + "sweep:wfr_worker", + ]); }); it("does not start termination when the signal is already aborted", async () => { diff --git a/src/node/services/tools/task_stop.ts b/src/node/services/tools/task_stop.ts index e7478a11497..2badf882b2a 100644 --- a/src/node/services/tools/task_stop.ts +++ b/src/node/services/tools/task_stop.ts @@ -144,7 +144,7 @@ async function interruptWorkflowRunsOwnedByAgentTaskTree( ); let activeRunCount = 0; - for (const ownerWorkspaceId of [taskId, ...userOwnedTaskIds]) { + for (const ownerWorkspaceId of [taskId, ...descendants.map((task) => task.taskId)]) { const workflowService = config.workflowServiceForWorkspace?.(ownerWorkspaceId) ?? (ownerWorkspaceId === config.workspaceId ? config.workflowService : null); From 1cab3a3b383716f4c84444429b363fe4d7482746 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 21:09:24 -0500 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20format=20tool=20exa?= =?UTF-8?q?mples=20for=20MDX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrap task invocation examples as inline code so generated hook documentation parses correctly in Mintlify. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- docs/hooks/tools.mdx | 36 +++++++++---------- src/common/utils/tools/toolDefinitions.ts | 18 +++++----- .../builtInSkillContent.generated.ts | 36 +++++++++---------- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/docs/hooks/tools.mdx b/docs/hooks/tools.mdx index b343a26317c..d5ffc58a7ca 100644 --- a/docs/hooks/tools.mdx +++ b/docs/hooks/tools.mdx @@ -333,17 +333,17 @@ If a value is too large for the environment, it may be omitted (not set). Mux al
bash (9) -| Env var | JSON path | Type | Description | -| --------------------------------------- | ------------------------ | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. | -| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. | -| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. | -| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. | -| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. | -| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await({ task_ids: [result.taskId] }) (returns only new output since last check). Stop with task_stop({ task_ids: [result.taskId] }). List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. | -| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute | -| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive | +| Env var | JSON path | Type | Description | +| --------------------------------------- | ------------------------ | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. | +| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. | +| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. | +| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. | +| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. | +| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with `task\_await({ task\_ids: [result.taskId] })` (returns only new output since last check). Stop with `task\_stop({ task\_ids: [result.taskId] })`. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. | +| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute | +| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |
@@ -798,19 +798,19 @@ If a value is too large for the environment, it may be omitted (not set). Mux al | ---------------------------------- | ------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `MUX_TOOL_INPUT_MODE` | `mode` | enum | Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. Use 'retry_from_checkpoint' only for failed runs; it re-executes work after the last checkpoint and is rejected when unsafe. | | `MUX_TOOL_INPUT_RUN_ID` | `run_id` | string | Workflow run ID (wfr\_...) to resume. Must belong to the current workspace. | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with task_await({ task_ids: [result.runId] }) when you need the result. | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with `task\_await({ task\_ids: [result.runId] })` when you need the result. |
workflow_run (4) -| Env var | JSON path | Type | Description | -| ---------------------------------- | ------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — | -| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result. | -| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. | -| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. | +| Env var | JSON path | Type | Description | +| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — | +| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with `task\_await({ task\_ids: [result.runId] })` before using the result. | +| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. | +| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |
diff --git a/src/common/utils/tools/toolDefinitions.ts b/src/common/utils/tools/toolDefinitions.ts index 12935d2d512..2324e8965d3 100644 --- a/src/common/utils/tools/toolDefinitions.ts +++ b/src/common/utils/tools/toolDefinitions.ts @@ -320,7 +320,7 @@ export function buildTaskToolDescription(runtimeMode: RuntimeMode | undefined): "\n\nWhen the user explicitly asks for best-of-n work, the parent should begin with light preliminary analysis to extract shared context, constraints, or evaluation criteria that would otherwise be duplicated across children. " + "Keep that pre-work lightweight: frame the task and provide useful starting points, but do not pre-solve the problem or over-constrain how the children reason about it. Then delegate the substantive analysis to the spawned sub-agents. " + "Do not also do a full parallel analysis in the parent. Call task_await when you are ready to act on child output; do not await reflexively just because tasks are running. " + - "task_await returns as soon as the first awaited task completes by default (min_completed), so you can start dependent work on each result as it lands instead of blocking on the whole batch. Pass returned camelCase IDs through task_await's snake_case input: task_await({ task_ids: [result.taskId] }) for one handle, or task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length }) when every grouped result is required (or use a foreground grouped spawn, below). " + + "task_await returns as soon as the first awaited task completes by default (min_completed), so you can start dependent work on each result as it lands instead of blocking on the whole batch. Pass returned camelCase IDs through task_await's snake_case input: `task_await({ task_ids: [result.taskId] })` for one handle, or `task_await({ task_ids: result.taskIds, min_completed: result.taskIds.length })` when every grouped result is required (or use a foreground grouped spawn, below). " + "\n\nWhen delegating, include a compact task brief (Task / Background / Scope / Starting points / Acceptance / Deliverables / Constraints). " + "For now, persisted sub-agent goals are not supported; pass sub-agent objectives, success criteria, and deliverables directly in the prompt. " + "Sub-agents observe the same system instructions as the parent (project/global AGENTS.md and custom instructions), so do not restate that shared context in the prompt; spend the prompt on task-specific information the sub-agent cannot infer from those instructions. " + @@ -725,7 +725,7 @@ const TaskAwaitToolArtifactsSchema = z * and can be re-fetched by ID after context compaction instead of re-running the work. */ export const COMPLETED_REPORT_REFETCH_NOTE = - 'Report persisted on disk; re-fetch anytime (even after context compaction) with task_await({ task_ids: [""], timeout_secs: 0 }).'; + 'Report persisted on disk; re-fetch anytime (even after context compaction) with `task_await({ task_ids: [""], timeout_secs: 0 })`.'; export const TaskAwaitToolCompletedResultSchema = z .object({ @@ -1406,7 +1406,7 @@ export const WorkflowRunToolArgsSchema = z .default(false) .describe( "Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. " + - "Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result." + "Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with `task_await({ task_ids: [result.runId] })` before using the result." ), }) .strict() @@ -1449,7 +1449,7 @@ export const WorkflowResumeToolArgsSchema = z .default(false) .describe( "Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. " + - "Set true to resume in the background and continue other work; await it with task_await({ task_ids: [result.runId] }) when you need the result." + "Set true to resume in the background and continue other work; await it with `task_await({ task_ids: [result.runId] })` when you need the result." ), mode: WorkflowResumeModeSchema.nullish().describe( "Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. " + @@ -1682,8 +1682,8 @@ export const TOOL_DEFINITIONS = { "Do NOT use for quick commands (<5s), interactive processes (no stdin support), " + "or processes requiring real-time output (use foreground with larger timeout instead). " + "Returns immediately with a taskId (bash:) and backgroundProcessId. " + - "Read output with task_await({ task_ids: [result.taskId] }) (returns only new output since last check). " + - "Stop with task_stop({ task_ids: [result.taskId] }). " + + "Read output with `task_await({ task_ids: [result.taskId] })` (returns only new output since last check). " + + "Stop with `task_stop({ task_ids: [result.taskId] })`. " + "List active tasks with task_list. " + "Process persists until timeout_secs expires, terminated, or workspace is removed." + "\\n\\nFor long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. " + @@ -2244,7 +2244,7 @@ export const TOOL_DEFINITIONS = { "When a terminal wake-up says a sub-agent report or failure is already injected into context, integrate it directly — do NOT call task_await for it. When a wake-up asks you to retrieve a workspace turn's terminal output, call task_await with the listed IDs and timeout_secs: 0 (a one-shot retrieval, not a wait). " + "\n\nIMPORTANT: Do not call task_await in the same parallel tool-call batch as task, bash, or workflow_run — " + "the taskId/runId is not available until the spawning tool returns. " + - "Always wait for the task/bash/workflow_run tool result first, then call task_await in a subsequent step. Map returned result fields explicitly: task_await({ task_ids: [result.taskId] }) or task_await({ task_ids: result.taskIds }). " + + "Always wait for the task/bash/workflow_run tool result first, then call task_await in a subsequent step. Map returned result fields explicitly: `task_await({ task_ids: [result.taskId] })` or `task_await({ task_ids: result.taskIds })`. " + "When omitting task_ids to await active tasks/workflows, ensure at least one background task or workflow was already spawned in a prior step. Omitted task_ids discover active in-scope descendant agent tasks, public workspace turns, background bash tasks, and top-level workflow runs, while excluding workflow-owned internal workers because their results are consumed through parent workflow runs. " + "\n\nAgent tasks and workflow runs return reports when completed. " + "Completed reports are persisted on disk and survive context compaction: calling task_await on an already-completed task/workflow run ID (timeout_secs: 0 for non-blocking) re-fetches the full report instead of re-running the work. " + @@ -2303,7 +2303,7 @@ export const TOOL_DEFINITIONS = { "the conductor follows the documented phases more faithfully and gains durable checkpoints, resume, and fresh delegated context per phase. " + "Use agent_skill_read / agent_skill_read_file to discover and inspect skill-packaged workflows; non-skill workflow files must be addressed by an explicit known path and can be inspected with normal file tools. " + "Prefer the default foreground mode (`run_in_background` omitted or false) so completed workflows return their result without an extra task_await round-trip. " + - "If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using or reporting the workflow output. " + + "If workflow_run returns status=running or status=backgrounded, await it with `task_await({ task_ids: [result.runId] })` before using or reporting the workflow output. " + "After a previous workflow_run error, abort, timeout, or uncertain result, do not start a fresh run until you rediscover existing workflow runs: either omit task_list statuses first, or query pending/running/backgrounded/interrupted/failed/completed together. " + "Use task_await for running/backgrounded runs, workflow_resume for pending/interrupted runs, workflow_resume({ mode: 'retry_from_checkpoint' }) only for eligible failed runs, and inspect/refetch completed results instead of rerunning. " + "Use background mode only when you intend to start another workflow/task or do independent work while the workflow runs; a background run is non-blocking and Mux wakes this workspace with the terminal workflow result, so call task_await only when the current request depends on the output before you can answer.", @@ -2317,7 +2317,7 @@ export const TOOL_DEFINITIONS = { "For failed runs, pass mode='retry_from_checkpoint' explicitly; it re-executes work after the last checkpoint, so only use it when that is acceptable, and start a fresh workflow_run when it is rejected as unsafe. " + "Calling this on a completed run returns its existing result without re-running anything. " + "Prefer foreground mode (run_in_background omitted or false) to get the final result directly; " + - "if the returned status is running or backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result.", + "if the returned status is running or backgrounded, await it with `task_await({ task_ids: [result.runId] })` before using the result.", schema: WorkflowResumeToolArgsSchema, }, agent_report: { diff --git a/src/node/services/agentSkills/builtInSkillContent.generated.ts b/src/node/services/agentSkills/builtInSkillContent.generated.ts index dd55ad84355..6e73e508429 100644 --- a/src/node/services/agentSkills/builtInSkillContent.generated.ts +++ b/src/node/services/agentSkills/builtInSkillContent.generated.ts @@ -5229,17 +5229,17 @@ export const BUILTIN_SKILL_FILES: Record> = { "
", "bash (9)", "", - "| Env var | JSON path | Type | Description |", - "| --------------------------------------- | ------------------------ | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |", - "| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. |", - "| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. |", - "| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. |", - "| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. |", - "| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. |", - "| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with task_await({ task_ids: [result.taskId] }) (returns only new output since last check). Stop with task_stop({ task_ids: [result.taskId] }). List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. |", - "| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute |", - "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |", + "| Env var | JSON path | Type | Description |", + "| --------------------------------------- | ------------------------ | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", + "| `MUX_TOOL_INPUT_DISPLAY_NAME` | `display_name` | string | Human-readable name for the process (e.g., 'Dev Server', 'TypeCheck Watch'). Required for all bash invocations since any process can be sent to background. |", + "| `MUX_TOOL_INPUT_MODEL_INTENT` | `model_intent` | string | Optional. Short user-facing purpose for this command, shown next to the command in collapsed chat. Use a present-participle phrase in plain English, under 100 characters. Do not repeat the command or include duration, because Mux appends those. Examples: 'Running the unit tests', 'Checking repository state', 'Inspecting build output'. |", + "| `MUX_TOOL_INPUT_MONITOR_COOLDOWN_MS` | `monitor.cooldown_ms` | number | Milliseconds to coalesce matching lines before one wake. Defaults to 1000. |", + "| `MUX_TOOL_INPUT_MONITOR_FILTER` | `monitor.filter` | string | Regex applied to each complete output line. |", + "| `MUX_TOOL_INPUT_MONITOR_FILTER_EXCLUDE` | `monitor.filter_exclude` | boolean | When true, wake for complete lines that do not match filter. |", + "| `MUX_TOOL_INPUT_MONITOR_MAX_EVENTS` | `monitor.max_events` | number | Stop monitoring after this many matching lines; the process keeps running. |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Run this command in the background without blocking. Use for processes running >5s (dev servers, builds, file watchers). Do NOT use for quick commands (<5s), interactive processes (no stdin support), or processes requiring real-time output (use foreground with larger timeout instead). Returns immediately with a taskId (bash:<processId>) and backgroundProcessId. Read output with `task\\_await({ task\\_ids: [result.taskId] })` (returns only new output since last check). Stop with `task\\_stop({ task\\_ids: [result.taskId] })`. List active tasks with task_list. Process persists until timeout_secs expires, terminated, or workspace is removed. For long-running tasks like builds or compilations, prefer background mode to continue productive work in parallel. Without a monitor, raw background bash does not automatically wake the parent workspace when it prints output or exits. With monitor, matching complete output lines wake this workspace, including after your current response; use task_await only if you need surrounding/full output. Before finishing, terminate monitored tasks that are no longer relevant so stale output cannot trigger a follow-up turn. Do not call task_await in the same parallel tool-call batch; wait for the bash result first, then pass result.taskId via task_ids. When you actually need the output, read it with task_await; do not poll task_await just because the process is still running. |", + "| `MUX_TOOL_INPUT_SCRIPT` | `script` | string | The bash script/command to execute |", + "| `MUX_TOOL_INPUT_TIMEOUT_SECS` | `timeout_secs` | number | Timeout in seconds. For foreground: max execution time before kill. For background: max lifetime before auto-termination. Start small and increase on retry; avoid large initial values to keep UX responsive |", "", "
", "", @@ -5694,19 +5694,19 @@ export const BUILTIN_SKILL_FILES: Record> = { "| ---------------------------------- | ------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", "| `MUX_TOOL_INPUT_MODE` | `mode` | enum | Defaults to 'resume', which continues interrupted or crash-orphaned runs from durable state and never re-executes completed steps. Use 'retry_from_checkpoint' only for failed runs; it re-executes work after the last checkpoint and is rejected when unsafe. |", "| `MUX_TOOL_INPUT_RUN_ID` | `run_id` | string | Workflow run ID (wfr\\_...) to resume. Must belong to the current workspace. |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with task_await({ task_ids: [result.runId] }) when you need the result. |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false (foreground): waits until the run reaches a terminal status and returns its result. Set true to resume in the background and continue other work; await it with `task\\_await({ task\\_ids: [result.runId] })` when you need the result. |", "", "", "", "
", "workflow_run (4)", "", - "| Env var | JSON path | Type | Description |", - "| ---------------------------------- | ------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |", - "| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — |", - "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with task_await({ task_ids: [result.runId] }) before using the result. |", - '| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. |', - "| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |", + "| Env var | JSON path | Type | Description |", + "| ---------------------------------- | ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |", + "| `MUX_TOOL_INPUT_ARGS` | `args` | unknown | — |", + "| `MUX_TOOL_INPUT_RUN_IN_BACKGROUND` | `run_in_background` | boolean | Defaults to false. Prefer foreground mode for a single workflow; when the returned status is completed, the result is available directly. Set true only when you will start another workflow/task or do independent work while it runs. If workflow_run returns status=running or status=backgrounded, await it with `task\\_await({ task\\_ids: [result.runId] })` before using the result. |", + '| `MUX_TOOL_INPUT_SCRIPT_PATH` | `script_path` | string | Explicit workflow script path, such as "skill://deep-research/workflow.js" or "./workflows/research.js". Use paths for reusable, reviewable, or skill-packaged workflows. |', + "| `MUX_TOOL_INPUT_SCRIPT_SOURCE` | `script_source` | string | Inline JavaScript workflow source for one-off conductors, including prose-described processes codified in place. The exact source is snapshotted into the durable run for replay/resume. |", "", "
", "", From 1c3c2c70b9ff333757dcd92a6f3a55de67074ad4 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 21:18:18 -0500 Subject: [PATCH 07/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20correlate=20workflo?= =?UTF-8?q?w=20workers=20to=20owning=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep ordinary task stops working without workflow services, validate every active workflow worker against its exact owner run, and retain lifecycle resolvers for cleanup even when workflow tools are disabled. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- src/node/services/aiService.ts | 4 +- src/node/services/taskService.test.ts | 15 ++- src/node/services/taskService.ts | 15 +++ src/node/services/tools/task_stop.test.ts | 110 +++++++++++++++++++++- src/node/services/tools/task_stop.ts | 109 +++++++++++++++------ 5 files changed, 217 insertions(+), 36 deletions(-) diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 7e1bb436d37..6d9e2d413e8 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -2539,9 +2539,9 @@ export class AIService extends EventEmitter { timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, workflowServiceForWorkspace: - workflowService != null + this.taskService != null ? (ownerWorkspaceId) => - ownerWorkspaceId === workspaceId + ownerWorkspaceId === workspaceId && workflowService != null ? workflowService : createWorkflowLifecycleService(ownerWorkspaceId) : undefined, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 7c69bc41b4f..6b929d05148 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -14098,9 +14098,18 @@ describe("TaskService", () => { const { workspaceService } = createWorkspaceServiceMocks(); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - expect( - new Set(taskService.listDescendantAgentTasks(rootWorkspaceId).map((task) => task.taskId)) - ).toEqual(new Set([regularTaskId, workflowChildTaskId, workflowTaskId])); + const descendants = taskService.listDescendantAgentTasks(rootWorkspaceId); + expect(new Set(descendants.map((task) => task.taskId))).toEqual( + new Set([regularTaskId, workflowChildTaskId, workflowTaskId]) + ); + expect(descendants.find((task) => task.taskId === workflowTaskId)).toMatchObject({ + workflowRunId: "wfr_target", + workflowOwnerWorkspaceId: rootWorkspaceId, + }); + expect(descendants.find((task) => task.taskId === workflowChildTaskId)).toMatchObject({ + workflowRunId: "wfr_target", + workflowOwnerWorkspaceId: rootWorkspaceId, + }); expect( taskService .listDescendantAgentTasks(rootWorkspaceId, { diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index fda6556f1d7..d085a33a14a 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -614,6 +614,10 @@ export interface DescendantAgentTaskInfo { executionStatus?: WorkspaceTurnTaskStatus; modelString?: string; thinkingLevel?: ThinkingLevel; + /** Effective owning workflow run for this task branch, inherited through workflow ancestry. */ + workflowRunId?: string; + /** Workspace session that persists the effective owning workflow run. */ + workflowOwnerWorkspaceId?: string; depth: number; } @@ -8768,6 +8772,9 @@ export class TaskService { ); const workflowOwned = next.workflowOwned || entry.workflowTask != null; + const workflowOwner = workflowOwned + ? this.findWorkflowTaskOwnerInAncestry(index, next.taskId) + : null; const status: AgentTaskStatus = entry.taskStatus ?? "running"; if ( (!statusFilter || statusFilter.has(status)) && @@ -8783,6 +8790,14 @@ export class TaskService { createdAt: entry.createdAt, executionTaskId: entry.taskExecutionId, executionStatus: entry.taskExecutionStatus, + ...(workflowOwner != null + ? { + workflowRunId: workflowOwner.workflowTask.runId, + ...(workflowOwner.workspace.parentWorkspaceId != null + ? { workflowOwnerWorkspaceId: workflowOwner.workspace.parentWorkspaceId } + : {}), + } + : {}), modelString: entry.aiSettings?.model, thinkingLevel: entry.aiSettings?.thinkingLevel, depth: next.depth, diff --git a/src/node/services/tools/task_stop.test.ts b/src/node/services/tools/task_stop.test.ts index d6cd1e0442e..b622e4c6297 100644 --- a/src/node/services/tools/task_stop.test.ts +++ b/src/node/services/tools/task_stop.test.ts @@ -124,6 +124,33 @@ describe("task_stop tool", () => { }); }); + it("stops ordinary task trees when dynamic workflow services are unavailable", async () => { + using tempDir = new TestTempDir("test-task-stop-without-workflow-service"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const taskService = { + listDescendantAgentTasks: mock(() => [ + { taskId: "ordinary-grandchild", status: "running" as const }, + ]), + stopDescendantAgentTask: mock( + (): Promise> => + Promise.resolve(Ok({ stoppedTaskIds: ["ordinary-grandchild", "ordinary-child"] })) + ), + } as unknown as TaskService; + const tool = createTaskStopTool({ ...baseConfig, taskService }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["ordinary-child"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "stopped", + taskId: "ordinary-child", + stoppedTaskIds: ["ordinary-grandchild", "ordinary-child"], + }, + ], + }); + }); + it("returns an interrupted error promptly while completed task IDs still resolve", async () => { using tempDir = new TestTempDir("test-task-terminate-abort"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); @@ -306,7 +333,14 @@ describe("task_stop tool", () => { (_taskId: string, options?: { excludeWorkflowTasks?: boolean }) => options?.excludeWorkflowTasks === true ? [] - : [{ taskId: "workflow-worker", status: "running" as const }] + : [ + { + taskId: "workflow-worker", + status: "running" as const, + workflowRunId: "wfr_run_1", + workflowOwnerWorkspaceId: "child-task", + }, + ] ), isDescendantAgentTask: mock(() => Promise.resolve(true)), isWorkflowOwnedDescendantAgentTask: mock(() => Promise.resolve(false)), @@ -386,6 +420,80 @@ describe("task_stop tool", () => { ]); }); + it("does not let a healthy workflow run mask an orphaned active workflow worker", async () => { + using tempDir = new TestTempDir("test-task-stop-orphaned-workflow-worker"); + const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); + const healthyRun = { ...buildWorkflowRun("running"), id: "wfr_healthy", workspaceId: "child" }; + const stopDescendantAgentTask = mock( + async ( + _workspaceId: string, + _taskId: string, + options?: { beforeStop?: () => Promise } + ): Promise> => { + const error = await options?.beforeStop?.(); + return error == null ? Ok({ stoppedTaskIds: ["child"] }) : Err(error); + } + ); + const taskService = { + listDescendantAgentTasks: mock( + (_taskId: string, options?: { excludeWorkflowTasks?: boolean }) => + options?.excludeWorkflowTasks === true + ? [] + : [ + { + taskId: "healthy-worker", + status: "running" as const, + workflowRunId: "wfr_healthy", + workflowOwnerWorkspaceId: "child", + }, + { + taskId: "orphan-worker", + status: "running" as const, + workflowRunId: "wfr_missing", + workflowOwnerWorkspaceId: "child", + }, + ] + ), + stopDescendantAgentTask, + } as unknown as TaskService; + const ownerWorkflowService = { + listRuns: mock(() => Promise.resolve([healthyRun])), + getRun: mock((input: { runId: string }) => + Promise.resolve(input.runId === "wfr_healthy" ? healthyRun : null) + ), + interruptRun: mock((input: { onRunInterrupted?: (runId: string) => void }) => { + input.onRunInterrupted?.("wfr_healthy"); + return Promise.resolve({ ...healthyRun, status: "interrupted" }); + }), + }; + const emptyWorkflowService = { + listRuns: mock(() => Promise.resolve([])), + getRun: mock(() => Promise.resolve(null)), + interruptRun: mock(() => Promise.resolve(null)), + }; + const tool = createTaskStopTool({ + ...baseConfig, + taskService, + workflowServiceForWorkspace: (ownerWorkspaceId) => + ownerWorkspaceId === "child" ? ownerWorkflowService : emptyWorkflowService, + }); + + expect( + await Promise.resolve(tool.execute!({ task_ids: ["child"] }, mockToolCallOptions)) + ).toEqual({ + results: [ + { + status: "error", + taskId: "child", + error: + "Owning workflow run wfr_missing for active descendant orphan-worker is missing or unreadable", + }, + ], + }); + expect(ownerWorkflowService.interruptRun).toHaveBeenCalledTimes(1); + expect(stopDescendantAgentTask).toHaveBeenCalledTimes(1); + }); + it("does not start termination when the signal is already aborted", async () => { using tempDir = new TestTempDir("test-task-terminate-preaborted"); const baseConfig = createTestToolConfig(tempDir.path, { workspaceId: "root-workspace" }); diff --git a/src/node/services/tools/task_stop.ts b/src/node/services/tools/task_stop.ts index 2badf882b2a..1f25ad89946 100644 --- a/src/node/services/tools/task_stop.ts +++ b/src/node/services/tools/task_stop.ts @@ -132,8 +132,9 @@ async function interruptWorkflowRunsOwnedByAgentTaskTree( } const descendants = listDescendants(taskId); - const userOwnedDescendants = listDescendants(taskId, { excludeWorkflowTasks: true }); - const userOwnedTaskIds = new Set(userOwnedDescendants.map((task) => task.taskId)); + const userOwnedTaskIds = new Set( + listDescendants(taskId, { excludeWorkflowTasks: true }).map((task) => task.taskId) + ); const activeWorkflowOwnedDescendants = descendants.filter( (task) => !userOwnedTaskIds.has(task.taskId) && @@ -142,14 +143,53 @@ async function interruptWorkflowRunsOwnedByAgentTaskTree( task.status === "running" || task.status === "awaiting_report") ); + const workflowServices = new Map(); + const resolveWorkflowService = (ownerWorkspaceId: string): ToolWorkflowService | null => { + if (workflowServices.has(ownerWorkspaceId)) { + return workflowServices.get(ownerWorkspaceId) ?? null; + } + const service = + config.workflowServiceForWorkspace?.(ownerWorkspaceId) ?? + (ownerWorkspaceId === config.workspaceId ? config.workflowService : null) ?? + null; + workflowServices.set(ownerWorkspaceId, service); + return service; + }; + const interruptedRunKeys = new Set(); + const interruptOwnedRun = async ( + ownerWorkspaceId: string, + runId: string, + workflowService: ToolWorkflowService + ): Promise => { + const runKey = `${ownerWorkspaceId}\u0000${runId}`; + if (interruptedRunKeys.has(runKey)) { + return null; + } + const outcome = await interruptWorkflowRun(workflowService, ownerWorkspaceId, runId, { + deferTaskSweep: true, + lockAlreadyHeld: true, + onRunInterrupted: (interruptedRunId) => deferredWorkflowRunIds.push(interruptedRunId), + }); + if (outcome.status === "error") { + return outcome.error; + } + if (outcome.status === "not_found") { + return `Workflow run ${runId} disappeared before it could be stopped`; + } + interruptedRunKeys.add(runKey); + if (outcome.status === "stopped") { + deferredWorkflowRunIds.push(runId); + } + return null; + }; - let activeRunCount = 0; + // A descendant workspace may own an independent background workflow even when that workspace is + // itself workflow-owned. Scan every resolvable descendant session, but do not require workflow + // services for ordinary task trees when dynamic workflows are unavailable. for (const ownerWorkspaceId of [taskId, ...descendants.map((task) => task.taskId)]) { - const workflowService = - config.workflowServiceForWorkspace?.(ownerWorkspaceId) ?? - (ownerWorkspaceId === config.workspaceId ? config.workflowService : null); + const workflowService = resolveWorkflowService(ownerWorkspaceId); if (workflowService?.listRuns == null || workflowService.interruptRun == null) { - return `Workflow service not available for descendant workspace ${ownerWorkspaceId}`; + continue; } const rawRuns = await workflowService.listRuns({ workspaceId: ownerWorkspaceId }); for (const rawRun of rawRuns) { @@ -162,33 +202,42 @@ async function interruptWorkflowRunsOwnedByAgentTaskTree( ) { continue; } - - activeRunCount += 1; - const outcome = await interruptWorkflowRun( - workflowService, - ownerWorkspaceId, - parsedRun.data.id, - { - deferTaskSweep: true, - lockAlreadyHeld: true, - onRunInterrupted: (runId) => deferredWorkflowRunIds.push(runId), - } - ); - if (outcome.status === "stopped") { - deferredWorkflowRunIds.push(parsedRun.data.id); - } - if (outcome.status === "error") { - return outcome.error; - } - if (outcome.status === "not_found") { - return `Workflow run ${parsedRun.data.id} disappeared before it could be stopped`; - } + const error = await interruptOwnedRun(ownerWorkspaceId, parsedRun.data.id, workflowService); + if (error != null) return error; } } - if (activeWorkflowOwnedDescendants.length > 0 && activeRunCount === 0) { - return "Active workflow-owned descendants have no active owning workflow run"; + // Correlate each still-active workflow worker to its exact owning run. A healthy run elsewhere in + // the tree must not mask a missing, unreadable, or already-terminal owner for this worker. + for (const worker of activeWorkflowOwnedDescendants) { + const ownerWorkspaceId = worker.workflowOwnerWorkspaceId; + const runId = worker.workflowRunId; + if (ownerWorkspaceId == null || runId == null) { + return `Active workflow-owned descendant ${worker.taskId} is missing workflow ownership metadata`; + } + const runKey = `${ownerWorkspaceId}\u0000${runId}`; + if (interruptedRunKeys.has(runKey)) { + continue; + } + const workflowService = resolveWorkflowService(ownerWorkspaceId); + if (workflowService?.getRun == null || workflowService.interruptRun == null) { + return `Workflow service not available for active workflow-owned descendant ${worker.taskId}`; + } + const rawRun = await workflowService.getRun({ workspaceId: ownerWorkspaceId, runId }); + const parsedRun = WorkflowRunRecordSchema.safeParse(rawRun); + if (!parsedRun.success) { + return `Owning workflow run ${runId} for active descendant ${worker.taskId} is missing or unreadable`; + } + if ( + !isActiveWorkflowRunStatus(parsedRun.data.status) && + parsedRun.data.status !== "interrupted" + ) { + return `Owning workflow run ${runId} for active descendant ${worker.taskId} is already ${parsedRun.data.status}`; + } + const error = await interruptOwnedRun(ownerWorkspaceId, runId, workflowService); + if (error != null) return error; } + return null; } From 6eaa8236db458a746c50858b246305abe9512784 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 21:23:19 -0500 Subject: [PATCH 08/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20reject=20workflow?= =?UTF-8?q?=20starts=20for=20missing=20workspaces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevent stale or removal-racing workflow requests from recreating session state after their workspace has been deleted. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- src/node/services/taskService.test.ts | 19 +++++++++++++++++++ src/node/services/taskService.ts | 8 ++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6b929d05148..2db56bdfd9f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -13170,6 +13170,25 @@ describe("TaskService", () => { expect(nestedTasks.every((workspace) => workspace.taskStatus === "queued")).toBe(true); }); + test("withWorkspaceOwnedWorkStartLock rejects missing workspaces", async () => { + const config = await createTestConfig(rootDir); + const { taskService } = createTaskServiceHarness(config); + const operation = mock(() => Promise.resolve("started")); + + let startError: unknown; + try { + await taskService.withWorkspaceOwnedWorkStartLock("removed-workspace", operation); + } catch (error: unknown) { + startError = error; + } + + expect(startError).toBeInstanceOf(Error); + expect((startError as Error).message).toBe( + "Cannot start workflow work from a missing workspace" + ); + expect(operation).not.toHaveBeenCalled(); + }); + test("task tree lifecycle locks serialize descendants with their ancestor", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index d085a33a14a..71d65800cd7 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2109,10 +2109,10 @@ export class TaskService { { await using _lock = await this.mutex.acquire(); const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); - if ( - entry != null && - isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt) - ) { + if (entry == null) { + throw new Error("Cannot start workflow work from a missing workspace"); + } + if (isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt)) { throw new Error("Cannot start workflow work from an archived workspace"); } if ( From 884c46ab6d5ce53f3d63af44090f5d4363cc5884 Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 21:37:04 -0500 Subject: [PATCH 09/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20descenda?= =?UTF-8?q?nts=20on=20rejected=20removals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validate parent and descendant eligibility before deleting archived workflow workers so rejected task_remove calls have no destructive side effects. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- src/node/services/taskService.test.ts | 67 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 52 +++++++++++++++------ 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 2db56bdfd9f..5f1b490e741 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -13438,6 +13438,73 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, parentTaskId)).toBeUndefined(); }); + test("rejected parent removals preserve archived workflow-owned descendants", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const ownerWorkspaceId = "owner-preserve-archived-workflow"; + const activeParentId = "active-parent-preserve-workflow"; + const activeWorkflowChildId = "active-parent-workflow-child"; + const blockedParentId = "blocked-parent-preserve-workflow"; + const userChildId = "blocked-parent-user-child"; + const blockedWorkflowChildId = "blocked-parent-workflow-child"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "owner", ownerWorkspaceId), + projectWorkspace(projectPath, "active-parent", activeParentId, { + parentWorkspaceId: ownerWorkspaceId, + taskStatus: "running", + }), + projectWorkspace(projectPath, "active-workflow-child", activeWorkflowChildId, { + parentWorkspaceId: activeParentId, + taskStatus: "interrupted", + workflowTask: { runId: "wfr_active_parent", stepId: "worker" }, + archivedAt: "2026-08-16T00:00:00.000Z", + }), + projectWorkspace(projectPath, "blocked-parent", blockedParentId, { + parentWorkspaceId: ownerWorkspaceId, + taskStatus: "reported", + }), + projectWorkspace(projectPath, "user-child", userChildId, { + parentWorkspaceId: blockedParentId, + taskStatus: "reported", + }), + projectWorkspace(projectPath, "blocked-workflow-child", blockedWorkflowChildId, { + parentWorkspaceId: blockedParentId, + taskStatus: "interrupted", + workflowTask: { runId: "wfr_blocked_parent", stepId: "worker" }, + archivedAt: "2026-08-16T00:00:00.000Z", + }), + ], + testTaskSettings() + ); + const remove = mock((): Promise> => Promise.resolve(Ok(undefined))); + const { workspaceService } = createWorkspaceServiceMocks({ remove }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + expect( + await taskService.removeInactiveDescendantAgentTask(ownerWorkspaceId, activeParentId) + ).toMatchObject({ success: true, data: { status: "active", taskId: activeParentId } }); + const blockedRemoval = await taskService.removeInactiveDescendantAgentTask( + ownerWorkspaceId, + blockedParentId + ); + expect(blockedRemoval).toMatchObject({ + success: true, + data: { status: "error", taskId: blockedParentId }, + }); + if (!blockedRemoval.success || blockedRemoval.data.status !== "error") { + throw new Error("Expected blocked parent removal to return an error lifecycle result"); + } + expect(new Set(blockedRemoval.data.descendantTaskIds ?? [])).toEqual( + new Set([userChildId, blockedWorkflowChildId]) + ); + expect(remove).not.toHaveBeenCalled(); + expect(findWorkspaceInConfig(config, activeWorkflowChildId)).toBeTruthy(); + expect(findWorkspaceInConfig(config, blockedWorkflowChildId)).toBeTruthy(); + }); + test("requestAgentFinalReportForTimeout records finalization token only after prompt send succeeds", async () => { const config = await createTestConfig(rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 71d65800cd7..6da013c4d29 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8619,21 +8619,31 @@ export class TaskService { workspaceId: taskId, ...(displayName != null ? { displayName } : {}), }; - // Workflow-owned workers are hidden from public lifecycle tools, but their archived - // workspaces must not permanently block removal of the user-owned parent subtree. - const workflowCleanupResult = - await this.removeArchivedWorkflowOwnedDescendantsUnderLifecycleLock(taskId); - if (!workflowCleanupResult.success) { + if ( + this.isActiveAgentTaskEntry({ ...entry.workspace, projectPath: entry.projectPath }) || + this.aiService.isStreaming(taskId) + ) { return Ok({ - status: "error", + status: "active", action: "remove", ...target, - error: workflowCleanupResult.error, + activeTaskIds: [taskId], + note: "Stop the sub-agent before removing it.", }); } const descendantTaskIds = this.listDescendantAgentTasks(taskId).map((task) => task.taskId); - if (descendantTaskIds.length > 0) { + const blockingDescendantTaskIds = descendantTaskIds.filter((descendantTaskId) => { + const descendant = index.byId.get(descendantTaskId); + return ( + descendant == null || + !this.isWorkflowOwnedTaskUsingIndex(index, descendantTaskId) || + !isWorkspaceArchived(descendant.archivedAt, descendant.unarchivedAt) || + this.isActiveAgentTaskEntry(descendant) || + this.aiService.isStreaming(descendantTaskId) + ); + }); + if (blockingDescendantTaskIds.length > 0) { return Ok({ status: "error", action: "remove", @@ -8643,16 +8653,28 @@ export class TaskService { }); } - if ( - this.isActiveAgentTaskEntry({ ...entry.workspace, projectPath: entry.projectPath }) || - this.aiService.isStreaming(taskId) - ) { + // Workflow-owned workers are hidden from public lifecycle tools, but their archived + // workspaces must not permanently block an otherwise-eligible parent removal. + const workflowCleanupResult = + await this.removeArchivedWorkflowOwnedDescendantsUnderLifecycleLock(taskId); + if (!workflowCleanupResult.success) { return Ok({ - status: "active", + status: "error", action: "remove", ...target, - activeTaskIds: [taskId], - note: "Stop the sub-agent before removing it.", + error: workflowCleanupResult.error, + }); + } + const remainingDescendantTaskIds = this.listDescendantAgentTasks(taskId).map( + (task) => task.taskId + ); + if (remainingDescendantTaskIds.length > 0) { + return Ok({ + status: "error", + action: "remove", + ...target, + descendantTaskIds: remainingDescendantTaskIds, + error: "Cannot remove a sub-agent while descendant sub-agents remain.", }); } From dcdb7a274b56bf8f5229e46c7a793e333f3f7b6c Mon Sep 17 00:00:00 2001 From: Ammar Date: Sat, 15 Aug 2026 21:46:17 -0500 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20hold=20lifecycle=20?= =?UTF-8?q?lock=20through=20background=20start?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep background workflow creation serialized until the run is durably running and the background runner has been scheduled. --- _Generated with `mux` • Model: `openai:gpt-5.6-sol` • Thinking: `xhigh` • Cost: `$147.89`_ --- .../workflows/WorkflowService.test.ts | 51 +++++++++++++++ .../services/workflows/WorkflowService.ts | 64 ++++++++++--------- 2 files changed, 85 insertions(+), 30 deletions(-) diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index 4396d7a98e9..0a328c0b4ee 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -363,6 +363,57 @@ export default function workflow() { return { reportMarkdown: "done" }; } }); }); + test("holds the lifecycle lock until a background start is durably running", async () => { + using tmp = new DisposableTempDir("workflow-service-background-start-lock"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + let lockHeld = false; + const events: string[] = []; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + withRunStartLock: async (_workspaceId, operation) => { + lockHeld = true; + events.push("lock:start"); + try { + return await operation(); + } finally { + events.push("lock:end"); + lockHeld = false; + } + }, + generateRunId: () => "wfr_background_lock", + runnerId: "runner-a", + }); + + await service.startWorkflowInBackground({ + script: createScript( + `export default function workflow() { return { reportMarkdown: "done" }; }\n` + ), + workspaceId: "workspace-1", + projectTrusted: true, + args: {}, + onRunCreated: ({ status }) => { + expect(lockHeld).toBe(true); + expect(status).toBe("pending"); + events.push("created:pending"); + }, + onBackgroundRunCreated: ({ status, run }) => { + expect(lockHeld).toBe(true); + expect(status).toBe("running"); + expect(run.status).toBe("running"); + events.push("created:running"); + }, + }); + + expect(lockHeld).toBe(false); + expect(events).toEqual(["lock:start", "created:pending", "created:running", "lock:end"]); + }); + test("foreground workflows that self-background persist notify_on_terminal policy", async () => { using tmp = new DisposableTempDir("workflow-service-self-background-notify"); const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 8ca3ab50733..3baee9a3c67 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -636,24 +636,26 @@ export class WorkflowService { } async startWorkflowInBackground(input: StartWorkflowInput): Promise { - const createdRun = await this.createWorkflowRun({ - ...input, - attentionPolicy: "notify_on_terminal", + return await this.withWorkflowRunStartLock(input.workspaceId, async () => { + const createdRun = await this.createWorkflowRunUnlocked({ + ...input, + attentionPolicy: "notify_on_terminal", + }); + const runId = createdRun.id; + await this.notifyRunStatusChanged(createdRun); + await input.onRunCreated?.({ runId, status: "pending", result: null, run: createdRun }); + const run = await this.runStore.appendStatus( + runId, + "running", + this.clock?.nowIso() ?? new Date().toISOString() + ); + await this.notifyRunStatusChanged(run); + await input.onBackgroundRunCreated?.({ runId, status: "running", result: null, run }); + void this.runInBackground(runId, "Background workflow run failed:", { + projectTrusted: input.projectTrusted, + }).catch(() => undefined); + return { runId, status: "running", result: null }; }); - const runId = createdRun.id; - await this.notifyRunStatusChanged(createdRun); - await input.onRunCreated?.({ runId, status: "pending", result: null, run: createdRun }); - const run = await this.runStore.appendStatus( - runId, - "running", - this.clock?.nowIso() ?? new Date().toISOString() - ); - await this.notifyRunStatusChanged(run); - await input.onBackgroundRunCreated?.({ runId, status: "running", result: null, run }); - void this.runInBackground(runId, "Background workflow run failed:", { - projectTrusted: input.projectTrusted, - }).catch(() => undefined); - return { runId, status: "running", result: null }; } async startWorkflow(input: StartWorkflowInput): Promise { @@ -893,6 +895,12 @@ export class WorkflowService { } private async createWorkflowRun(input: StartWorkflowInput): Promise { + return await this.withWorkflowRunStartLock(input.workspaceId, async () => + this.createWorkflowRunUnlocked(input) + ); + } + + private async createWorkflowRunUnlocked(input: StartWorkflowInput): Promise { assert( input.workspaceId.length > 0, "WorkflowService.createWorkflowRun: workspaceId is required" @@ -903,19 +911,15 @@ export class WorkflowService { const normalized = normalizeWorkflowArgsForSource(input.script.source, input.args, { defaultArgs: input.defaultArgs, }); - const createRun = async () => - await this.runStore.createRun({ - id: runId, - workspaceId: input.workspaceId, - workflow: buildWorkflowScriptDescriptor(input.script), - source: input.script.source, - args: normalized.args, - ...(input.attentionPolicy != null ? { attentionPolicy: input.attentionPolicy } : {}), - now: this.clock?.nowIso() ?? new Date().toISOString(), - }); - return this.withRunStartLock != null - ? await this.withRunStartLock(input.workspaceId, createRun) - : await createRun(); + return await this.runStore.createRun({ + id: runId, + workspaceId: input.workspaceId, + workflow: buildWorkflowScriptDescriptor(input.script), + source: input.script.source, + args: normalized.args, + ...(input.attentionPolicy != null ? { attentionPolicy: input.attentionPolicy } : {}), + now: this.clock?.nowIso() ?? new Date().toISOString(), + }); } private async runInBackground(