feat(editor): correct, add and time words in the transcript

electron/ai-edition/agent-tools.test.ts

@@ -188,6 +188,7 @@ describe("the mutating-tool table", () => {
 				"setClipRange",
 				"setSpeed",
 				"setTrim",
+				"setWordText",
 				"setZoom",
 			].sort(),
 		);
@@ -2054,3 +2055,159 @@ describe("setZoom answers for the focus it kept", () => {
 		expect(result.resultJson).not.toContain("cursorAnchor");
 	});
 });
+
+// ─── Correcting a word from the chat ─────────────────────────────
+// The model could READ the transcript and CUT it, and that was all. Asked to fix a
+// misheard name it had exactly one tool that touched a word — addTrim — which removes the
+// audio with it. These two close that: one read that hands out word ids, one write that
+// changes text and nothing else.
+
+/** A transcript with real words, one of them already corrected by the user. */
+function documentWithWords(): AxcutDocument {
+	const base = fixtureDocument();
+	return {
+		...base,
+		transcripts: [
+			{
+				assetId: "asset_1",
+				language: "en",
+				segments: [
+					{
+						id: "seg_1",
+						kind: "speech",
+						startSec: 0,
+						endSec: 3,
+						text: "I use Cuber Nettes",
+						wordIds: ["word_1", "word_2", "word_3"],
+					},
+				],
+				words: [
+					{ id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 1, text: "I" },
+					{ id: "word_2", segmentId: "seg_1", startSec: 1, endSec: 2, text: "use" },
+					{
+						id: "word_3",
+						segmentId: "seg_1",
+						startSec: 2,
+						endSec: 3,
+						text: "Cuber Nettes",
+					},
+				],
+			},
+		],
+	};
+}
+
+function run(document: AxcutDocument, name: string, args: unknown) {
+	return executeAgentTool(document, name, JSON.stringify(args), { editsAllowed: true });
+}
+
+describe("getTranscriptWords", () => {
+	it("hands out the ids setWordText takes", () => {
+		const result = run(documentWithWords(), "getTranscriptWords", {});
+		const payload = JSON.parse(result.resultJson) as {
+			words: Array<{ id: string; text: string }>;
+			total: number;
+		};
+		expect(result.ok).toBe(true);
+		expect(payload.total).toBe(3);
+		expect(payload.words.map((w) => w.id)).toEqual(["word_1", "word_2", "word_3"]);
+	});
+
+	// A half-hour transcript is ~70k tokens. Fixing one name should cost one phrase.
+	it("returns only the words touching the span it is given", () => {
+		const result = run(documentWithWords(), "getTranscriptWords", { startSec: 2, endSec: 3 });
+		const payload = JSON.parse(result.resultJson) as {
+			words: Array<{ id: string }>;
+			total: number;
+		};
+		// Touching counts: `word_2` ends exactly where the span begins. Inclusive on
+		// purpose — a word with no duration at all (one the user typed in) sits on a
+		// single point, and a strict overlap would drop it from every span it meets.
+		expect(payload.words.map((w) => w.id)).toEqual(["word_2", "word_3"]);
+		// `total` still reports the whole transcript, so a filtered read never reads as
+		// the entire thing.
+		expect(payload.total).toBe(3);
+	});
+
+	it("says nothing about provenance for a plainly transcribed word", () => {
+		const result = run(documentWithWords(), "getTranscriptWords", {});
+		const payload = JSON.parse(result.resultJson) as { words: Array<Record<string, unknown>> };
+		expect(payload.words[0]).not.toHaveProperty("source");
+		expect(payload.words[0]).not.toHaveProperty("originalText");
+	});
+
+	it("names what the transcriber had heard, once a word is corrected", () => {
+		const corrected = run(documentWithWords(), "setWordText", {
+			wordId: "word_3",
+			text: "Kubernetes",
+		});
+		const result = run(corrected.document as AxcutDocument, "getTranscriptWords", {});
+		const payload = JSON.parse(result.resultJson) as {
+			words: Array<{ id: string; source?: string; originalText?: string }>;
+		};
+		expect(payload.words.find((w) => w.id === "word_3")).toMatchObject({
+			source: "user",
+			originalText: "Cuber Nettes",
+		});
+	});
+
+	it("refuses an asset with no transcript instead of answering with nothing", () => {
+		const result = run({ ...fixtureDocument(), transcripts: [] }, "getTranscriptWords", {});
+		expect(result.ok).toBe(false);
+		expect(result.resultJson).toContain("No transcript");
+	});
+});
+
+describe("setWordText", () => {
+	it("changes the text and leaves the timeline alone", () => {
+		const before = documentWithWords();
+		const result = run(before, "setWordText", { wordId: "word_3", text: "Kubernetes" });
+		expect(result.ok).toBe(true);
+		const next = result.document as AxcutDocument;
+		expect(next.transcripts[0].words.find((w) => w.id === "word_3")?.text).toBe("Kubernetes");
+		expect(next.timeline).toEqual(before.timeline);
+		expect(next.transcripts[0].segments[0].text).toBe("I use Kubernetes");
+	});
+
+	// The document carries the transcript twice; a write that reaches only one leaves the
+	// legacy mirror serving the old text forever.
+	it("writes the legacy mirror too", () => {
+		const result = run(documentWithWords(), "setWordText", {
+			wordId: "word_3",
+			text: "Kubernetes",
+		});
+		const next = result.document as AxcutDocument;
+		expect(next.transcript).toBe(next.transcripts.find((t) => t.assetId === "asset_1"));
+	});
+
+	it("empties a word without cutting the speech around it", () => {
+		const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "" });
+		const next = result.document as AxcutDocument;
+		expect(next.transcripts[0].words.find((w) => w.id === "word_2")?.text).toBe("");
+		expect(next.transcripts[0].segments[0].text).toBe("I Cuber Nettes");
+		expect(JSON.parse(result.resultJson)).toMatchObject({ blanked: true });
+	});
+
+	it("points an unknown id at the read that hands them out", () => {
+		const result = run(documentWithWords(), "setWordText", { wordId: "seg_1", text: "x" });
+		expect(result.ok).toBe(false);
+		// `seg_1` is a real id — of a SEGMENT. The two namespaces are the trap.
+		expect(result.resultJson).toContain("getTranscriptWords");
+	});
+
+	it("refuses a write that would change nothing", () => {
+		const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "use" });
+		expect(result.ok).toBe(false);
+		expect(result.document).toBeUndefined();
+	});
+
+	it("is a consented edit, not a read", () => {
+		const result = executeAgentTool(
+			documentWithWords(),
+			"setWordText",
+			JSON.stringify({ wordId: "word_3", text: "Kubernetes" }),
+			{ editsAllowed: false },
+		);
+		expect(result.document).toBeUndefined();
+	});
+});

electron/ai-edition/agent-tools.ts

@@ -26,6 +26,7 @@ import {
 	replaceTimeline,
 	setClipSourceRange,
 } from "../../src/lib/ai-edition/document/timeline";
+import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript";
 import type { AxcutDocument } from "../../src/lib/ai-edition/schema";
 import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera";
 import {
@@ -490,6 +491,18 @@ export const setCameraFullscreenArgs = z.object({
 	endSec: secondsSchema.optional(),
 });
 
+export const getTranscriptWordsArgs = z.object({
+	assetId: z.string().min(1).optional(),
+	startSec: secondsSchema.optional(),
+	endSec: secondsSchema.optional(),
+});
+
+export const setWordTextArgs = z.object({
+	wordId: z.string().min(1),
+	text: z.string(),
+	assetId: z.string().min(1).optional(),
+});
+
 export const removeTrimArgs = z.object({
 	trimRangeId: z.string().min(1),
 });
@@ -525,7 +538,9 @@ export const removeClipArgs = z.object({
 export const OPENSCREEN_TOOL_NAMES = [
 	"getCurrentDocument",
 	"getTranscript",
+	"getTranscriptWords",
 	"getCursorTrack",
+	"setWordText",
 	"addTrim",
 	"addTrims",
 	"setTrim",
@@ -592,6 +607,9 @@ export const PHANTOM_TOOL_NAMES = [
  * remaining surfaces (descriptions, built tools, executor cases) to each other.
  */
 export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set([
+	// Writes the transcript, not the timeline — but it writes the document, so it is a
+	// consented edit like any other.
+	"setWordText",
 	"addTrim",
 	"addTrims",
 	"addZooms",
@@ -1203,6 +1221,96 @@ export function executeAgentTool(
 			};
 		}
 
+		// The word-level read. `getTranscript` answers in SEGMENTS, whose ids belong to a
+		// different namespace than the words — so on its own it cannot address anything
+		// `setWordText` takes. This is the one that can. It is separate rather than folded
+		// in because a whole transcript is already ~70k tokens and most turns never touch a
+		// word; the span filter is there so fixing one name costs one phrase, not the film.
+		case "getTranscriptWords": {
+			const parsed = getTranscriptWordsArgs.safeParse(args);
+			if (!parsed.success) return failure(parsed.error.message);
+			const assetId =
+				parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id;
+			const transcript =
+				document.transcripts.find((t) => t.assetId === assetId) ??
+				(document.transcript?.assetId === assetId ? document.transcript : null);
+			if (!transcript) {
+				return failure(`No transcript for asset ${assetId ?? "(none)"}.`);
+			}
+			const from = parsed.data.startSec ?? Number.NEGATIVE_INFINITY;
+			const to = parsed.data.endSec ?? Number.POSITIVE_INFINITY;
+			const words = transcript.words
+				.filter((word) => word.endSec >= from && word.startSec <= to)
+				.map((word) => ({
+					id: word.id,
+					text: word.text,
+					startSec: word.startSec,
+					endSec: word.endSec,
+					// Only the words that are NOT plain transcription say so, so the common
+					// case costs nothing to read.
+					...(word.source ? { source: word.source } : {}),
+					...(word.originalText !== undefined ? { originalText: word.originalText } : {}),
+				}));
+			return {
+				ok: true,
+				resultJson: JSON.stringify({
+					assetId,
+					language: transcript.language,
+					total: transcript.words.length,
+					returned: words.length,
+					words,
+				}),
+			};
+		}
+
+		// Correcting what the transcriber HEARD. This writes text and nothing else: the
+		// captions follow it, the film does not move. The tool for making a spoken word go
+		// away is addTrim, which removes its audio with it.
+		case "setWordText": {
+			const parsed = setWordTextArgs.safeParse(args);
+			if (!parsed.success) return failure(parsed.error.message);
+			const assetId =
+				parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id;
+			if (!assetId) return failure("Project has no assets — nothing to correct.");
+			const { wordId, text } = parsed.data;
+			const transcript = document.transcripts.find((t) => t.assetId === assetId);
+			const before = transcript?.words.find((word) => word.id === wordId);
+			if (!before) {
+				return failure(
+					`No word ${wordId} in the transcript for asset ${assetId}. ` +
+						`Call getTranscriptWords to read the ids.`,
+				);
+			}
+			if (before.text === text) {
+				return failure(`Word ${wordId} already reads "${text}" — nothing to change.`);
+			}
+			let next: AxcutDocument;
+			try {
+				next = setDocumentWordText(document, assetId, wordId, text);
+			} catch (error) {
+				return failure(error instanceof Error ? error.message : String(error));
+			}
+			const after = next.transcripts
+				.find((t) => t.assetId === assetId)
+				?.words.find((word) => word.id === wordId);
+			return {
+				ok: true,
+				document: next,
+				resultJson: JSON.stringify({
+					wordId,
+					assetId,
+					text: after?.text ?? text,
+					was: before.text,
+					// Absent once the word is back to what the transcriber said — the pair is
+					// cleared on that round trip, and the model should be able to see it.
+					originalText: after?.originalText,
+					blanked: text.trim().length === 0,
+				}),
+				summary:
+					text.trim().length === 0 ? `blanked "${before.text}"` : `"${before.text}" → "${text}"`,
+			};
+		}
+
 		case "addTrim": {
 			const parsed = addTrimArgs.safeParse(args);
 			if (!parsed.success) return failure(parsed.error.message);

electron/ai-edition/deep-agent/service.test.ts

@@ -57,7 +57,9 @@ const PHANTOM_TOOLS: readonly string[] = PHANTOM_TOOL_NAMES;
 const ARGS: Record<string, unknown> = {
 	getCurrentDocument: {},
 	getTranscript: {},
+	getTranscriptWords: {},
 	getCursorTrack: {},
+	setWordText: { wordId: "word_1", text: "Hullo" },
 	addTrim: { startSec: 1, endSec: 2 },
 	addTrims: { ranges: [{ startSec: 1, endSec: 2 }] },
 	setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 },
@@ -109,9 +111,18 @@ function fixtureDocument(): AxcutDocument {
 				assetId: "asset_1",
 				language: "en",
 				segments: [
-					{ id: "seg_1", kind: "speech", startSec: 0, endSec: 5, text: "Hello", wordIds: [] },
+					{
+						id: "seg_1",
+						kind: "speech",
+						startSec: 0,
+						endSec: 5,
+						text: "Hello",
+						// A real word, so `setWordText` lands on its WRITE branch in the table
+						// below — a tool refused for an unknown id would look non-mutating.
+						wordIds: ["word_1"],
+					},
 				],
-				words: [],
+				words: [{ id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 5, text: "Hello" }],
 			},
 		],
 		timeline: {
@@ -351,6 +362,7 @@ describe("one description of the tools, not two", () => {
 		expect(OPENSCREEN_TOOLS.filter((n) => !isMutatingTool(n))).toEqual([
 			"getCurrentDocument",
 			"getTranscript",
+			"getTranscriptWords",
 			"getCursorTrack",
 		]);
 	});

electron/ai-edition/deep-agent/service.ts

@@ -37,6 +37,7 @@ import {
 	executeAgentTool,
 	getCursorTrackArgs,
 	getTranscriptArgs,
+	getTranscriptWordsArgs,
 	isMutatingTool,
 	moveClipArgs,
 	removeClipArgs,
@@ -49,6 +50,7 @@ import {
 	setClipRangeArgs,
 	setSpeedArgs,
 	setTrimArgs,
+	setWordTextArgs,
 	setZoomArgs,
 } from "../agent-tools";
 import {
@@ -143,6 +145,10 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
 		"Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.",
 	getCursorTrack:
 		"Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.",
+	getTranscriptWords:
+		'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.',
+	setWordText:
+		"Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use addTrim, which cuts the audio with it.",
 	addTrim:
 		"Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).",
 	addTrims:
@@ -323,7 +329,9 @@ export function buildTools(
 	return [
 		build("getCurrentDocument", z.object({})),
 		build("getTranscript", getTranscriptArgs),
+		build("getTranscriptWords", getTranscriptWordsArgs),
 		build("getCursorTrack", getCursorTrackArgs),
+		build("setWordText", setWordTextArgs),
 		build("addTrim", addTrimArgs),
 		build("addTrims", addTrimsArgs),
 		build("setTrim", setTrimArgs),

src/components/ai-edition/EditorEmptyState.test.tsx

@@ -47,6 +47,7 @@ const sampleDoc = vi.hoisted(
 			clips: [],
 			gaps: [],
 			trimRanges: [],
+			insertRanges: [],
 			muteRanges: [],
 			speedRanges: [],
 			captionRanges: [],