diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 8f8361e59..15e121f93 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -188,6 +188,7 @@ describe("the mutating-tool table", () => { "setClipRange", "setSpeed", "setTrim", + "setWordText", "setZoom", ].sort(), ); @@ -2054,3 +2055,159 @@ describe("setZoom answers for the focus it kept", () => { expect(result.resultJson).not.toContain("cursorAnchor"); }); }); + +// ─── Correcting a word from the chat ───────────────────────────── +// The model could READ the transcript and CUT it, and that was all. Asked to fix a +// misheard name it had exactly one tool that touched a word — addTrim — which removes the +// audio with it. These two close that: one read that hands out word ids, one write that +// changes text and nothing else. + +/** A transcript with real words, one of them already corrected by the user. */ +function documentWithWords(): AxcutDocument { + const base = fixtureDocument(); + return { + ...base, + transcripts: [ + { + assetId: "asset_1", + language: "en", + segments: [ + { + id: "seg_1", + kind: "speech", + startSec: 0, + endSec: 3, + text: "I use Cuber Nettes", + wordIds: ["word_1", "word_2", "word_3"], + }, + ], + words: [ + { id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 1, text: "I" }, + { id: "word_2", segmentId: "seg_1", startSec: 1, endSec: 2, text: "use" }, + { + id: "word_3", + segmentId: "seg_1", + startSec: 2, + endSec: 3, + text: "Cuber Nettes", + }, + ], + }, + ], + }; +} + +function run(document: AxcutDocument, name: string, args: unknown) { + return executeAgentTool(document, name, JSON.stringify(args), { editsAllowed: true }); +} + +describe("getTranscriptWords", () => { + it("hands out the ids setWordText takes", () => { + const result = run(documentWithWords(), "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string; text: string }>; + total: number; + }; + expect(result.ok).toBe(true); + expect(payload.total).toBe(3); + expect(payload.words.map((w) => w.id)).toEqual(["word_1", "word_2", "word_3"]); + }); + + // A half-hour transcript is ~70k tokens. Fixing one name should cost one phrase. + it("returns only the words touching the span it is given", () => { + const result = run(documentWithWords(), "getTranscriptWords", { startSec: 2, endSec: 3 }); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string }>; + total: number; + }; + // Touching counts: `word_2` ends exactly where the span begins. Inclusive on + // purpose — a word with no duration at all (one the user typed in) sits on a + // single point, and a strict overlap would drop it from every span it meets. + expect(payload.words.map((w) => w.id)).toEqual(["word_2", "word_3"]); + // `total` still reports the whole transcript, so a filtered read never reads as + // the entire thing. + expect(payload.total).toBe(3); + }); + + it("says nothing about provenance for a plainly transcribed word", () => { + const result = run(documentWithWords(), "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { words: Array> }; + expect(payload.words[0]).not.toHaveProperty("source"); + expect(payload.words[0]).not.toHaveProperty("originalText"); + }); + + it("names what the transcriber had heard, once a word is corrected", () => { + const corrected = run(documentWithWords(), "setWordText", { + wordId: "word_3", + text: "Kubernetes", + }); + const result = run(corrected.document as AxcutDocument, "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string; source?: string; originalText?: string }>; + }; + expect(payload.words.find((w) => w.id === "word_3")).toMatchObject({ + source: "user", + originalText: "Cuber Nettes", + }); + }); + + it("refuses an asset with no transcript instead of answering with nothing", () => { + const result = run({ ...fixtureDocument(), transcripts: [] }, "getTranscriptWords", {}); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("No transcript"); + }); +}); + +describe("setWordText", () => { + it("changes the text and leaves the timeline alone", () => { + const before = documentWithWords(); + const result = run(before, "setWordText", { wordId: "word_3", text: "Kubernetes" }); + expect(result.ok).toBe(true); + const next = result.document as AxcutDocument; + expect(next.transcripts[0].words.find((w) => w.id === "word_3")?.text).toBe("Kubernetes"); + expect(next.timeline).toEqual(before.timeline); + expect(next.transcripts[0].segments[0].text).toBe("I use Kubernetes"); + }); + + // The document carries the transcript twice; a write that reaches only one leaves the + // legacy mirror serving the old text forever. + it("writes the legacy mirror too", () => { + const result = run(documentWithWords(), "setWordText", { + wordId: "word_3", + text: "Kubernetes", + }); + const next = result.document as AxcutDocument; + expect(next.transcript).toBe(next.transcripts.find((t) => t.assetId === "asset_1")); + }); + + it("empties a word without cutting the speech around it", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "" }); + const next = result.document as AxcutDocument; + expect(next.transcripts[0].words.find((w) => w.id === "word_2")?.text).toBe(""); + expect(next.transcripts[0].segments[0].text).toBe("I Cuber Nettes"); + expect(JSON.parse(result.resultJson)).toMatchObject({ blanked: true }); + }); + + it("points an unknown id at the read that hands them out", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "seg_1", text: "x" }); + expect(result.ok).toBe(false); + // `seg_1` is a real id — of a SEGMENT. The two namespaces are the trap. + expect(result.resultJson).toContain("getTranscriptWords"); + }); + + it("refuses a write that would change nothing", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "use" }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + }); + + it("is a consented edit, not a read", () => { + const result = executeAgentTool( + documentWithWords(), + "setWordText", + JSON.stringify({ wordId: "word_3", text: "Kubernetes" }), + { editsAllowed: false }, + ); + expect(result.document).toBeUndefined(); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 5a0966398..ff444b192 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -26,6 +26,7 @@ import { replaceTimeline, setClipSourceRange, } from "../../src/lib/ai-edition/document/timeline"; +import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript"; import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; import { @@ -490,6 +491,18 @@ export const setCameraFullscreenArgs = z.object({ endSec: secondsSchema.optional(), }); +export const getTranscriptWordsArgs = z.object({ + assetId: z.string().min(1).optional(), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), +}); + +export const setWordTextArgs = z.object({ + wordId: z.string().min(1), + text: z.string(), + assetId: z.string().min(1).optional(), +}); + export const removeTrimArgs = z.object({ trimRangeId: z.string().min(1), }); @@ -525,7 +538,9 @@ export const removeClipArgs = z.object({ export const OPENSCREEN_TOOL_NAMES = [ "getCurrentDocument", "getTranscript", + "getTranscriptWords", "getCursorTrack", + "setWordText", "addTrim", "addTrims", "setTrim", @@ -592,6 +607,9 @@ export const PHANTOM_TOOL_NAMES = [ * remaining surfaces (descriptions, built tools, executor cases) to each other. */ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ + // Writes the transcript, not the timeline — but it writes the document, so it is a + // consented edit like any other. + "setWordText", "addTrim", "addTrims", "addZooms", @@ -1203,6 +1221,96 @@ export function executeAgentTool( }; } + // The word-level read. `getTranscript` answers in SEGMENTS, whose ids belong to a + // different namespace than the words — so on its own it cannot address anything + // `setWordText` takes. This is the one that can. It is separate rather than folded + // in because a whole transcript is already ~70k tokens and most turns never touch a + // word; the span filter is there so fixing one name costs one phrase, not the film. + case "getTranscriptWords": { + const parsed = getTranscriptWordsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const assetId = + parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id; + const transcript = + document.transcripts.find((t) => t.assetId === assetId) ?? + (document.transcript?.assetId === assetId ? document.transcript : null); + if (!transcript) { + return failure(`No transcript for asset ${assetId ?? "(none)"}.`); + } + const from = parsed.data.startSec ?? Number.NEGATIVE_INFINITY; + const to = parsed.data.endSec ?? Number.POSITIVE_INFINITY; + const words = transcript.words + .filter((word) => word.endSec >= from && word.startSec <= to) + .map((word) => ({ + id: word.id, + text: word.text, + startSec: word.startSec, + endSec: word.endSec, + // Only the words that are NOT plain transcription say so, so the common + // case costs nothing to read. + ...(word.source ? { source: word.source } : {}), + ...(word.originalText !== undefined ? { originalText: word.originalText } : {}), + })); + return { + ok: true, + resultJson: JSON.stringify({ + assetId, + language: transcript.language, + total: transcript.words.length, + returned: words.length, + words, + }), + }; + } + + // Correcting what the transcriber HEARD. This writes text and nothing else: the + // captions follow it, the film does not move. The tool for making a spoken word go + // away is addTrim, which removes its audio with it. + case "setWordText": { + const parsed = setWordTextArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const assetId = + parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id; + if (!assetId) return failure("Project has no assets — nothing to correct."); + const { wordId, text } = parsed.data; + const transcript = document.transcripts.find((t) => t.assetId === assetId); + const before = transcript?.words.find((word) => word.id === wordId); + if (!before) { + return failure( + `No word ${wordId} in the transcript for asset ${assetId}. ` + + `Call getTranscriptWords to read the ids.`, + ); + } + if (before.text === text) { + return failure(`Word ${wordId} already reads "${text}" — nothing to change.`); + } + let next: AxcutDocument; + try { + next = setDocumentWordText(document, assetId, wordId, text); + } catch (error) { + return failure(error instanceof Error ? error.message : String(error)); + } + const after = next.transcripts + .find((t) => t.assetId === assetId) + ?.words.find((word) => word.id === wordId); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + wordId, + assetId, + text: after?.text ?? text, + was: before.text, + // Absent once the word is back to what the transcriber said — the pair is + // cleared on that round trip, and the model should be able to see it. + originalText: after?.originalText, + blanked: text.trim().length === 0, + }), + summary: + text.trim().length === 0 ? `blanked "${before.text}"` : `"${before.text}" → "${text}"`, + }; + } + case "addTrim": { const parsed = addTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 7729bd624..4a126ec0a 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -57,7 +57,9 @@ const PHANTOM_TOOLS: readonly string[] = PHANTOM_TOOL_NAMES; const ARGS: Record = { getCurrentDocument: {}, getTranscript: {}, + getTranscriptWords: {}, getCursorTrack: {}, + setWordText: { wordId: "word_1", text: "Hullo" }, addTrim: { startSec: 1, endSec: 2 }, addTrims: { ranges: [{ startSec: 1, endSec: 2 }] }, setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 }, @@ -109,9 +111,18 @@ function fixtureDocument(): AxcutDocument { assetId: "asset_1", language: "en", segments: [ - { id: "seg_1", kind: "speech", startSec: 0, endSec: 5, text: "Hello", wordIds: [] }, + { + id: "seg_1", + kind: "speech", + startSec: 0, + endSec: 5, + text: "Hello", + // A real word, so `setWordText` lands on its WRITE branch in the table + // below — a tool refused for an unknown id would look non-mutating. + wordIds: ["word_1"], + }, ], - words: [], + words: [{ id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 5, text: "Hello" }], }, ], timeline: { @@ -351,6 +362,7 @@ describe("one description of the tools, not two", () => { expect(OPENSCREEN_TOOLS.filter((n) => !isMutatingTool(n))).toEqual([ "getCurrentDocument", "getTranscript", + "getTranscriptWords", "getCursorTrack", ]); }); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index d804a3b1a..72598c09f 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -37,6 +37,7 @@ import { executeAgentTool, getCursorTrackArgs, getTranscriptArgs, + getTranscriptWordsArgs, isMutatingTool, moveClipArgs, removeClipArgs, @@ -49,6 +50,7 @@ import { setClipRangeArgs, setSpeedArgs, setTrimArgs, + setWordTextArgs, setZoomArgs, } from "../agent-tools"; import { @@ -143,6 +145,10 @@ export const TOOL_DESCRIPTIONS: Record = { "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", + getTranscriptWords: + 'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.', + setWordText: + "Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use addTrim, which cuts the audio with it.", addTrim: "Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).", addTrims: @@ -323,7 +329,9 @@ export function buildTools( return [ build("getCurrentDocument", z.object({})), build("getTranscript", getTranscriptArgs), + build("getTranscriptWords", getTranscriptWordsArgs), build("getCursorTrack", getCursorTrackArgs), + build("setWordText", setWordTextArgs), build("addTrim", addTrimArgs), build("addTrims", addTrimsArgs), build("setTrim", setTrimArgs), diff --git a/src/components/ai-edition/EditorEmptyState.test.tsx b/src/components/ai-edition/EditorEmptyState.test.tsx index acb513e4b..a4c6cd2b8 100644 --- a/src/components/ai-edition/EditorEmptyState.test.tsx +++ b/src/components/ai-edition/EditorEmptyState.test.tsx @@ -47,6 +47,7 @@ const sampleDoc = vi.hoisted( clips: [], gaps: [], trimRanges: [], + insertRanges: [], muteRanges: [], speedRanges: [], captionRanges: [], diff --git a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx index c5b45637c..f467fcc92 100644 --- a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx +++ b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx @@ -71,6 +71,7 @@ const DOC: AxcutDocument = { ], gaps: [], trimRanges: [], + insertRanges: [], muteRanges: [], speedRanges: [], captionRanges: [], diff --git a/src/components/ai-edition/ExportDialog.test.ts b/src/components/ai-edition/ExportDialog.test.ts index 3aa8d85b5..fa61fec61 100644 --- a/src/components/ai-edition/ExportDialog.test.ts +++ b/src/components/ai-edition/ExportDialog.test.ts @@ -51,6 +51,7 @@ function doc(assets: AxcutAsset[], clips: AxcutClip[]): AxcutDocument { clips, gaps: [], trimRanges: [], + insertRanges: [], muteRanges: [], speedRanges: [], captionRanges: [], diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index b371d100c..2973e9c9b 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -13,6 +13,12 @@ import { applyProbedDuration, replaceTimeline as replaceTimelineOp, } from "@/lib/ai-edition/document/timeline"; +import { + type InsertSide, + insertDocumentWord, + removeDocumentWords, + setDocumentWordText, +} from "@/lib/ai-edition/document/transcript"; import { isModalOpen } from "@/lib/ai-edition/modalGuard"; import { type AxcutClip, documentSchema } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; @@ -604,6 +610,72 @@ export function NewEditorShell() { [applyTimelineOp], ); + // transcript-pane → the word's own text. Unlike Backspace (which writes a trimRange and + // cuts the media), this writes only `transcript.words[].text`: the captions follow, the + // film is untouched. Queued on the SAME chain as the trims so correcting a word and + // cutting the next one cannot overwrite each other's save. + const handleSetWordText = useCallback( + (assetId: string, wordId: string, text: string) => { + void enqueueTimelineWrite(async () => { + // Read inside the chain: the previous save has resolved by now, so the store + // holds the document this edit has to be applied to. + const doc = useProjectStore.getState().document; + if (!doc) return; + try { + await saveDocument(setDocumentWordText(doc, assetId, wordId, text), { history: true }); + } catch (err) { + // The word or its transcript vanished under the edit (a regeneration landed + // mid-typing). Nothing to retry — say so rather than dropping it silently. + toast.error(te("errors.wordEditFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + + // transcript-pane → a word nobody said. It takes the silence it is dropped into and no + // audio at all, so unlike a cut it changes nothing about the film; today it reaches the + // captions and stops there. + const handleInsertWord = useCallback( + (assetId: string, anchorWordId: string, side: InsertSide, text: string) => { + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc) return; + try { + await saveDocument(insertDocumentWord(doc, assetId, anchorWordId, side, text), { + history: true, + }); + } catch (err) { + toast.error(te("errors.wordInsertFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + + // Deleting inserted words. One save for the whole set, so a Backspace over several of + // them is one Ctrl+Z, and the document layer refuses anything that was actually spoken. + const handleRemoveWords = useCallback( + (assetId: string, wordIds: string[]) => { + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc || wordIds.length === 0) return; + try { + await saveDocument(removeDocumentWords(doc, assetId, wordIds), { history: true }); + } catch (err) { + toast.error(te("errors.wordRemoveFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + const handleSelectProject = useCallback( async (id: string) => { try { @@ -1151,6 +1223,9 @@ export function NewEditorShell() { onSeek: handleSeek, onAddTrimRange: handleAddTrimRange, onRemoveTrimRange: handleRemoveTrimRange, + onSetWordText: handleSetWordText, + onInsertWord: handleInsertWord, + onRemoveWords: handleRemoveWords, onTranscribe: handleTranscribe, canTranscribe: hasAsset, isTranscribing: transcriptGate.state === "pending", @@ -1240,6 +1315,7 @@ export function NewEditorShell() { speedRegions={tl.speedRegions} cameraFullscreenRegions={tl.cameraFullscreenRegions} trimRanges={tl.trimRanges} + insertRanges={document?.timeline?.insertRanges ?? []} selectedZoomRegionId={tl.selection?.kind === "zoom" ? tl.selection.id : null} onZoomFocusChange={tl.updateZoomFocusLive} onZoomFocusCommit={() => void tl.commitZoomFocus()} diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index aaf04f2db..64e199514 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -4,6 +4,7 @@ import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutAnnotationRegion, AxcutClip, + AxcutInsertRange, AxcutTrimRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; @@ -26,6 +27,7 @@ interface PreviewProps { speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; + insertRanges?: AxcutInsertRange[]; selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; @@ -57,6 +59,7 @@ export function Preview({ speedRegions, cameraFullscreenRegions, trimRanges, + insertRanges, selectedZoomRegionId, onZoomFocusChange, onZoomFocusCommit, @@ -183,6 +186,7 @@ export function Preview({ speedRegions={speedRegions} cameraFullscreenRegions={cameraFullscreenRegions} trimRanges={trimRanges} + insertRanges={insertRanges} selectedZoomRegionId={selectedZoomRegionId} onZoomFocusChange={onZoomFocusChange} onZoomFocusCommit={onZoomFocusCommit} diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index 6f59e066f..53abadbb6 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -35,6 +35,7 @@ import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat" import type { AxcutAnnotationRegion, AxcutClip, + AxcutInsertRange, AxcutTrimRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; @@ -70,6 +71,7 @@ interface PreviewCanvasProps { speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; + insertRanges?: AxcutInsertRange[]; selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 2e15f7b12..8df7af234 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -15,12 +15,13 @@ import { MousePointerClick, Sliders, Trash2, + Undo2, } from "lucide-react"; import { type ChangeEvent, type CSSProperties, - type FormEvent, + Fragment, memo, type ClipboardEvent as ReactClipboardEvent, type KeyboardEvent as ReactKeyboardEvent, @@ -39,6 +40,7 @@ import GradientEditor, { type GradientEditorState } from "@/components/ui/gradie import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { collectNativeFormats } from "@/lib/ai-edition/document/outputFormat"; +import type { InsertSide } from "@/lib/ai-edition/document/transcript"; import type { AxcutAsset, AxcutClip, @@ -57,6 +59,7 @@ import { type ClipSection, type ClipWord, findCueWordId, + isInsertedWord, isSilenceWord, type TrimRun, } from "@/lib/ai-edition/timeline/aggregated-transcript"; @@ -704,6 +707,9 @@ export function TranscriptPane({ onSeek, onAddTrimRange, onRemoveTrimRange, + onSetWordText, + onInsertWord, + onRemoveWords, onTranscribe, canTranscribe, isTranscribing, @@ -722,6 +728,15 @@ export function TranscriptPane({ onSeek: (sec: number) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; onRemoveTrimRange: (trimId: string) => void; + /** Rewrite ONE word's text. Takes the bare `AxcutWord.id`, never the clip-scoped + * `ClipWord.id`: the transcript belongs to the asset, so a correction lands on the + * media and shows on every clip that plays it — which is the point. */ + onSetWordText: (assetId: string, wordId: string, text: string) => void; + /** Add a word nobody said, beside the word the caret was resting on. Bare id, as above. */ + onInsertWord: (assetId: string, anchorWordId: string, side: InsertSide, text: string) => void; + /** Delete inserted words. Only ever called with `source: "synth"` ids — a transcribed + * word is cut with a trim, never deleted. */ + onRemoveWords: (assetId: string, wordIds: string[]) => void; onTranscribe: () => void; canTranscribe: boolean; isTranscribing: boolean; @@ -767,13 +782,17 @@ export function TranscriptPane({ // engine, nothing attempted) leaves the button worth pressing. const silentMedia = blocked?.reason === "no-audio"; + // The insert gesture is dev-only until TTS (see openInsertion), so the copy follows + // the same gate: release builds must not advertise a dead gesture. + const helpText = + ts("transcript.help") + (import.meta.env.DEV ? ` ${ts("transcript.helpInsert")}` : ""); + const editingHint = ts( + import.meta.env.DEV ? "transcript.editingHintDev" : "transcript.editingHint", + ); + if (clips.length === 0 || !hasAnyTranscript) { return ( - } - helpText={ts("transcript.help")} - > + } helpText={helpText}>
-
-

{ts("transcript.title")}

-
-
- {sections.map((section, idx) => ( - - ))} -
-
+ } helpText={helpText}> + {/* The gestures are invisible until tried: nothing on a plain word stream says + * that double-click corrects and Backspace cuts. One muted line names them; the + * ? popover above carries the long version (amber inserts, hover-bin restore). */} +

+ {editingHint} +

+ {sections.map((section, idx) => ( + + ))} + ); } @@ -859,6 +889,9 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ onSeek, onAddTrimRange, onRemoveTrimRange, + onSetWordText, + onInsertWord, + onRemoveWords, }: { index: number; section: ClipSection; @@ -867,6 +900,9 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ onSeek: (sec: number) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; onRemoveTrimRange: (trimId: string) => void; + onSetWordText: (assetId: string, wordId: string, text: string) => void; + onInsertWord: (assetId: string, anchorWordId: string, side: InsertSide, text: string) => void; + onRemoveWords: (assetId: string, wordIds: string[]) => void; }) { const ts = useScopedT("settings"); const { clip, asset, words } = section; @@ -939,6 +975,17 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // Only skip words that are currently kept (don't double-skip). const keptRange = rangeWords.filter((w) => w.kept); if (keptRange.length === 0) return; + // An inserted word has no audio to cut, so Backspace deletes it outright. Only a + // range made entirely of inserts takes this path: mixed with spoken words the trim + // covers them anyway — they sit inside its span and read as cut, which is what the + // keystroke asked for. + if (keptRange.every((w) => isInsertedWord(w.word))) { + onRemoveWords( + clip.assetId, + keptRange.map((w) => w.word.id), + ); + return; + } pendingCaretWordIdRef.current = keptRange[0].id; const startSec = Math.min(...keptRange.map((w) => w.word.startSec)); const endSec = Math.max(...keptRange.map((w) => w.word.endSec)); @@ -949,7 +996,7 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ `Skip ${formatMs(startSec * 1000)}-${formatMs(endSec * 1000)} from ${clip.assetId}.`, ); }, - [busy, clip.assetId, trimTarget, onAddTrimRange], + [busy, clip.assetId, trimTarget, onAddTrimRange, onRemoveWords], ); const removeTrimRun = useCallback( @@ -1012,38 +1059,98 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ [cutNativeSelection], ); - const handleBeforeInput = useCallback( - (event: FormEvent) => { - const inputEvent = event.nativeEvent as InputEvent; - if (inputEvent.inputType.startsWith("delete")) { + // The word an insert will sit beside, and what has been typed into it so far. Held on + // the block rather than the word, because the field belongs BETWEEN two words: the id is + // only how it finds its place in the stream. + const [insertion, setInsertion] = useState<{ + clipWordId: string; + side: InsertSide; + draft: string; + } | null>(null); + const insertionAbandonedRef = useRef(false); + + const openInsertion = useCallback( + (seed: string) => { + // ponytail: word insertion ships dev-only until a voice can be synthesized for + // the word — without one it only borrows free silence, and once it creates + // timeline time (the pause gesture) it is a silent freeze frame. Drop this gate + // when TTS lands. + if (!import.meta.env.DEV) return; + if (busy || !seed.trim()) return; + const editor = editorRef.current; + const selection = globalThis.getSelection(); + if (!editor || !selection) return; + if (!editor.contains(selection.anchorNode)) return; + const caret = findInsertionAnchor(editor, selection.anchorNode, selection.anchorOffset); + if (!caret) return; + const anchor = resolveInsertionAnchor(words, caret.clipWordId, caret.side); + if (!anchor) return; + setInsertion({ ...anchor, draft: seed }); + }, + [busy, words], + ); + + const commitInsertion = useCallback(() => { + const pending = insertion; + setInsertion(null); + if (!pending) return; + const text = pending.draft.trim(); + if (!text) return; + const anchor = words.find((w) => w.id === pending.clipWordId); + if (!anchor) return; + onInsertWord(clip.assetId, anchor.word.id, pending.side, text); + }, [insertion, words, onInsertWord, clip.assetId]); + + // Attached to the DOM, not through React's `onBeforeInput`. + // + // React 18 does not build that synthetic event from the native `beforeinput`: it + // derives it from the legacy `textInput`, whose event object is a `TextEvent` and + // carries no `inputType` at all. So the guard that was supposed to keep typed text out + // of the projection threw `Cannot read properties of undefined (reading 'startsWith')` + // on every character, never reached its own `preventDefault`, and let the character + // land in the contentEditable — the exact desynchronisation between the DOM and `words` + // it was written to prevent. Verified in the browser before this was moved. + // + // The native event is a real `InputEvent`, its `inputType` is the thing both branches + // switch on, and preventing it actually stops the browser. + useEffect(() => { + const editor = editorRef.current; + if (!editor) return; + const onBeforeInput = (event: InputEvent) => { + // The word editor and the insertion field are ``s INSIDE this element, so + // their own typing bubbles here natively — React's `stopPropagation` only ever + // stopped the synthetic tree. Their text is theirs. + if (event.target instanceof HTMLInputElement) return; + if (event.inputType.startsWith("delete")) { event.preventDefault(); - cutNativeSelection( - inputEvent.inputType === "deleteContentForward" ? "forward" : "backward", - ); + cutNativeSelection(event.inputType === "deleteContentForward" ? "forward" : "backward"); return; } - // Inserts are blocked to keep the projection stable: every run of text - // here maps back to a `transcript.words` entry by id, and free text has - // no id to land on. Deletion is fine because it goes through - // `cutNativeSelection`, which resolves the selection to word ids first. - // - // This used to defer to `SourceTranscriptModal`, deleted with the v3 - // media pane — it never got past read-only, so it was never the answer - // it was cited as. Editing a word's TEXT therefore has no in-app path - // today. Adding one means a word-level mutation alongside - // `skipWordRange`, reached from here; lifting this guard on its own - // would only desynchronise the DOM from `words`. - if (inputEvent.inputType === "insertText" || inputEvent.inputType === "insertFromPaste") { + // Free text never lands in the block itself: every run of text here maps back to a + // `transcript.words` entry by id, and typed characters have no id. What they open + // instead is a field beside the word the caret was on, whose commit creates a real + // word to hold them. So the gesture is the document one — put the caret somewhere + // and type — without the DOM ever getting ahead of `words`. + if (event.inputType.startsWith("insert")) { event.preventDefault(); + openInsertion(event.data ?? ""); } + }; + editor.addEventListener("beforeinput", onBeforeInput); + return () => editor.removeEventListener("beforeinput", onBeforeInput); + }, [cutNativeSelection, openInsertion]); + + const handlePaste = useCallback( + (event: ReactClipboardEvent) => { + // Handled here rather than through `insertFromPaste`: preventing the paste stops + // that beforeinput from ever firing, and this is the only place the clipboard text + // is still readable. + event.preventDefault(); + openInsertion(event.clipboardData.getData("text/plain")); }, - [cutNativeSelection], + [openInsertion], ); - const handlePaste = useCallback((event: ReactClipboardEvent) => { - event.preventDefault(); - }, []); - const handlePointerUp = useCallback( (event: ReactPointerEvent) => { if (event.button !== 0) return; @@ -1183,11 +1290,13 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ spellCheck={false} aria-label={ts("transcript.editorAria", { filename })} aria-multiline="true" - onBeforeInput={handleBeforeInput} onKeyDown={handleKeyDown} onPaste={handlePaste} onPointerUp={handlePointerUp} style={{ + // Inline so a split clip reads as one sentence rather than one line per + // piece. The block that fronts a run still owns the header above it. + display: "inline", padding: "4px 4px", font: "400 13px/1.65 var(--font-body)", color: "var(--fg)", @@ -1205,16 +1314,38 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // scrollbar that breaks the cue auto-scroll UX. }} > - {words.map((cw) => ( - - ))} + {words.map((cw) => { + const field = + insertion?.clipWordId === cw.id ? ( + setInsertion({ ...insertion, draft })} + onCommit={commitInsertion} + onCancel={() => { + insertionAbandonedRef.current = true; + setInsertion(null); + }} + abandonedRef={insertionAbandonedRef} + /> + ) : null; + return ( + + {insertion?.side === "before" ? field : null} + + {insertion?.side === "after" ? field : null} + + ); + })} )} @@ -1243,19 +1374,62 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ const TranscriptWord = memo(function TranscriptWord({ cw, isCue, + editable, target, onRestore, onAddTrimRange, + onSetWordText, + onRemoveWords, }: { cw: ClipWord; isCue: boolean; + /** False while this clip's transcript is being regenerated — the words on screen are + * about to be replaced, so an edit typed into them would be thrown away. */ + editable: boolean; target: TrimTarget; onRestore: (run: TrimRun) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; + onSetWordText: (assetId: string, wordId: string, text: string) => void; + onRemoveWords: (assetId: string, wordIds: string[]) => void; }) { const ts = useScopedT("settings"); const [hover, setHover] = useState(false); + // The text being typed, or null when the word is not under edit. + const [draft, setDraft] = useState(null); + // Escape unmounts the field, and an abandoned field's blur must not commit what the + // user just walked away from. + const abandonedRef = useRef(false); const removed = !cw.kept; + // `originalText` is only ever written by a user edit (see `document/transcript.ts`), so + // it is what tells a corrected word from a transcribed one. + const original = cw.word.originalText; + const corrected = original !== undefined; + const blanked = corrected && cw.word.text.trim().length === 0; + + const startEditing = useCallback(() => { + if (!editable) return; + setDraft(cw.word.text); + }, [editable, cw.word.text]); + + const commitDraft = useCallback(() => { + const next = (draft ?? "").trim(); + setDraft(null); + if (next === cw.word.text) return; + onSetWordText(target.assetId, cw.word.id, next); + }, [draft, cw.word.text, cw.word.id, onSetWordText, target.assetId]); + + const inserted = isInsertedWord(cw.word); + + const removeInserted = useCallback(() => { + onRemoveWords(target.assetId, [cw.word.id]); + }, [onRemoveWords, target.assetId, cw.word.id]); + + const revert = useCallback(() => { + if (original === undefined) return; + // Writing the original back through the same path is what clears the provenance + // pair — there is no separate "unedit" operation that could fall out of step. + onSetWordText(target.assetId, cw.word.id, original); + }, [original, cw.word.id, onSetWordText, target.assetId]); if (isSilenceWord(cw.word)) { const durationSec = cw.word.endSec - cw.word.startSec; @@ -1333,25 +1507,191 @@ const TranscriptWord = memo(function TranscriptWord({ ); } + // The inline editor. `contentEditable={false}` keeps the browser from treating it as + // part of the enclosing editable block, and every event it raises is stopped here rather + // than in the block handlers: Backspace inside the field has to type, not cut, and a + // click in it must not seek. + if (draft !== null) { + return ( + setDraft(event.target.value)} + onFocus={(event) => event.currentTarget.select()} + onBlur={() => { + if (abandonedRef.current) { + abandonedRef.current = false; + return; + } + commitDraft(); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") { + event.preventDefault(); + commitDraft(); + } else if (event.key === "Escape") { + event.preventDefault(); + abandonedRef.current = true; + setDraft(null); + } + }} + onPaste={(event) => event.stopPropagation()} + onPointerUp={(event) => event.stopPropagation()} + style={{ + display: "inline", + // `ch` is the digit width, not the real glyph width, so this only + // approximates the word it replaces — the slack keeps it from clipping. + width: `${Math.max(draft.length, 3) + 2}ch`, + margin: 0, + padding: "0 2px", + border: 0, + borderBottom: "2px solid var(--accent)", + borderRadius: 0, + background: "var(--accent-soft)", + color: "var(--fg)", + font: "inherit", + outline: "none", + }} + /> + ); + } + + // A word nobody said. Amber rather than the accent: this one is not a fix to what was + // heard, it is text with no sound underneath — the caveat is the point. Double-click + // rewrites it like any other word; the cross deletes it, because there is no audio for a + // trim to remove. + if (inserted) { + return ( + setHover(true)} + onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} + > + + {cw.word.text} + + {hover ? ( + + + ) : null}{" "} + + ); + } + + // A word the user emptied. It still owns a span of the media, so it keeps a place in + // the stream: rendered as its own (empty) text it would be a bare space — invisible, + // impossible to click, and therefore impossible to undo. + if (blanked) { + return ( + setHover(true)} + onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} + > + + {ts("transcript.blankedWord")} + + {hover ? ( + + ) : null}{" "} + + ); + } + return ( setHover(true)} onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + // Without this the browser selects the word inside the enclosing + // contentEditable; the field about to replace it does its own selecting. + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} > {/* no filler chip. axcut renders every word the same way; the LLM is the only place that names a word a filler (via the @@ -1394,10 +1734,138 @@ const TranscriptWord = memo(function TranscriptWord({ ); }); +/** Hover affordance on a corrected word: put the transcriber's own text back. Mirrors the + * bin on a cut word — same size, same place, the accent rather than the danger colour, + * since reverting a correction restores something instead of removing it. */ +function RevertWordButton({ label, onRevert }: { label: string; onRevert: () => void }) { + return ( + + + ); +} + +/** The one hover control shape the word stream uses, in whichever colour says what it does. + * `contentEditable={false}` keeps it out of the enclosing editable block, and the click is + * stopped so it never reaches the seek handler underneath. */ +function WordChipButton({ + label, + tone, + onPress, + children, +}: { + label: string; + tone: string; + onPress: () => void; + children: ReactNode; +}) { + return ( + + ); +} + +/** + * The field a typed character opens between two words. It is not a word yet — nothing is + * written until it commits — so it carries no `data-word-id` and no place in `words`. + * + * Every event it raises is stopped at the field, for the same reason the word editor stops + * its own: the block around it reads Backspace as a cut and a click as a seek. + */ +function InsertionField({ + value, + label, + onChange, + onCommit, + onCancel, + abandonedRef, +}: { + value: string; + label: string; + onChange: (value: string) => void; + onCommit: () => void; + onCancel: () => void; + abandonedRef: { current: boolean }; +}) { + return ( + onChange(event.target.value)} + onBlur={() => { + if (abandonedRef.current) { + abandonedRef.current = false; + return; + } + onCommit(); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") { + event.preventDefault(); + onCommit(); + } else if (event.key === "Escape") { + event.preventDefault(); + onCancel(); + } + }} + onBeforeInput={(event) => event.stopPropagation()} + onPaste={(event) => event.stopPropagation()} + onPointerUp={(event) => event.stopPropagation()} + style={{ + display: "inline", + width: `${Math.max(value.length, 3) + 2}ch`, + margin: "0 3px 2px 0", + padding: "0 5px", + border: "1px solid var(--warn)", + borderRadius: 999, + background: "var(--warn-soft)", + color: "var(--fg)", + font: "inherit", + outline: "none", + }} + /> + ); +} + // ─── Caret / selection helpers ──────────────────────────────────── // Ponytail port of axcut's findCollapsedDeletionWordId. The non-collapsed // path uses findWordId directly (a range selection's endpoints already @@ -1492,6 +1960,79 @@ function findCollapsedDeletionWordId( return pool.find((wordNode) => isKept(wordNode.dataset.wordId ?? null))?.dataset.wordId ?? null; } +/** + * Where a typed character goes: beside the word the caret was resting on, never inside it. + * + * A caret in the middle of a word anchors AFTER that word rather than splitting it in two — + * a split would need two words where the transcript has one, and neither half would own the + * audio any more. At the very start of the block there is nothing to sit after, so the + * anchor is the first word and the new one lands before it. + */ +function findInsertionAnchor( + editor: HTMLElement, + node: Node | null, + offset: number, +): { clipWordId: string; side: InsertSide } | null { + const wordNodes = Array.from(editor.querySelectorAll("[data-word-id]")); + if (wordNodes.length === 0 || !node) return null; + + const direct = closestWordElement(node); + if (direct?.dataset.wordId) { + const atStart = node.nodeType === Node.TEXT_NODE && offset <= 0; + return { clipWordId: direct.dataset.wordId, side: atStart ? "before" : "after" }; + } + + // The caret is between the block's own children, and `offset` is a child index — the + // same shape `findCollapsedDeletionWordId` reads when it resolves a cut. Walk back for + // the word to sit after; if there is none, the caret is at the head of the stream and + // the new word goes before the first word ahead of it. + const childNodes = Array.from(node.childNodes); + for (const candidate of childNodes.slice(0, clampRangeOffset(node, offset)).reverse()) { + const wordId = findWordId(candidate) ?? findDescendantWordId(candidate); + if (wordId) return { clipWordId: wordId, side: "after" }; + } + for (const candidate of childNodes.slice(clampRangeOffset(node, offset))) { + const wordId = findWordId(candidate) ?? findDescendantWordId(candidate); + if (wordId) return { clipWordId: wordId, side: "before" }; + } + const first = wordNodes[0]; + return first?.dataset.wordId ? { clipWordId: first.dataset.wordId, side: "before" } : null; +} + +/** + * Pull the DOM's answer back onto a word the TRANSCRIPT has. + * + * `[silence]` pills carry a `data-word-id` like everything else in the stream, but they are + * pseudo-words `withSilenceGaps` invents per clip — there is nothing in `transcript.words` + * for a new word to be inserted next to. So the anchor walks off a silence to the nearest + * real word in the direction the caret was already facing, and only crosses to the other + * side when that direction runs out of stream. + */ +function resolveInsertionAnchor( + words: ClipWord[], + clipWordId: string, + side: InsertSide, +): { clipWordId: string; side: InsertSide } | null { + const from = words.findIndex((w) => w.id === clipWordId); + if (from < 0) return null; + const real = (index: number) => + index >= 0 && index < words.length && !isSilenceWord(words[index].word); + if (side === "after") { + for (let i = from; i >= 0; i--) if (real(i)) return { clipWordId: words[i].id, side: "after" }; + for (let i = 0; i < words.length; i++) { + if (real(i)) return { clipWordId: words[i].id, side: "before" }; + } + return null; + } + for (let i = from; i < words.length; i++) { + if (real(i)) return { clipWordId: words[i].id, side: "before" }; + } + for (let i = words.length - 1; i >= 0; i--) { + if (real(i)) return { clipWordId: words[i].id, side: "after" }; + } + return null; +} + function findDescendantWordId(node: Node): string | null { if (node instanceof HTMLElement && node.dataset.wordId) { return node.dataset.wordId; diff --git a/src/components/ai-edition/TranscriptPane.gating.test.tsx b/src/components/ai-edition/TranscriptPane.gating.test.tsx index 008c6161f..bb62d61eb 100644 --- a/src/components/ai-edition/TranscriptPane.gating.test.tsx +++ b/src/components/ai-edition/TranscriptPane.gating.test.tsx @@ -56,6 +56,9 @@ function renderPane( onSeek={vi.fn()} onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={overrides.isTranscribing ?? false} diff --git a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx index 600982fe0..97ba78739 100644 --- a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx +++ b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx @@ -84,6 +84,9 @@ function renderPane( onSeek={vi.fn()} onAddTrimRange={onAddTrimRange} onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} @@ -227,6 +230,9 @@ describe("keyboard cut with the caret between words", () => { ]) } onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx index f11151122..37a03b25a 100644 --- a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx +++ b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx @@ -78,6 +78,9 @@ function renderPane(onSeek: (sec: number) => void = vi.fn()) { onSeek={onSeek} onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx new file mode 100644 index 000000000..74d2c021f --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx @@ -0,0 +1,224 @@ +// @vitest-environment jsdom +// Correcting a word's TEXT in the transcript pane, as opposed to cutting it. +// +// The two gestures share one word stream on purpose (no mode, no second tab), so what +// keeps them apart is which gesture the user makes: Backspace cuts the media, a +// double-click rewrites the text. These tests hold that line — a keystroke inside the +// editing field must never reach the cut path, and a correction must never touch the +// timeline. + +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutClip, AxcutTranscript, AxcutWord } from "@/lib/ai-edition/schema"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 3, + cameraTrack: null, +}; + +const CLIP: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 3, + timelineStartSec: 0, + timelineEndSec: 3, + wordRefs: [], + origin: "user", + reason: "", +}; + +// Contiguous: a gap would insert a `[silence]` pill between the words and move the +// indices these tests address words by. +const WORDS: AxcutWord[] = [ + { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "Bonjour" }, + { id: "w2", segmentId: "s", startSec: 1, endSec: 2, text: "Kubernetes" }, + { id: "w3", segmentId: "s", startSec: 2, endSec: 3, text: "tout" }, +]; + +function transcript(words: AxcutWord[] = WORDS): AxcutTranscript { + return { assetId: "asset_1", language: "fr", segments: [], words }; +} + +function renderPane(words?: AxcutWord[], busyAssetIds: string[] = []) { + const onSetWordText = vi.fn(); + const onAddTrimRange = vi.fn(); + const view = render( + + + , + ); + const wordEl = (id: string) => { + const el = view.container.querySelector(`[data-word-id="clip_1:${id}"]`); + if (!el) throw new Error(`word ${id} not rendered`); + return el; + }; + const field = () => view.container.querySelector("input[data-word-editor]"); + return { ...view, wordEl, field, onSetWordText, onAddTrimRange }; +} + +afterEach(cleanup); + +describe("telling the user the gestures exist", () => { + it("shows the editing hint line and the ? help when a transcript is on screen", () => { + // The gestures are invisible until tried — the pane must name them itself. + const view = renderPane(); + expect(view.getByText(/Double-click a word to correct it/)).toBeInTheDocument(); + expect(view.getByRole("button", { name: "Help" })).toBeInTheDocument(); + }); +}); + +describe("correcting a word", () => { + it("opens an editing field on the word a double-click lands on", () => { + const view = renderPane(); + expect(view.field()).toBeNull(); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toHaveValue("Kubernetes"); + }); + + it("commits on Enter, addressing the word by its BARE id and the clip's asset", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Kubernetes 1.31" } }); + fireEvent.keyDown(field, { key: "Enter" }); + // `clip_1:w2` is what the DOM node carries; the transcript knows only `w2`. + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Kubernetes 1.31"); + }); + + it("commits on blur, so clicking away does not throw the correction out", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w1")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Bonsoir" } }); + fireEvent.blur(field); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w1", "Bonsoir"); + }); + + it("abandons on Escape, and a blur afterwards does not resurrect the draft", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w1")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Bonsoir" } }); + fireEvent.keyDown(field, { key: "Escape" }); + fireEvent.blur(field); + expect(view.onSetWordText).not.toHaveBeenCalled(); + expect(view.field()).toBeNull(); + }); + + it("writes nothing when the text comes back unchanged", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onSetWordText).not.toHaveBeenCalled(); + }); + + // The field lives inside the block's contentEditable, whose Backspace handler cuts the + // media. Without the stopPropagation on the field, deleting a letter would trim the clip. + it("does not cut the media when Backspace is pressed inside the field", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.keyDown(field, { key: "Backspace" }); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); + + it("stays read-only while the transcript is being regenerated", () => { + const view = renderPane(undefined, ["asset_1"]); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toBeNull(); + }); +}); + +describe("a word already corrected", () => { + const CORRECTED: AxcutWord[] = [ + WORDS[0], + { ...WORDS[1], text: "Kubernetes", originalText: "Cuber Nettes", source: "user" }, + WORDS[2], + ]; + + it("is marked as corrected and names what the transcriber heard", () => { + const view = renderPane(CORRECTED); + const el = view.wordEl("w2"); + expect(el).toHaveAttribute("data-corrected", "true"); + expect(el.title).toContain("Cuber Nettes"); + }); + + it("offers a revert that writes the transcriber's own text back", () => { + const view = renderPane(CORRECTED); + fireEvent.mouseEnter(view.wordEl("w2")); + const revert = view.wordEl("w2").querySelector("button"); + if (!revert) throw new Error("no revert control"); + fireEvent.click(revert); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Cuber Nettes"); + }); + + it("leaves an untouched word unmarked and without a revert", () => { + const view = renderPane(CORRECTED); + fireEvent.mouseEnter(view.wordEl("w1")); + expect(view.wordEl("w1")).not.toHaveAttribute("data-corrected"); + expect(view.wordEl("w1").querySelector("button")).toBeNull(); + }); +}); + +// Emptying a word is how a junk token gets out of the captions without cutting the audio. +// Rendered as its own (empty) text it would be a bare space: invisible, un-clickable, and +// therefore impossible to undo. +describe("a word the user emptied", () => { + const BLANKED: AxcutWord[] = [ + WORDS[0], + { ...WORDS[1], text: "", originalText: "Kubernetes", source: "user" }, + WORDS[2], + ]; + + it("keeps a visible, clickable place in the stream", () => { + const view = renderPane(BLANKED); + const el = view.wordEl("w2"); + expect(el).toHaveAttribute("data-blanked", "true"); + expect(el.textContent?.trim()).not.toBe(""); + }); + + it("can be reopened for editing and reverted", () => { + const view = renderPane(BLANKED); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toHaveValue(""); + + fireEvent.keyDown(view.field() as HTMLInputElement, { key: "Escape" }); + fireEvent.mouseEnter(view.wordEl("w2")); + const revert = view.wordEl("w2").querySelector("button"); + if (!revert) throw new Error("no revert control"); + fireEvent.click(revert); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Kubernetes"); + }); +}); diff --git a/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx new file mode 100644 index 000000000..caed55d61 --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx @@ -0,0 +1,259 @@ +// @vitest-environment jsdom +// Typing a word into the transcript that nobody said. +// +// This is the third gesture on the one word stream, and the one that had to get past a +// guard: the block used to swallow every keystroke outright, because free text has no +// `transcript.words` entry to land on. It still never lands in the block — what a typed +// character opens is a field beside the word the caret was on, and only its commit makes a +// word. These tests hold that: the DOM never gets ahead of `words`, and Backspace inside +// the field types instead of cutting the clip out from under it. + +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutClip, AxcutTranscript, AxcutWord } from "@/lib/ai-edition/schema"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 3, + cameraTrack: null, +}; + +const CLIP: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 3, + timelineStartSec: 0, + timelineEndSec: 3, + wordRefs: [], + origin: "user", + reason: "", +}; + +// Contiguous, so no `[silence]` pill sits between them to shift the caret indices. +const WORDS: AxcutWord[] = [ + { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "Bonjour" }, + { id: "w2", segmentId: "s", startSec: 1, endSec: 2, text: "à" }, + { id: "w3", segmentId: "s", startSec: 2, endSec: 3, text: "tous" }, +]; + +function renderPane(words: AxcutWord[] = WORDS, busyAssetIds: string[] = []) { + const onInsertWord = vi.fn(); + const onRemoveWords = vi.fn(); + const onAddTrimRange = vi.fn(); + const transcript: AxcutTranscript = { + assetId: "asset_1", + language: "fr", + segments: [], + words, + }; + const view = render( + + + , + ); + const editor = view.container.querySelector('[role="textbox"]'); + if (!editor) throw new Error("transcript editor not rendered"); + const field = () => view.container.querySelector("input[data-word-inserter]"); + const wordEl = (id: string) => { + const el = view.container.querySelector(`[data-word-id="clip_1:${id}"]`); + if (!el) throw new Error(`word ${id} not rendered`); + return el; + }; + return { ...view, editor, field, wordEl, onInsertWord, onRemoveWords, onAddTrimRange }; +} + +/** Park the caret between words at editor level, the way `restoreCaretBeforeWord` does. */ +function caretBeforeWordAt(editor: HTMLElement, index: number) { + const range = document.createRange(); + range.setStart(editor, index); + range.collapse(true); + const selection = window.getSelection(); + selection?.removeAllRanges(); + selection?.addRange(range); +} + +/** + * A real native `beforeinput`, because that is what the block listens to. + * + * Not `fireEvent.beforeInput`: React 18 builds its `onBeforeInput` from the legacy + * `textInput` event, whose `TextEvent` has no `inputType` — which is exactly why the guard + * moved off React and onto the DOM. Driving the synthetic one here would test a path the + * browser never takes. + */ +function type(editor: HTMLElement, data: string) { + // Through `fireEvent` so the state the listener sets is flushed, but with an event + // built by hand — `fireEvent.beforeInput` does not exist here, and the point is to + // dispatch the real thing. + fireEvent( + editor, + new InputEvent("beforeinput", { + data, + inputType: "insertText", + bubbles: true, + cancelable: true, + }), + ); +} + +afterEach(() => { + cleanup(); + window.getSelection()?.removeAllRanges(); +}); + +describe("typing between two words", () => { + it("opens a field there instead of dropping the keystroke", () => { + const view = renderPane(); + expect(view.field()).toBeNull(); + caretBeforeWordAt(view.editor, 2); // between "à" and "tous" + type(view.editor, "v"); + expect(view.field()).toHaveValue("v"); + }); + + it("stays inert outside dev builds — the gesture waits for TTS", () => { + // An inserted word with no voice only borrows free silence, so the gesture ships + // dev-only (see openInsertion). Release builds must drop the keystroke silently, + // the same way they did before the feature existed. + vi.stubEnv("DEV", false); + try { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.field()).toBeNull(); + expect(view.onInsertWord).not.toHaveBeenCalled(); + expect(view.editor.textContent).not.toContain("v "); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("never writes the typed text into the block itself", () => { + // The whole reason inserts were blocked: a run of text with no word id behind it + // desynchronises the DOM from `words`. + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.editor.textContent).not.toContain("v "); + expect(view.onInsertWord).not.toHaveBeenCalled(); + }); + + it("commits on Enter, against the word the caret was after", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.change(field, { target: { value: "vraiment" } }); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w2", "after", "vraiment"); + }); + + it("anchors before the first word when the caret is at the very start", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 0); + type(view.editor, "E"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w1", "before", "E"); + }); + + it("abandons on Escape without writing anything", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Escape" }); + fireEvent.blur(field); + expect(view.onInsertWord).not.toHaveBeenCalled(); + expect(view.field()).toBeNull(); + }); + + it("commits on blur", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 1); + type(view.editor, "x"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.change(field, { target: { value: "donc" } }); + fireEvent.blur(field); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w1", "after", "donc"); + }); + + it("does not cut the media when Backspace is pressed inside the field", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Backspace" }); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); + + it("stays shut while this clip's transcript is being regenerated", () => { + const view = renderPane(WORDS, ["asset_1"]); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.field()).toBeNull(); + }); +}); + +describe("a word that was inserted", () => { + const INSERTED: AxcutWord[] = [ + WORDS[0], + { id: "synth_1", segmentId: "s", startSec: 1, endSec: 1, text: "vraiment", source: "synth" }, + WORDS[1], + WORDS[2], + ]; + + it("reads as its own thing, not as a transcribed word", () => { + const view = renderPane(INSERTED); + const el = view.wordEl("synth_1"); + expect(el).toHaveAttribute("data-inserted", "true"); + expect(el.textContent).toContain("vraiment"); + }); + + // There is no audio for a trim to remove, so the gesture that makes a spoken word go + // away cannot be the one that makes this go away. + it("is deleted outright by its own control", () => { + const view = renderPane(INSERTED); + fireEvent.mouseEnter(view.wordEl("synth_1")); + const remove = view.wordEl("synth_1").querySelector("button"); + if (!remove) throw new Error("no delete control"); + fireEvent.click(remove); + expect(view.onRemoveWords).toHaveBeenCalledWith("asset_1", ["synth_1"]); + }); + + it("is deleted, not trimmed, when Backspace lands on it alone", () => { + const view = renderPane(INSERTED); + caretBeforeWordAt(view.editor, 2); // right after the insert + fireEvent.keyDown(view.editor, { key: "Backspace" }); + expect(view.onRemoveWords).toHaveBeenCalledWith("asset_1", ["synth_1"]); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx index 804cfa5a5..077cb62c8 100644 --- a/src/components/ai-edition/VirtualPreview.tsx +++ b/src/components/ai-edition/VirtualPreview.tsx @@ -5,7 +5,12 @@ import { MAX_NATIVE_PLAYBACK_RATE, } from "@/components/video-editor/types"; import { resolvePlaybackSegments } from "@/lib/ai-edition/document/timeline"; -import type { AxcutClip, AxcutTrimRange, AxcutZoomRegion } from "@/lib/ai-edition/schema"; +import type { + AxcutClip, + AxcutInsertRange, + AxcutTrimRange, + AxcutZoomRegion, +} from "@/lib/ai-edition/schema"; import { audioGainScalar } from "@/lib/ai-edition/store/editorSettings"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { PlaybackClockRef } from "@/lib/ai-edition/timeline/playback-clock"; @@ -116,6 +121,8 @@ interface VirtualPreviewProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; trimRanges?: AxcutTrimRange[]; + /** The pauses added words created — they lengthen playback, they do not cut it. */ + insertRanges?: AxcutInsertRange[]; seekTarget?: { timeSec: number; isSource?: boolean; requestId: number } | null; onTimeChange?: (timeSec: number) => void; onLoadedMetadata?: ( @@ -160,6 +167,7 @@ export function VirtualPreview({ zoomRegions = [], speedRegions = [], trimRanges = [], + insertRanges = [], seekTarget, onTimeChange, onLoadedMetadata, @@ -411,8 +419,8 @@ export function VirtualPreview({ // source time to a RAW virtual time that jumps discontinuously by exactly the trim's // width the moment the video itself jumps — matching the marker's own pixel span. const playbackClips = useMemo( - () => resolvePlaybackSegments(clips, trimRanges), - [clips, trimRanges], + () => resolvePlaybackSegments(clips, trimRanges, insertRanges), + [clips, trimRanges, insertRanges], ); const playbackClipsRef = useRef(playbackClips); playbackClipsRef.current = playbackClips; diff --git a/src/components/ai-edition/WebcamOverlay.test.tsx b/src/components/ai-edition/WebcamOverlay.test.tsx index 16da96666..019c0d415 100644 --- a/src/components/ai-edition/WebcamOverlay.test.tsx +++ b/src/components/ai-edition/WebcamOverlay.test.tsx @@ -68,6 +68,7 @@ function makeDocument(): AxcutDocument { clips: [CLIP_WITH_CAMERA, CLIP_WITHOUT_CAMERA], gaps: [], trimRanges: [], + insertRanges: [], muteRanges: [], speedRanges: [], captionRanges: [], diff --git a/src/components/ai-edition/v4/EditorShellV4.module.css b/src/components/ai-edition/v4/EditorShellV4.module.css index 1e313e34c..fd19b3f57 100644 --- a/src/components/ai-edition/v4/EditorShellV4.module.css +++ b/src/components/ai-edition/v4/EditorShellV4.module.css @@ -1643,6 +1643,41 @@ background: var(--danger-soft); color: var(--danger); } + +/* Where the user has ADDED a word: text with no audio behind it. A thin amber tick + over the waveform, at the moment the word sits on, wide enough to hit and no wider + — the clip underneath still has to be draggable everywhere else. Amber is the + colour the transcript pane gives the same word, so the two read as one thing. */ +.tlClipInsert { + position: absolute; + top: 0; + bottom: 0; + z-index: 2; + width: 9px; + margin-left: -4px; + padding: 0; + border: 0; + background: transparent; + cursor: pointer; +} +.tlClipInsert::before { + content: ""; + position: absolute; + left: 3px; + top: 4px; + bottom: 4px; + width: 3px; + border-radius: 2px; + background: var(--warn); + box-shadow: 0 0 0 1px color-mix(in srgb, #080a0d 45%, transparent); + transition: box-shadow var(--motion-fast) var(--ease); +} +.tlClipInsert:hover::before, +.tlClipInsert:focus-visible::before { + box-shadow: + 0 0 0 1px color-mix(in srgb, #080a0d 45%, transparent), + 0 0 0 4px var(--warn-soft); +} .tlDropHint { position: absolute; inset: 0; diff --git a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx index 8e515568c..f8d0b9b67 100644 --- a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx +++ b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx @@ -75,6 +75,9 @@ function renderTimeline( ) { const tl = { clips, + // Marks for added words are read straight off the transcript (see the pane's + // amber words) — no project here has any. + transcripts: [], assets, annotationRegions: [annotation], speedRegions: [], diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index f6c94e2ca..131ab310d 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -29,7 +29,7 @@ import { useScopedT } from "@/contexts/I18nContext"; import { useAudioPeaks } from "@/hooks/useAudioPeaks"; import { createId } from "@/lib/ai-edition/document/ids"; import { setUiProbeScrubbing } from "@/lib/ai-edition/perf/uiFrameProbe"; -import type { AxcutClip } from "@/lib/ai-edition/schema"; +import type { AxcutClip, AxcutWord } from "@/lib/ai-edition/schema"; import { audioGainScalar } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useTimelineTranscriptGate } from "@/lib/ai-edition/store/transcriptionStore"; @@ -38,6 +38,12 @@ import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { hasAnyClipWithCamera } from "@/lib/ai-edition/timeline/camera"; import { formatSec } from "@/lib/ai-edition/timeline/format"; +import { + expandRawSec, + type RulerInsert, + rulerInserts, + totalInsertedSec, +} from "@/lib/ai-edition/timeline/inserted-time"; import { newRegionDurationSec, setTimelineScale, @@ -198,6 +204,10 @@ interface RulerTick { interface PlayheadOverlayProps { /** Full timeline length in seconds — the denominator for the playhead's percentage. */ totalSec: number; + /** The pauses added words created. `currentTimeSec` is a STORED second; the ruler it is + * drawn on counts the pauses, so it has to be placed through them or it drifts from + * the clips by the whole added time. */ + inserts: readonly RulerInsert[]; /** Live scrub position, when a drag is in flight. Takes precedence over the store. */ overrideTimeSec: number | null; canvasStyle: React.CSSProperties; @@ -223,13 +233,14 @@ interface PlayheadOverlayProps { */ const PlayheadOverlay = memo(function PlayheadOverlay({ totalSec, + inserts, overrideTimeSec, canvasStyle, onPointerDown, playheadRef, }: PlayheadOverlayProps) { const storeTimeSec = useProjectStore((s) => s.currentTimeSec); - const pct = ((overrideTimeSec ?? storeTimeSec) / totalSec) * 100; + const pct = (expandRawSec(overrideTimeSec ?? storeTimeSec, inserts) / totalSec) * 100; return (
@@ -439,15 +450,26 @@ export function V4Timeline({ // clicked instead of looking like it worked. Same question, same helper as the Layout // pane: is a camera attached anywhere on this timeline? const hasAnyCamera = useMemo(() => hasAnyClipWithCamera(tl.assets, clips), [tl.assets, clips]); + // The pauses added words created, placed on the ruler. Everything below measures the + // EXPANDED ruler — stored clip geometry plus the time those pauses add — because that + // is the film's real length and the one the playhead runs along. Stored geometry is + // never rewritten for this: only what is drawn moves. + // `?? []` because the key is additive: a document written before it has no pauses. + const inserts = useMemo( + () => rulerInserts(tl.insertRanges ?? [], clips), + [tl.insertRanges, clips], + ); const total = useMemo( () => Math.max( 1, - clips.reduce((m, c) => Math.max(m, c.timelineEndSec), 0), + clips.reduce((m, c) => Math.max(m, c.timelineEndSec), 0) + totalInsertedSec(inserts), ), - [clips], + [clips, inserts], ); const pctOf = useCallback((sec: number) => (sec / total) * 100, [total]); + /** Stored raw seconds → a percentage of the expanded ruler. */ + const pctAt = useCallback((sec: number) => pctOf(expandRawSec(sec, inserts)), [pctOf, inserts]); const showLanes = variant === "edit"; // The visible fraction of the timeline, and what one second is worth on screen @@ -509,6 +531,32 @@ export function V4Timeline({ label: `${(p.member.customScale ?? ZOOM_DEPTH_SCALES[p.member.depth]).toFixed(2)}×`, sourceIds: p.ids, })); + // Where the user has ADDED words. Derived from the transcript on every render and + // stored nowhere: the word carries `source: "synth"` and its own source time, so a mark + // built from it cannot drift from the amber word the transcript pane shows. Grouped by + // clip because each mark is positioned inside its clip's own box — it then travels with + // the clip through a reorder for free, with no ruler arithmetic of its own. + const insertedWordsByClip = useMemo(() => { + const byAsset = new Map(); + for (const transcript of tl.transcripts) { + const added = transcript.words.filter((word) => word.source === "synth"); + if (added.length > 0) byAsset.set(transcript.assetId, added); + } + if (byAsset.size === 0) return new Map>(); + const out = new Map>(); + for (const clip of clips) { + const words = byAsset.get(clip.assetId); + const sourceEnd = clip.sourceEndSec ?? clip.sourceStartSec; + const span = sourceEnd - clip.sourceStartSec; + if (!words || span <= 0) continue; + const marks = words + .filter((word) => word.startSec >= clip.sourceStartSec && word.startSec <= sourceEnd) + .map((word) => ({ word, atPct: ((word.startSec - clip.sourceStartSec) / span) * 100 })); + if (marks.length > 0) out.set(clip.id, marks); + } + return out; + }, [tl.transcripts, clips]); + // trims: content-free (no per-instance text/settings), so touching rows — // inevitable once a trim is ventilated across a clip boundary — are // coalesced into one pill. This is what makes growing a trim across a @@ -1142,8 +1190,10 @@ export function V4Timeline({ compact ? ` ${styles.lanePillCompact}` : "" }${seg.interactive && isPillSelected(p.id) ? ` ${styles.lanePillSel}` : ""}`} style={{ - left: `${pctOf(seg.segStart)}%`, - width: `${pctOf(durSec)}%`, + left: `${pctAt(seg.segStart)}%`, + // Measured on the expanded ruler at BOTH ends: a region straddling a pause + // covers it, so its box has to grow by that pause and not merely slide. + width: `${pctOf(expandRawSec(seg.segEnd, inserts) - expandRawSec(seg.segStart, inserts))}%`, transform: seg.shiftPx ? `translateX(${seg.shiftPx}px)` : undefined, transition: !clipDrag ? undefined @@ -1443,7 +1493,7 @@ export function V4Timeline({
{tick.major ? ( {fmtTick(tick.sec, rulerTicks.step)} @@ -1503,6 +1553,12 @@ export function V4Timeline({ > {clips.map((c, i) => { const dur = c.timelineEndSec - c.timelineStartSec; + // On the expanded ruler the box also carries whatever pauses fall + // inside it — the film really does stay on this clip's frame for + // them, so they belong to its box rather than between boxes. + const boxStart = expandRawSec(c.timelineStartSec, inserts); + const boxEnd = expandRawSec(c.timelineEndSec, inserts); + const boxLen = boxEnd - boxStart; const asset = tl.assets.find((a) => a.id === c.assetId); const clipVideoUrl = videoSources.find((v) => v.id === c.assetId)?.src; const selected = tl.clipSelection === c.id; @@ -1530,12 +1586,12 @@ export function V4Timeline({ dragging ? ` ${styles.tlClipDragging}` : "" }`} style={{ - left: `${pctOf(c.timelineStartSec)}%`, + left: `${pctOf(boxStart)}%`, // Minus the gutter that separates two cards (it used to be the // flex row's `gap`). A clip shorter than the gutter lands on // .tlClip's 1px min-width instead of collapsing — same rule as // the lane pills above. - width: `calc(${pctOf(dur)}% - ${CLIP_GUTTER_PX}px)`, + width: `calc(${pctOf(boxLen)}% - ${CLIP_GUTTER_PX}px)`, transform: clipTransform, }} onPointerDown={(e) => startClipDrag(e, c)} @@ -1578,6 +1634,41 @@ export function V4Timeline({ {tl.assets.find((a) => a.id === c.assetId)?.label ?? c.assetId}
+ {(insertedWordsByClip.get(c.id) ?? []).map(({ word, atPct }) => { + // A word whose pause the film actually holds gets a BAND as wide + // as the time it adds — that width is the added time, drawn. One + // that fitted in silence already there adds nothing and stays the + // hairline it was: there is nothing to show. + const pause = inserts.find((ins) => ins.wordId === word.id); + const left = pause + ? ((expandRawSec(pause.atRawSec, inserts) - boxStart) / boxLen) * 100 + : atPct; + const width = pause ? (pause.durationSec / boxLen) * 100 : 0; + return ( +