From a6c16a4e2052dc8a11752fdc22b5400467cd3598 Mon Sep 17 00:00:00 2001 From: sunyuchenyaobo <261746743+sunyuchenyaobo@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:07:30 +0800 Subject: [PATCH 01/12] feat(document): add immutable transcript word edits --- .../ai-edition/document/transcript.test.ts | 263 ++++++++++++++++++ src/lib/ai-edition/document/transcript.ts | 68 +++++ 2 files changed, 331 insertions(+) create mode 100644 src/lib/ai-edition/document/transcript.test.ts create mode 100644 src/lib/ai-edition/document/transcript.ts diff --git a/src/lib/ai-edition/document/transcript.test.ts b/src/lib/ai-edition/document/transcript.test.ts new file mode 100644 index 000000000..1678374bc --- /dev/null +++ b/src/lib/ai-edition/document/transcript.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, it } from "vitest"; +import type { AxcutTranscript } from "../schema"; +import { setWordText } from "./transcript"; + +function fixture(language = "en"): AxcutTranscript { + return { + assetId: "asset_1", + language, + sourceDslPath: "transcript.dsl", + sourceJsonPath: "transcript.json", + segments: [ + { + id: "segment_1", + kind: "speech", + startSec: 1, + endSec: 4, + text: "I use OpenScreen", + wordIds: ["word_1", "word_2", "word_3"], + }, + { + id: "segment_2", + kind: "speech", + startSec: 5, + endSec: 6, + text: "Untouched segment", + wordIds: ["word_4", "word_5"], + }, + ], + // Deliberately shuffled: segment.wordIds, not this array, defines segment order. + words: [ + { id: "word_3", segmentId: "segment_1", startSec: 3, endSec: 4, text: "OpenScreen" }, + { id: "word_1", segmentId: "segment_1", startSec: 1, endSec: 2, text: "I" }, + { id: "word_5", segmentId: "segment_2", startSec: 5.5, endSec: 6, text: "segment" }, + { id: "word_2", segmentId: "segment_1", startSec: 2, endSec: 3, text: "use" }, + { id: "word_4", segmentId: "segment_2", startSec: 5, endSec: 5.5, text: "Untouched" }, + ], + }; +} + +function transcriptForTokens(language: string, tokens: string[]): AxcutTranscript { + const wordIds = tokens.map((_, index) => `word_${index + 1}`); + return { + assetId: "asset_tokens", + language, + segments: [ + { + id: "segment_tokens", + kind: "speech", + startSec: 0, + endSec: tokens.length, + text: tokens.join(" "), + wordIds, + }, + { + id: "segment_other", + kind: "speech", + startSec: 20, + endSec: 21, + text: "other", + wordIds: ["word_other"], + }, + ], + words: [ + ...tokens.map((text, index) => ({ + id: wordIds[index], + segmentId: "segment_tokens", + startSec: index, + endSec: index + 1, + text, + })), + { + id: "word_other", + segmentId: "segment_other", + startSec: 20, + endSec: 21, + text: "other", + }, + ], + }; +} + +describe("setWordText", () => { + it("immutably updates the exact word and rebuilds only its owning segment", () => { + const transcript = fixture(); + const originalSnapshot = structuredClone(transcript); + const originalTarget = transcript.words.find((word) => word.id === "word_2"); + const originalOtherWord = transcript.words.find((word) => word.id === "word_4"); + const originalOtherSegment = transcript.segments[1]; + + const result = setWordText(transcript, "word_2", "prefer"); + + expect(result).not.toBe(transcript); + expect(result.words.map((word) => word.id)).toEqual(transcript.words.map((word) => word.id)); + expect(result.segments.map((segment) => segment.id)).toEqual( + transcript.segments.map((segment) => segment.id), + ); + expect(result.words.find((word) => word.id === "word_2")).toEqual({ + ...originalTarget, + text: "prefer", + }); + expect(result.segments[0]).toEqual({ + ...transcript.segments[0], + text: "I prefer OpenScreen", + }); + for (const originalWord of transcript.words) { + if (originalWord.id !== "word_2") { + expect(result.words.find((word) => word.id === originalWord.id)).toBe(originalWord); + } + } + expect(result.words.find((word) => word.id === "word_4")).toBe(originalOtherWord); + expect(result.segments[1]).toBe(originalOtherSegment); + expect(result.assetId).toBe("asset_1"); + expect(result.language).toBe("en"); + expect(result.sourceDslPath).toBe("transcript.dsl"); + expect(result.sourceJsonPath).toBe("transcript.json"); + expect(transcript).toEqual(originalSnapshot); + }); + + it("uses segment.wordIds order even when transcript.words is shuffled", () => { + const result = setWordText(fixture(), "word_3", "Studio"); + + expect(result.segments[0].text).toBe("I use Studio"); + expect(result.words.map((word) => word.id)).toEqual([ + "word_3", + "word_1", + "word_5", + "word_2", + "word_4", + ]); + }); + + it("joins English words with one space", () => { + const result = setWordText( + transcriptForTokens("en", ["I", "use", "OpenScreen"]), + "word_2", + "prefer", + ); + + expect(result.segments[0].text).toBe("I prefer OpenScreen"); + }); + + it("preserves the passed word text exactly while trimming its segment contribution", () => { + const result = setWordText(fixture(), "word_2", " prefer "); + + expect(result.words.find((word) => word.id === "word_2")?.text).toBe(" prefer "); + expect(result.segments[0].text).toBe("I prefer OpenScreen"); + }); + + it.each([ + "zh", + "zh-CN", + "zh-TW", + "ZH-cn", + ])("does not add artificial spaces between adjacent Chinese content for %s", (language) => { + const result = setWordText(transcriptForTokens(language, ["你", "好", "世界"]), "word_2", "们"); + + expect(result.segments[0].text).toBe("你们世界"); + }); + + it.each([ + "ja", + "ja-JP", + "JA-jp", + ])("does not add artificial spaces between adjacent Japanese content for %s", (language) => { + const result = setWordText( + transcriptForTokens(language, ["私", "は", "テスト", "です"]), + "word_3", + "開発者", + ); + + expect(result.segments[0].text).toBe("私は開発者です"); + }); + + it("does not add a space after Chinese closing punctuation between CJK tokens", () => { + const result = setWordText(transcriptForTokens("zh-CN", ["你好，", "世"]), "word_2", "世界"); + + expect(result.segments[0].text).toBe("你好，世界"); + }); + + it("does not add a space after Japanese closing punctuation between CJK tokens", () => { + const result = setWordText( + transcriptForTokens("ja-JP", ["これは。", "試験"]), + "word_2", + "テスト", + ); + + expect(result.segments[0].text).toBe("これは。テスト"); + }); + + it("keeps readable boundaries in mixed CJK and Latin content", () => { + const result = setWordText( + transcriptForTokens("zh-CN", ["我们用", "GitHub", "Action", "部署"]), + "word_3", + "Actions", + ); + + expect(result.segments[0].text).toBe("我们用 GitHub Actions 部署"); + }); + + it("does not put spaces before common closing punctuation", () => { + const result = setWordText( + transcriptForTokens("en", ["Hello", ",", "world", "?"]), + "word_4", + "!", + ); + + expect(result.segments[0].text).toBe("Hello, world!"); + }); + + it("does not put spaces immediately after common opening punctuation", () => { + const result = setWordText(transcriptForTokens("en", ["(", "hello", ")"]), "word_2", "world"); + + expect(result.segments[0].text).toBe("(world)"); + }); + + it.each([ + { tokens: ["I", "use", "OpenScreen"], targetId: "word_2", expected: "I OpenScreen" }, + { tokens: ["I", "use", "OpenScreen"], targetId: "word_1", expected: "use OpenScreen" }, + { tokens: ["I", "use", "OpenScreen"], targetId: "word_3", expected: "I use" }, + ])("keeps the emptied word but creates no duplicate or edge whitespace", ({ + tokens, + targetId, + expected, + }) => { + const result = setWordText(transcriptForTokens("en", tokens), targetId, ""); + + expect(result.words.find((word) => word.id === targetId)?.text).toBe(""); + expect(result.segments[0].text).toBe(expected); + }); + + it.each(["missing_word", "silence_1"])("rejects non-document word ID %s", (wordId) => { + expect(() => setWordText(fixture(), wordId, "replacement")).toThrowError(wordId); + }); + + it("rejects a target whose owning segment is missing", () => { + const transcript = fixture(); + const target = transcript.words.find((word) => word.id === "word_2"); + if (!target) throw new Error("fixture target missing"); + target.segmentId = "segment_missing"; + + expect(() => setWordText(transcript, "word_2", "replacement")).toThrowError( + /word_2.*segment_missing|segment_missing.*word_2/, + ); + }); + + it("rejects an owning segment that references a missing word", () => { + const transcript = fixture(); + transcript.segments[0].wordIds.splice(1, 0, "word_missing"); + + expect(() => setWordText(transcript, "word_2", "replacement")).toThrowError( + /segment_1.*word_missing|word_missing.*segment_1/, + ); + }); + + it("rejects an owning segment that omits the target word", () => { + const transcript = fixture(); + transcript.segments[0].wordIds = ["word_1", "word_3"]; + + expect(() => setWordText(transcript, "word_2", "replacement")).toThrowError( + /segment_1.*word_2|word_2.*segment_1/, + ); + }); +}); diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts new file mode 100644 index 000000000..45e538d88 --- /dev/null +++ b/src/lib/ai-edition/document/transcript.ts @@ -0,0 +1,68 @@ +import type { AxcutTranscript } from "../schema"; + +const CJK_EDGE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u; +const CLOSING_PUNCTUATION = /^[,.;:!?%。，、；：！？…）)\]}>》」』】〕]/u; +const TRAILING_CLOSING_PUNCTUATION = /[,.;:!?%。，、；：！？…）)\]}>》」』】〕]+$/u; +const OPENING_PUNCTUATION = /[([<{《「『【〔（]$/u; + +function joinSegmentText(language: string, texts: string[]): string { + const tokens = texts.map((text) => text.trim()).filter((text) => text.length > 0); + const primaryLanguage = language.split("-")[0].toLowerCase(); + const compactCjk = primaryLanguage === "zh" || primaryLanguage === "ja"; + + return tokens.reduce((joined, token) => { + if (joined.length === 0) return token; + if (CLOSING_PUNCTUATION.test(token) || OPENING_PUNCTUATION.test(joined)) { + return joined + token; + } + const leftContentEdge = joined.replace(TRAILING_CLOSING_PUNCTUATION, "").at(-1) ?? ""; + if (compactCjk && CJK_EDGE.test(leftContentEdge) && CJK_EDGE.test(token[0] ?? "")) { + return joined + token; + } + return `${joined} ${token}`; + }, ""); +} + +export function setWordText( + transcript: AxcutTranscript, + wordId: string, + text: string, +): AxcutTranscript { + const targetWord = transcript.words.find((word) => word.id === wordId); + if (!targetWord) { + throw new Error(`Cannot set text for missing transcript word "${wordId}"`); + } + + const owningSegment = transcript.segments.find((segment) => segment.id === targetWord.segmentId); + if (!owningSegment) { + throw new Error( + `Transcript word "${wordId}" references missing segment "${targetWord.segmentId}"`, + ); + } + if (!owningSegment.wordIds.includes(wordId)) { + throw new Error(`Segment "${owningSegment.id}" does not reference target word "${wordId}"`); + } + + const wordsById = new Map(transcript.words.map((word) => [word.id, word])); + for (const referencedWordId of owningSegment.wordIds) { + if (!wordsById.has(referencedWordId)) { + throw new Error( + `Segment "${owningSegment.id}" references missing word "${referencedWordId}"`, + ); + } + } + + const words = transcript.words.map((word) => (word.id === wordId ? { ...word, text } : word)); + const updatedWordsById = new Map(words.map((word) => [word.id, word])); + const segmentText = joinSegmentText( + transcript.language, + owningSegment.wordIds.map( + (referencedWordId) => updatedWordsById.get(referencedWordId)?.text ?? "", + ), + ); + const segments = transcript.segments.map((segment) => + segment.id === owningSegment.id ? { ...segment, text: segmentText } : segment, + ); + + return { ...transcript, words, segments }; +} From fbce0ca9daa8e7016ede84685ddd2d32ef6013bb Mon Sep 17 00:00:00 2001 From: sunyuchenyaobo <261746743+sunyuchenyaobo@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:51:35 +0800 Subject: [PATCH 02/12] fix(document): validate referenced word ownership and join CJK independently of language tag --- .../ai-edition/document/transcript.test.ts | 11 ++++++++++ src/lib/ai-edition/document/transcript.ts | 21 ++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/lib/ai-edition/document/transcript.test.ts b/src/lib/ai-edition/document/transcript.test.ts index 1678374bc..f9ed108a7 100644 --- a/src/lib/ai-edition/document/transcript.test.ts +++ b/src/lib/ai-edition/document/transcript.test.ts @@ -151,6 +151,8 @@ describe("setWordText", () => { "zh-CN", "zh-TW", "ZH-cn", + "auto", + "yue", ])("does not add artificial spaces between adjacent Chinese content for %s", (language) => { const result = setWordText(transcriptForTokens(language, ["你", "好", "世界"]), "word_2", "们"); @@ -252,6 +254,15 @@ describe("setWordText", () => { ); }); + it("rejects an owning segment that references a word owned by another segment", () => { + const transcript = fixture(); + transcript.segments[0].wordIds.push("word_4"); + + expect(() => setWordText(transcript, "word_2", "replacement")).toThrowError( + /segment_1.*word_4.*segment_2/, + ); + }); + it("rejects an owning segment that omits the target word", () => { const transcript = fixture(); transcript.segments[0].wordIds = ["word_1", "word_3"]; diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts index 45e538d88..7d516dde1 100644 --- a/src/lib/ai-edition/document/transcript.ts +++ b/src/lib/ai-edition/document/transcript.ts @@ -5,18 +5,20 @@ const CLOSING_PUNCTUATION = /^[,.;:!?%。，、；：！？…）)\]}>》」』 const TRAILING_CLOSING_PUNCTUATION = /[,.;:!?%。，、；：！？…）)\]}>》」』】〕]+$/u; const OPENING_PUNCTUATION = /[([<{《「『【〔（]$/u; -function joinSegmentText(language: string, texts: string[]): string { +// The CJK-compaction rule is deliberately LANGUAGE-AGNOSTIC: two adjacent Han / +// Hiragana / Katakana characters never carry a space between them in any script +// that uses them. Gating it on the `language` tag would corrupt transcripts whose +// stored tag is "auto" (a real persisted value — see transcribe.ts's language +// fallback) or "yue": the join would inject ASCII spaces between Chinese runs. +function joinSegmentText(texts: string[]): string { const tokens = texts.map((text) => text.trim()).filter((text) => text.length > 0); - const primaryLanguage = language.split("-")[0].toLowerCase(); - const compactCjk = primaryLanguage === "zh" || primaryLanguage === "ja"; - return tokens.reduce((joined, token) => { if (joined.length === 0) return token; if (CLOSING_PUNCTUATION.test(token) || OPENING_PUNCTUATION.test(joined)) { return joined + token; } const leftContentEdge = joined.replace(TRAILING_CLOSING_PUNCTUATION, "").at(-1) ?? ""; - if (compactCjk && CJK_EDGE.test(leftContentEdge) && CJK_EDGE.test(token[0] ?? "")) { + if (CJK_EDGE.test(leftContentEdge) && CJK_EDGE.test(token[0] ?? "")) { return joined + token; } return `${joined} ${token}`; @@ -45,17 +47,22 @@ export function setWordText( const wordsById = new Map(transcript.words.map((word) => [word.id, word])); for (const referencedWordId of owningSegment.wordIds) { - if (!wordsById.has(referencedWordId)) { + const referencedWord = wordsById.get(referencedWordId); + if (!referencedWord) { throw new Error( `Segment "${owningSegment.id}" references missing word "${referencedWordId}"`, ); } + if (referencedWord.segmentId !== owningSegment.id) { + throw new Error( + `Segment "${owningSegment.id}" references word "${referencedWordId}" which belongs to segment "${referencedWord.segmentId}"`, + ); + } } const words = transcript.words.map((word) => (word.id === wordId ? { ...word, text } : word)); const updatedWordsById = new Map(words.map((word) => [word.id, word])); const segmentText = joinSegmentText( - transcript.language, owningSegment.wordIds.map( (referencedWordId) => updatedWordsById.get(referencedWordId)?.text ?? "", ), From c0012785bc50d4a9390887ed30e58f10a8133535 Mon Sep 17 00:00:00 2001 From: sunyuchenyaobo <261746743+sunyuchenyaobo@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:01:29 +0800 Subject: [PATCH 03/12] fix(document): read CJK segment edges by code point for non-BMP Han --- src/lib/ai-edition/document/transcript.test.ts | 16 ++++++++++++++++ src/lib/ai-edition/document/transcript.ts | 7 +++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/lib/ai-edition/document/transcript.test.ts b/src/lib/ai-edition/document/transcript.test.ts index f9ed108a7..a1c872ffb 100644 --- a/src/lib/ai-edition/document/transcript.test.ts +++ b/src/lib/ai-edition/document/transcript.test.ts @@ -159,6 +159,22 @@ describe("setWordText", () => { expect(result.segments[0].text).toBe("你们世界"); }); + it("does not add a space after a non-BMP Han word (edge read by code point)", () => { + const result = setWordText(transcriptForTokens("zh", ["\u{20000}", "好"]), "word_2", "世界"); + + expect(result.segments[0].text).toBe("\u{20000}世界"); + }); + + it("does not add a space before a token starting with a non-BMP Han character", () => { + const result = setWordText( + transcriptForTokens("zh", ["好", "\u{20000}"]), + "word_2", + "\u{20000}", + ); + + expect(result.segments[0].text).toBe("好\u{20000}"); + }); + it.each([ "ja", "ja-JP", diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts index 7d516dde1..f64588f9e 100644 --- a/src/lib/ai-edition/document/transcript.ts +++ b/src/lib/ai-edition/document/transcript.ts @@ -17,8 +17,11 @@ function joinSegmentText(texts: string[]): string { if (CLOSING_PUNCTUATION.test(token) || OPENING_PUNCTUATION.test(joined)) { return joined + token; } - const leftContentEdge = joined.replace(TRAILING_CLOSING_PUNCTUATION, "").at(-1) ?? ""; - if (CJK_EDGE.test(leftContentEdge) && CJK_EDGE.test(token[0] ?? "")) { + const leftContentEdge = [...joined.replace(TRAILING_CLOSING_PUNCTUATION, "")].at(-1) ?? ""; + // Spread reads the edges by CODE POINT: `.at(-1)` / `[0]` would return half a + // surrogate pair, so a non-BMP Han edge (e.g. U+20000) would miss CJK_EDGE + // and receive an ASCII space. + if (CJK_EDGE.test(leftContentEdge) && CJK_EDGE.test([...token][0] ?? "")) { return joined + token; } return `${joined} ${token}`; From b74565a4124144f23c8c3466f7a065ba7075ad66 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 12:05:15 +0200 Subject: [PATCH 04/12] feat(document): make a corrected word survive its re-transcription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `setWordText` gave the transcript a way to be corrected, but nothing said a word had been. Two consequences: a transcription run REPLACES the asset's transcript, so a user who fixed twenty proper nouns and regenerated lost all twenty without a word about it; and there was no revert, because nothing kept what the transcriber had originally said. `wordSchema` gains the pair that fixes both — `originalText`, captured the first time a user rewrites the word and never overwritten afterwards, and `source`, which also reserves the `"synth"` value a word with no audio behind it will need. Both optional and absent from every document written before them, like `cameraTrack.width`: additive, so no schema bump. `document/transcript.ts` is their only writer and keeps them consistent — typing the original back clears the pair, which IS the revert. `carryOverWordEdits` then re-applies the corrections onto a fresh transcript, and the transcription store calls it on the one path where a run lands. The match is deliberately strict: same original text, overlapping span, one new word per correction. A correction is carried only when the run reproduced the very same mistake at the very same moment, so re-transcribing in another language carries nothing rather than stamping the old language's corrections onto the new words. What could not be carried is counted, not guessed at — saying so to the user needs a string in thirteen locales and belongs with the editing UI, so for now it is a warning in the log. `withTranscript` moves here from `transcribe.ts` and gains `setDocumentWordText` beside it. The document carries the same transcript twice (the per-asset entry and the legacy `transcript` mirror), and writing one without the other leaves the mirror serving pre-edit text forever — the failure that closed #469. It is a pure document operation; the Whisper adapter was not the place to look for it. --- src/lib/ai-edition/document/transcribe.ts | 18 +- .../ai-edition/document/transcript.test.ts | 199 +++++++++++++++++- src/lib/ai-edition/document/transcript.ts | 128 ++++++++++- src/lib/ai-edition/schema/index.ts | 17 ++ .../ai-edition/store/transcriptionStore.ts | 19 +- 5 files changed, 360 insertions(+), 21 deletions(-) diff --git a/src/lib/ai-edition/document/transcribe.ts b/src/lib/ai-edition/document/transcribe.ts index 04d170718..dcfb04216 100644 --- a/src/lib/ai-edition/document/transcribe.ts +++ b/src/lib/ai-edition/document/transcribe.ts @@ -122,18 +122,6 @@ export async function transcribeAsset( }; } -export function withTranscript( - document: AxcutDocument, - transcript: AxcutTranscript, -): AxcutDocument { - const transcripts = [ - ...document.transcripts.filter((t) => t.assetId !== transcript.assetId), - transcript, - ]; - return { - ...document, - transcript: - document.project.primaryAssetId === transcript.assetId ? transcript : document.transcript, - transcripts, - }; -} +// `withTranscript` used to live here. It moved to `document/transcript.ts`, next to +// the other writers of the same object: it is a pure document operation, and the +// Whisper adapter is not where a caller should have to look for it. diff --git a/src/lib/ai-edition/document/transcript.test.ts b/src/lib/ai-edition/document/transcript.test.ts index a1c872ffb..1bc1f5451 100644 --- a/src/lib/ai-edition/document/transcript.test.ts +++ b/src/lib/ai-edition/document/transcript.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import type { AxcutTranscript } from "../schema"; -import { setWordText } from "./transcript"; +import { type AxcutTranscript, createEmptyDocument } from "../schema"; +import { carryOverWordEdits, setDocumentWordText, setWordText, withTranscript } from "./transcript"; function fixture(language = "en"): AxcutTranscript { return { @@ -94,9 +94,13 @@ describe("setWordText", () => { expect(result.segments.map((segment) => segment.id)).toEqual( transcript.segments.map((segment) => segment.id), ); + // The provenance pair rides along with the new text — see "setWordText + // provenance" below for the rules it follows. expect(result.words.find((word) => word.id === "word_2")).toEqual({ ...originalTarget, text: "prefer", + originalText: "use", + source: "user", }); expect(result.segments[0]).toEqual({ ...transcript.segments[0], @@ -288,3 +292,194 @@ describe("setWordText", () => { ); }); }); + +// ─── Provenance ────────────────────────────────────────────────── +// Every field below is what makes a correction survivable: revertible by the +// user, and carryable across a re-transcription. Without them a corrected word +// is indistinguishable from a transcribed one the moment it is written. + +describe("setWordText provenance", () => { + it("records the transcriber's text the first time a word is rewritten", () => { + const word = setWordText(fixture(), "word_3", "OpenScreenApp").words.find( + (w) => w.id === "word_3", + ); + expect(word).toMatchObject({ + text: "OpenScreenApp", + originalText: "OpenScreen", + source: "user", + }); + }); + + it("keeps the FIRST original across later edits, so revert reaches the transcriber's text", () => { + const once = setWordText(fixture(), "word_3", "OpenScreenApp"); + const twice = setWordText(once, "word_3", "OpenScreen Studio"); + expect(twice.words.find((w) => w.id === "word_3")).toMatchObject({ + text: "OpenScreen Studio", + originalText: "OpenScreen", + }); + }); + + it("clears the markers when the original is typed back — that round trip IS the revert", () => { + const edited = setWordText(fixture(), "word_3", "OpenScreenApp"); + const reverted = setWordText(edited, "word_3", "OpenScreen"); + const word = reverted.words.find((w) => w.id === "word_3"); + expect(word?.text).toBe("OpenScreen"); + expect(word).not.toHaveProperty("originalText"); + expect(word).not.toHaveProperty("source"); + }); + + it("leaves a synthesized word synthesized — it has no transcribed text to revert to", () => { + const base = fixture(); + const synth: AxcutTranscript = { + ...base, + words: base.words.map((w) => + w.id === "word_3" ? { ...w, source: "synth" as const, text: "spoken" } : w, + ), + }; + const word = setWordText(synth, "word_3", "rewritten").words.find((w) => w.id === "word_3"); + expect(word).toMatchObject({ text: "rewritten", source: "synth" }); + expect(word).not.toHaveProperty("originalText"); + }); + + it("does not mark the untouched words", () => { + const result = setWordText(fixture(), "word_3", "OpenScreenApp"); + for (const word of result.words.filter((w) => w.id !== "word_3")) { + expect(word).not.toHaveProperty("source"); + } + }); +}); + +// ─── Document-level write ──────────────────────────────────────── +// The document carries the transcript twice. A word edit that writes only one +// copy leaves the legacy mirror serving pre-edit text forever — the failure that +// closed the standalone Python editor (#469). + +function makeDoc(primaryAssetId = "asset_1") { + const base = createEmptyDocument({ title: "Test", projectId: "proj_transcript" }); + return withTranscript({ ...base, project: { ...base.project, primaryAssetId } }, fixture()); +} + +describe("setDocumentWordText", () => { + it("writes BOTH the per-asset transcript and the legacy mirror", () => { + const result = setDocumentWordText(makeDoc(), "asset_1", "word_3", "OpenScreenApp"); + const stored = result.transcripts.find((t) => t.assetId === "asset_1"); + expect(stored?.words.find((w) => w.id === "word_3")?.text).toBe("OpenScreenApp"); + expect(result.transcript?.words.find((w) => w.id === "word_3")?.text).toBe("OpenScreenApp"); + expect(result.transcript).toBe(stored); + }); + + it("leaves the mirror alone when the edited asset is not the primary one", () => { + const doc = makeDoc("asset_other"); + const result = setDocumentWordText(doc, "asset_1", "word_3", "OpenScreenApp"); + expect(result.transcript).toBe(doc.transcript); + expect(result.transcripts.find((t) => t.assetId === "asset_1")?.words).not.toBe( + doc.transcripts.find((t) => t.assetId === "asset_1")?.words, + ); + }); + + it("rejects an asset with no transcript rather than writing a second one", () => { + expect(() => setDocumentWordText(makeDoc(), "asset_missing", "word_3", "x")).toThrow( + /no transcript/, + ); + }); + + it("keeps the input document untouched", () => { + const doc = makeDoc(); + const before = JSON.stringify(doc); + setDocumentWordText(doc, "asset_1", "word_3", "OpenScreenApp"); + expect(JSON.stringify(doc)).toBe(before); + }); +}); + +// ─── Carry-over across a re-transcription ──────────────────────── + +function retranscribed(words: Array<[string, string, number, number]>): AxcutTranscript { + return { + assetId: "asset_1", + language: "en", + segments: [ + { + id: "segment_1", + kind: "speech", + startSec: words[0][2], + endSec: words[words.length - 1][3], + text: words.map(([, text]) => text).join(" "), + wordIds: words.map(([id]) => id), + }, + ], + words: words.map(([id, text, startSec, endSec]) => ({ + id, + segmentId: "segment_1", + startSec, + endSec, + text, + })), + }; +} + +describe("carryOverWordEdits", () => { + const corrected = () => setWordText(fixture(), "word_3", "OpenScreenApp"); + + it("re-applies a correction when the run repeats the same mistake at the same moment", () => { + const next = retranscribed([ + ["w1", "I", 1, 2], + ["w2", "use", 2, 3], + ["w3", "OpenScreen", 3.1, 3.9], + ]); + const result = carryOverWordEdits(corrected(), next); + expect(result.carried).toBe(1); + expect(result.dropped).toBe(0); + expect(result.transcript.words.find((w) => w.id === "w3")).toMatchObject({ + text: "OpenScreenApp", + originalText: "OpenScreen", + source: "user", + }); + // The segment text is rebuilt too, so the captions follow. + expect(result.transcript.segments[0].text).toBe("I use OpenScreenApp"); + }); + + it("drops the correction when the run heard something else there", () => { + const next = retranscribed([["w3", "Open Screen", 3, 4]]); + const result = carryOverWordEdits(corrected(), next); + expect(result).toMatchObject({ carried: 0, dropped: 1 }); + expect(result.transcript).toBe(next); + }); + + it("drops the correction when the same word lands somewhere else entirely", () => { + const next = retranscribed([["w3", "OpenScreen", 40, 41]]); + expect(carryOverWordEdits(corrected(), next)).toMatchObject({ carried: 0, dropped: 1 }); + }); + + it("never lands two corrections on the same new word", () => { + // Both corrections have the SAME original text and both spans overlap the one + // word the new run produced. Without the claim, the second would overwrite the + // first and the count would claim two were saved. + const previous = setWordText( + setWordText( + retranscribed([ + ["p1", "the", 1, 2], + ["p2", "the", 2, 3], + ]), + "p1", + "a", + ), + "p2", + "an", + ); + const result = carryOverWordEdits(previous, retranscribed([["w1", "the", 1, 3]])); + expect(result).toMatchObject({ carried: 1, dropped: 1 }); + expect(result.transcript.words[0].text).toBe("a"); + }); + + it("returns the new transcript untouched when nothing was ever corrected", () => { + const next = retranscribed([["w1", "I", 1, 2]]); + const result = carryOverWordEdits(fixture(), next); + expect(result.transcript).toBe(next); + expect(result).toMatchObject({ carried: 0, dropped: 0 }); + }); + + it("handles a first-ever transcription (no previous transcript)", () => { + const next = retranscribed([["w1", "I", 1, 2]]); + expect(carryOverWordEdits(null, next).transcript).toBe(next); + }); +}); diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts index f64588f9e..6728575b1 100644 --- a/src/lib/ai-edition/document/transcript.ts +++ b/src/lib/ai-edition/document/transcript.ts @@ -1,4 +1,4 @@ -import type { AxcutTranscript } from "../schema"; +import type { AxcutDocument, AxcutTranscript, AxcutWord } from "../schema"; const CJK_EDGE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u; const CLOSING_PUNCTUATION = /^[,.;:!?%。，、；：！？…）)\]}>》」』】〕]/u; @@ -28,6 +28,27 @@ function joinSegmentText(texts: string[]): string { }, ""); } +/** + * Apply the new text to ONE word, keeping its provenance straight. + * + * `originalText` is the transcriber's own text, captured the first time the user + * rewrites the word and never overwritten afterwards — a second edit still reverts + * to what Whisper said, not to the first correction. Typing the original back + * clears the pair, so a round trip leaves no word flagged as corrected whose + * correction is a no-op. + */ +function rewriteWord(word: AxcutWord, text: string): AxcutWord { + // A synthesized word has no transcribed text behind it, so there is nothing to + // revert to and nothing to record: rewriting one leaves it synthesized. + if (word.source === "synth") return { ...word, text }; + const original = word.originalText ?? word.text; + if (text === original) { + const { originalText: _reverted, source: _wasUser, ...rest } = word; + return { ...rest, text }; + } + return { ...word, text, originalText: original, source: "user" }; +} + export function setWordText( transcript: AxcutTranscript, wordId: string, @@ -63,7 +84,9 @@ export function setWordText( } } - const words = transcript.words.map((word) => (word.id === wordId ? { ...word, text } : word)); + const words = transcript.words.map((word) => + word.id === wordId ? rewriteWord(word, text) : word, + ); const updatedWordsById = new Map(words.map((word) => [word.id, word])); const segmentText = joinSegmentText( owningSegment.wordIds.map( @@ -76,3 +99,104 @@ export function setWordText( return { ...transcript, words, segments }; } + +/** + * Write a transcript into the document — the ONLY safe way to do it. + * + * The document carries the same transcript twice: the per-asset `transcripts[]` + * entry, and the legacy `transcript` mirror that a couple of readers still fall + * back to. Writing one without the other leaves two divergent copies on disk, + * where the mirror keeps serving the pre-edit text forever. Nothing outside this + * function may assemble that pair. + * + * Lives here rather than in `transcribe.ts` (which re-exports it for its existing + * importers): it is a pure document operation, and the Whisper adapter is not the + * place a caller should have to look for it. + */ +export function withTranscript( + document: AxcutDocument, + transcript: AxcutTranscript, +): AxcutDocument { + const transcripts = [ + ...document.transcripts.filter((t) => t.assetId !== transcript.assetId), + transcript, + ]; + return { + ...document, + transcript: + document.project.primaryAssetId === transcript.assetId ? transcript : document.transcript, + transcripts, + }; +} + +/** + * {@link setWordText}, addressed the way the UI has it: an asset and a word, not a + * transcript object. Goes through `withTranscript`, so a caller cannot forget the + * legacy mirror. + */ +export function setDocumentWordText( + document: AxcutDocument, + assetId: string, + wordId: string, + text: string, +): AxcutDocument { + const transcript = document.transcripts.find((t) => t.assetId === assetId); + if (!transcript) { + throw new Error(`Cannot edit a word of asset "${assetId}": it has no transcript`); + } + return withTranscript(document, setWordText(transcript, wordId, text)); +} + +/** What {@link carryOverWordEdits} managed to save from the previous transcript. */ +export interface WordEditCarryOver { + transcript: AxcutTranscript; + /** Corrections re-applied to the new transcript. */ + carried: number; + /** Corrections the new transcript left no place for. These are lost. */ + dropped: number; +} + +/** + * Re-apply the user's word corrections onto a freshly transcribed transcript. + * + * A transcription run REPLACES the asset's transcript wholesale, so without this a + * user who fixed twenty proper nouns and then regenerated lost all twenty, silently. + * + * The match is deliberately strict — same original text, overlapping span, one new + * word per correction. A correction is carried only when the new run reproduced the + * very same mistake at the very same moment; re-transcribing in another language + * therefore carries nothing rather than stamping French corrections onto Spanish + * words. What could not be carried is counted, not guessed at, so the caller can say + * so. + */ +export function carryOverWordEdits( + previous: AxcutTranscript | null | undefined, + next: AxcutTranscript, +): WordEditCarryOver { + const edits = (previous?.words ?? []).filter( + (word) => word.source === "user" && word.originalText !== undefined, + ); + if (edits.length === 0) return { transcript: next, carried: 0, dropped: 0 }; + + // Candidates are read from `next` throughout, never from the transcript being + // built up: a word already rewritten by an earlier correction no longer carries + // the text the next one matches on, and `claimed` is what stops two corrections + // from landing on the same word. + const claimed = new Set(); + let transcript = next; + let carried = 0; + for (const edit of edits) { + const match = next.words.find( + (word) => + !claimed.has(word.id) && + word.text === edit.originalText && + word.endSec > edit.startSec && + word.startSec < edit.endSec, + ); + if (!match) continue; + claimed.add(match.id); + transcript = setWordText(transcript, match.id, edit.text); + carried += 1; + } + return { transcript, carried, dropped: edits.length - carried }; +} diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index 00c429de5..9a511fee5 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -63,6 +63,23 @@ export const wordSchema = z startSec: z.number().nonnegative(), endSec: z.number().nonnegative(), text: z.string(), + // Provenance of the TEXT, so a hand-corrected word can be told from a + // transcribed one. Both fields are additive and absent on every document + // written before them (like `cameraTrack.width`), so no schema bump: an + // older build simply drops them on save. + // + // `document/transcript.ts` is the only writer, and it keeps the pair + // consistent: `originalText` is set from the ASR text the first time a user + // rewrites the word and never overwritten afterwards, so it stays the revert + // target however many times the word is edited; typing the original back + // clears both, which IS the revert. + // + // Absent `source` means the word came from the transcriber. It is what makes + // a re-transcription able to carry the user's corrections forward + // (`carryOverWordEdits`) instead of silently discarding them — and what a + // future TTS pass will read to know which words it has to speak. + originalText: z.string().optional(), + source: z.enum(["asr", "user", "synth"]).optional(), }) .refine((data) => data.endSec >= data.startSec, { message: "endSec must be greater than or equal to startSec", diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.ts index 64d910b13..940dfbbf5 100644 --- a/src/lib/ai-edition/store/transcriptionStore.ts +++ b/src/lib/ai-edition/store/transcriptionStore.ts @@ -26,7 +26,8 @@ import { useEffect, useMemo } from "react"; import { toast } from "sonner"; import { create } from "zustand"; import { toastText as translateToast } from "@/i18n/toastText"; -import { transcribeAsset, withTranscript } from "../document/transcribe"; +import { transcribeAsset } from "../document/transcribe"; +import { carryOverWordEdits, withTranscript } from "../document/transcript"; import type { AxcutDocument } from "../schema"; import { type AssetTranscriptionView, @@ -399,6 +400,20 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { dropJob(assetId, runId); return; } + // A run REPLACES the asset's transcript, so any word the user had corrected by + // hand would go with it. Carry those corrections onto the new words first — + // strictly, so nothing is invented (see `carryOverWordEdits`). What could not be + // carried is lost; telling the user so is the UI's job, and there is no surface + // for it yet. + const merged = carryOverWordEdits( + current.transcripts.find((t) => t.assetId === assetId), + transcript, + ); + if (merged.dropped > 0) { + console.warn( + `[transcription] ${merged.dropped} word correction(s) on asset ${assetId} could not be carried over to the new transcript.`, + ); + } // One save: the transcript, and (on a successful retry) the removal of // the verdict remembered on the asset. // `history: false`: a transcript landing from a background job is not an edit @@ -412,7 +427,7 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { a.id === assetId && a.transcriptionFailure ? { ...a, transcriptionFailure: null } : a, ), }, - transcript, + merged.transcript, ), { history: false }, ); From f33571cbf0893e6dfd19331993257f3c1c75a089 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 13:43:33 +0200 Subject: [PATCH 05/12] feat(editor): correct a word in the transcript without cutting the film MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transcript pane looked like a document and behaved like one only halfway: Backspace on a word wrote a trimRange and cut the media, while typing was blocked outright — `handleBeforeInput` said so in a comment, and correcting a mis-transcribed word had no in-app path at all. The two gestures now share the one word stream, with no mode to remember and no second tab: Backspace still cuts, and a double-click opens the word for editing in place. Enter or a click away commits, Escape abandons. A correction writes `transcript.words[].text` and nothing else — the captions follow it, the timeline does not move. Everything the field raises is stopped at the field, so a Backspace inside it types instead of cutting the clip out from under the caret. A corrected word says so: the accent colour, a dotted underline, and a tooltip naming what the transcriber actually heard. Hovering it offers the revert, which writes the original back through the same path that clears the provenance pair — there is no second "unedit" operation that could fall out of step with the first. It sits where the bin sits on a cut word, in the accent rather than the danger colour, and the two never appear together. Emptying a word is how a junk token leaves the captions without the audio going with it, and the caption pipeline already drops empty words. Rendered as its own text, though, such a word is a bare space: invisible, un-clickable, impossible to undo. It gets a chip instead, so it keeps a place in the stream. Writes go through `setDocumentWordText` on the same serialised queue as the trims, so correcting a word and cutting the next one cannot overwrite each other's save. Verified end to end in the browser preview: the edit lands on the word, rebuilds its segment, reaches the legacy `document.transcript` mirror, and the revert takes all three back. --- src/components/ai-edition/NewEditorShell.tsx | 27 +++ src/components/ai-edition/RightPanes.tsx | 207 ++++++++++++++++- .../ai-edition/TranscriptPane.gating.test.tsx | 1 + .../TranscriptPane.keyboardCut.test.tsx | 2 + .../TranscriptPane.sharedMedia.test.tsx | 1 + .../TranscriptPane.wordEdit.test.tsx | 213 ++++++++++++++++++ src/i18n/locales/ar/editor.json | 3 +- src/i18n/locales/ar/settings.json | 6 +- src/i18n/locales/en/editor.json | 3 +- src/i18n/locales/en/settings.json | 6 +- src/i18n/locales/es/editor.json | 3 +- src/i18n/locales/es/settings.json | 6 +- src/i18n/locales/fr/editor.json | 3 +- src/i18n/locales/fr/settings.json | 6 +- src/i18n/locales/it/editor.json | 3 +- src/i18n/locales/it/settings.json | 6 +- src/i18n/locales/ja-JP/editor.json | 3 +- src/i18n/locales/ja-JP/settings.json | 6 +- src/i18n/locales/ko-KR/editor.json | 3 +- src/i18n/locales/ko-KR/settings.json | 6 +- src/i18n/locales/pt-BR/editor.json | 3 +- src/i18n/locales/pt-BR/settings.json | 6 +- src/i18n/locales/ru/editor.json | 3 +- src/i18n/locales/ru/settings.json | 6 +- src/i18n/locales/tr/editor.json | 3 +- src/i18n/locales/tr/settings.json | 6 +- src/i18n/locales/vi/editor.json | 3 +- src/i18n/locales/vi/settings.json | 6 +- src/i18n/locales/zh-CN/editor.json | 3 +- src/i18n/locales/zh-CN/settings.json | 6 +- src/i18n/locales/zh-TW/editor.json | 3 +- src/i18n/locales/zh-TW/settings.json | 6 +- .../store/documentWriteAudit.test.ts | 3 + 33 files changed, 542 insertions(+), 29 deletions(-) create mode 100644 src/components/ai-edition/TranscriptPane.wordEdit.test.tsx diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index b371d100c..ddafae140 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -13,6 +13,7 @@ import { applyProbedDuration, replaceTimeline as replaceTimelineOp, } from "@/lib/ai-edition/document/timeline"; +import { setDocumentWordText } from "@/lib/ai-edition/document/transcript"; import { isModalOpen } from "@/lib/ai-edition/modalGuard"; import { type AxcutClip, documentSchema } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; @@ -604,6 +605,31 @@ export function NewEditorShell() { [applyTimelineOp], ); + // transcript-pane → the word's own text. Unlike Backspace (which writes a trimRange and + // cuts the media), this writes only `transcript.words[].text`: the captions follow, the + // film is untouched. Queued on the SAME chain as the trims so correcting a word and + // cutting the next one cannot overwrite each other's save. + const handleSetWordText = useCallback( + (assetId: string, wordId: string, text: string) => { + void enqueueTimelineWrite(async () => { + // Read inside the chain: the previous save has resolved by now, so the store + // holds the document this edit has to be applied to. + const doc = useProjectStore.getState().document; + if (!doc) return; + try { + await saveDocument(setDocumentWordText(doc, assetId, wordId, text), { history: true }); + } catch (err) { + // The word or its transcript vanished under the edit (a regeneration landed + // mid-typing). Nothing to retry — say so rather than dropping it silently. + toast.error(te("errors.wordEditFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + const handleSelectProject = useCallback( async (id: string) => { try { @@ -1151,6 +1177,7 @@ export function NewEditorShell() { onSeek: handleSeek, onAddTrimRange: handleAddTrimRange, onRemoveTrimRange: handleRemoveTrimRange, + onSetWordText: handleSetWordText, onTranscribe: handleTranscribe, canTranscribe: hasAsset, isTranscribing: transcriptGate.state === "pending", diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 2e15f7b12..d01e98753 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -15,6 +15,7 @@ import { MousePointerClick, Sliders, Trash2, + Undo2, } from "lucide-react"; import { @@ -704,6 +705,7 @@ export function TranscriptPane({ onSeek, onAddTrimRange, onRemoveTrimRange, + onSetWordText, onTranscribe, canTranscribe, isTranscribing, @@ -722,6 +724,10 @@ export function TranscriptPane({ onSeek: (sec: number) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; onRemoveTrimRange: (trimId: string) => void; + /** Rewrite ONE word's text. Takes the bare `AxcutWord.id`, never the clip-scoped + * `ClipWord.id`: the transcript belongs to the asset, so a correction lands on the + * media and shows on every clip that plays it — which is the point. */ + onSetWordText: (assetId: string, wordId: string, text: string) => void; onTranscribe: () => void; canTranscribe: boolean; isTranscribing: boolean; @@ -832,6 +838,7 @@ export function TranscriptPane({ onSeek={onSeek} onAddTrimRange={onAddTrimRange} onRemoveTrimRange={onRemoveTrimRange} + onSetWordText={onSetWordText} /> ))} @@ -859,6 +866,7 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ onSeek, onAddTrimRange, onRemoveTrimRange, + onSetWordText, }: { index: number; section: ClipSection; @@ -867,6 +875,7 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ onSeek: (sec: number) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; onRemoveTrimRange: (trimId: string) => void; + onSetWordText: (assetId: string, wordId: string, text: string) => void; }) { const ts = useScopedT("settings"); const { clip, asset, words } = section; @@ -1210,9 +1219,11 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ key={cw.id} cw={cw} isCue={cw.id === cueWordId} + editable={!busy} target={trimTarget} onRestore={removeTrimRun} onAddTrimRange={onAddTrimRange} + onSetWordText={onSetWordText} /> ))} @@ -1243,19 +1254,54 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ const TranscriptWord = memo(function TranscriptWord({ cw, isCue, + editable, target, onRestore, onAddTrimRange, + onSetWordText, }: { cw: ClipWord; isCue: boolean; + /** False while this clip's transcript is being regenerated — the words on screen are + * about to be replaced, so an edit typed into them would be thrown away. */ + editable: boolean; target: TrimTarget; onRestore: (run: TrimRun) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; + onSetWordText: (assetId: string, wordId: string, text: string) => void; }) { const ts = useScopedT("settings"); const [hover, setHover] = useState(false); + // The text being typed, or null when the word is not under edit. + const [draft, setDraft] = useState(null); + // Escape unmounts the field, and an abandoned field's blur must not commit what the + // user just walked away from. + const abandonedRef = useRef(false); const removed = !cw.kept; + // `originalText` is only ever written by a user edit (see `document/transcript.ts`), so + // it is what tells a corrected word from a transcribed one. + const original = cw.word.originalText; + const corrected = original !== undefined; + const blanked = corrected && cw.word.text.trim().length === 0; + + const startEditing = useCallback(() => { + if (!editable) return; + setDraft(cw.word.text); + }, [editable, cw.word.text]); + + const commitDraft = useCallback(() => { + const next = (draft ?? "").trim(); + setDraft(null); + if (next === cw.word.text) return; + onSetWordText(target.assetId, cw.word.id, next); + }, [draft, cw.word.text, cw.word.id, onSetWordText, target.assetId]); + + const revert = useCallback(() => { + if (original === undefined) return; + // Writing the original back through the same path is what clears the provenance + // pair — there is no separate "unedit" operation that could fall out of step. + onSetWordText(target.assetId, cw.word.id, original); + }, [original, cw.word.id, onSetWordText, target.assetId]); if (isSilenceWord(cw.word)) { const durationSec = cw.word.endSec - cw.word.startSec; @@ -1333,25 +1379,140 @@ const TranscriptWord = memo(function TranscriptWord({ ); } + // The inline editor. `contentEditable={false}` keeps the browser from treating it as + // part of the enclosing editable block, and every event it raises is stopped here rather + // than in the block handlers: Backspace inside the field has to type, not cut, and a + // click in it must not seek. + if (draft !== null) { + return ( + setDraft(event.target.value)} + onFocus={(event) => event.currentTarget.select()} + onBlur={() => { + if (abandonedRef.current) { + abandonedRef.current = false; + return; + } + commitDraft(); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") { + event.preventDefault(); + commitDraft(); + } else if (event.key === "Escape") { + event.preventDefault(); + abandonedRef.current = true; + setDraft(null); + } + }} + onBeforeInput={(event) => event.stopPropagation()} + onPaste={(event) => event.stopPropagation()} + onPointerUp={(event) => event.stopPropagation()} + style={{ + display: "inline", + // `ch` is the digit width, not the real glyph width, so this only + // approximates the word it replaces — the slack keeps it from clipping. + width: `${Math.max(draft.length, 3) + 2}ch`, + margin: 0, + padding: "0 2px", + border: 0, + borderBottom: "2px solid var(--accent)", + borderRadius: 0, + background: "var(--accent-soft)", + color: "var(--fg)", + font: "inherit", + outline: "none", + }} + /> + ); + } + + // A word the user emptied. It still owns a span of the media, so it keeps a place in + // the stream: rendered as its own (empty) text it would be a bare space — invisible, + // impossible to click, and therefore impossible to undo. + if (blanked) { + return ( + setHover(true)} + onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} + > + + {ts("transcript.blankedWord")} + + {hover ? ( + + ) : null}{" "} + + ); + } + return ( setHover(true)} onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + // Without this the browser selects the word inside the enclosing + // contentEditable; the field about to replace it does its own selecting. + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} > {/* no filler chip. axcut renders every word the same way; the LLM is the only place that names a word a filler (via the @@ -1394,10 +1555,50 @@ const TranscriptWord = memo(function TranscriptWord({ ); }); +/** Hover affordance on a corrected word: put the transcriber's own text back. Mirrors the + * bin on a cut word — same size, same place, the accent rather than the danger colour, + * since reverting a correction restores something instead of removing it. */ +function RevertWordButton({ label, onRevert }: { label: string; onRevert: () => void }) { + return ( + + ); +} + // ─── Caret / selection helpers ──────────────────────────────────── // Ponytail port of axcut's findCollapsedDeletionWordId. The non-collapsed // path uses findWordId directly (a range selection's endpoints already diff --git a/src/components/ai-edition/TranscriptPane.gating.test.tsx b/src/components/ai-edition/TranscriptPane.gating.test.tsx index 008c6161f..34d97dfd3 100644 --- a/src/components/ai-edition/TranscriptPane.gating.test.tsx +++ b/src/components/ai-edition/TranscriptPane.gating.test.tsx @@ -56,6 +56,7 @@ function renderPane( onSeek={vi.fn()} onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={overrides.isTranscribing ?? false} diff --git a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx index 600982fe0..520e4b81b 100644 --- a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx +++ b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx @@ -84,6 +84,7 @@ function renderPane( onSeek={vi.fn()} onAddTrimRange={onAddTrimRange} onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} @@ -227,6 +228,7 @@ describe("keyboard cut with the caret between words", () => { ]) } onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx index f11151122..c3ad72571 100644 --- a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx +++ b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx @@ -78,6 +78,7 @@ function renderPane(onSeek: (sec: number) => void = vi.fn()) { onSeek={onSeek} onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} + onSetWordText={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx new file mode 100644 index 000000000..2514fb23f --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx @@ -0,0 +1,213 @@ +// @vitest-environment jsdom +// Correcting a word's TEXT in the transcript pane, as opposed to cutting it. +// +// The two gestures share one word stream on purpose (no mode, no second tab), so what +// keeps them apart is which gesture the user makes: Backspace cuts the media, a +// double-click rewrites the text. These tests hold that line — a keystroke inside the +// editing field must never reach the cut path, and a correction must never touch the +// timeline. + +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutClip, AxcutTranscript, AxcutWord } from "@/lib/ai-edition/schema"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 3, + cameraTrack: null, +}; + +const CLIP: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 3, + timelineStartSec: 0, + timelineEndSec: 3, + wordRefs: [], + origin: "user", + reason: "", +}; + +// Contiguous: a gap would insert a `[silence]` pill between the words and move the +// indices these tests address words by. +const WORDS: AxcutWord[] = [ + { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "Bonjour" }, + { id: "w2", segmentId: "s", startSec: 1, endSec: 2, text: "Kubernetes" }, + { id: "w3", segmentId: "s", startSec: 2, endSec: 3, text: "tout" }, +]; + +function transcript(words: AxcutWord[] = WORDS): AxcutTranscript { + return { assetId: "asset_1", language: "fr", segments: [], words }; +} + +function renderPane(words?: AxcutWord[], busyAssetIds: string[] = []) { + const onSetWordText = vi.fn(); + const onAddTrimRange = vi.fn(); + const view = render( + + + , + ); + const wordEl = (id: string) => { + const el = view.container.querySelector(`[data-word-id="clip_1:${id}"]`); + if (!el) throw new Error(`word ${id} not rendered`); + return el; + }; + const field = () => view.container.querySelector("input[data-word-editor]"); + return { ...view, wordEl, field, onSetWordText, onAddTrimRange }; +} + +afterEach(cleanup); + +describe("correcting a word", () => { + it("opens an editing field on the word a double-click lands on", () => { + const view = renderPane(); + expect(view.field()).toBeNull(); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toHaveValue("Kubernetes"); + }); + + it("commits on Enter, addressing the word by its BARE id and the clip's asset", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Kubernetes 1.31" } }); + fireEvent.keyDown(field, { key: "Enter" }); + // `clip_1:w2` is what the DOM node carries; the transcript knows only `w2`. + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Kubernetes 1.31"); + }); + + it("commits on blur, so clicking away does not throw the correction out", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w1")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Bonsoir" } }); + fireEvent.blur(field); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w1", "Bonsoir"); + }); + + it("abandons on Escape, and a blur afterwards does not resurrect the draft", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w1")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Bonsoir" } }); + fireEvent.keyDown(field, { key: "Escape" }); + fireEvent.blur(field); + expect(view.onSetWordText).not.toHaveBeenCalled(); + expect(view.field()).toBeNull(); + }); + + it("writes nothing when the text comes back unchanged", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onSetWordText).not.toHaveBeenCalled(); + }); + + // The field lives inside the block's contentEditable, whose Backspace handler cuts the + // media. Without the stopPropagation on the field, deleting a letter would trim the clip. + it("does not cut the media when Backspace is pressed inside the field", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.keyDown(field, { key: "Backspace" }); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); + + it("stays read-only while the transcript is being regenerated", () => { + const view = renderPane(undefined, ["asset_1"]); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toBeNull(); + }); +}); + +describe("a word already corrected", () => { + const CORRECTED: AxcutWord[] = [ + WORDS[0], + { ...WORDS[1], text: "Kubernetes", originalText: "Cuber Nettes", source: "user" }, + WORDS[2], + ]; + + it("is marked as corrected and names what the transcriber heard", () => { + const view = renderPane(CORRECTED); + const el = view.wordEl("w2"); + expect(el).toHaveAttribute("data-corrected", "true"); + expect(el.title).toContain("Cuber Nettes"); + }); + + it("offers a revert that writes the transcriber's own text back", () => { + const view = renderPane(CORRECTED); + fireEvent.mouseEnter(view.wordEl("w2")); + const revert = view.wordEl("w2").querySelector("button"); + if (!revert) throw new Error("no revert control"); + fireEvent.click(revert); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Cuber Nettes"); + }); + + it("leaves an untouched word unmarked and without a revert", () => { + const view = renderPane(CORRECTED); + fireEvent.mouseEnter(view.wordEl("w1")); + expect(view.wordEl("w1")).not.toHaveAttribute("data-corrected"); + expect(view.wordEl("w1").querySelector("button")).toBeNull(); + }); +}); + +// Emptying a word is how a junk token gets out of the captions without cutting the audio. +// Rendered as its own (empty) text it would be a bare space: invisible, un-clickable, and +// therefore impossible to undo. +describe("a word the user emptied", () => { + const BLANKED: AxcutWord[] = [ + WORDS[0], + { ...WORDS[1], text: "", originalText: "Kubernetes", source: "user" }, + WORDS[2], + ]; + + it("keeps a visible, clickable place in the stream", () => { + const view = renderPane(BLANKED); + const el = view.wordEl("w2"); + expect(el).toHaveAttribute("data-blanked", "true"); + expect(el.textContent?.trim()).not.toBe(""); + }); + + it("can be reopened for editing and reverted", () => { + const view = renderPane(BLANKED); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toHaveValue(""); + + fireEvent.keyDown(view.field() as HTMLInputElement, { key: "Escape" }); + fireEvent.mouseEnter(view.wordEl("w2")); + const revert = view.wordEl("w2").querySelector("button"); + if (!revert) throw new Error("no revert control"); + fireEvent.click(revert); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Kubernetes"); + }); +}); diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 39860c0da..93f340aaa 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "فشل حفظ التصدير", "failedToSaveExportedVideo": "فشل حفظ الفيديو المُصدَّر", "failedToRevealInFolder": "خطأ في الكشف في المجلد: {{error}}", - "previewCompositorUnavailable": "المعاينة غير متوفرة على هذا الجهاز" + "previewCompositorUnavailable": "المعاينة غير متوفرة على هذا الجهاز", + "wordEditFailed": "تعذّر تغيير هذه الكلمة" }, "export": { "canceled": "تم إلغاء التصدير", diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json index 5fc1ac58c..b211f0a7d 100644 --- a/src/i18n/locales/ar/settings.json +++ b/src/i18n/locales/ar/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "النص الحالي", - "help": "نص مجمّع لكل المقاطع على المخطط الزمني. اضغط Backspace / Delete على كلمة أو تحديد لتمييزه كمتخطّى (بالأحمر). مرّر المؤشر فوق المقطع الأحمر لاستعادته.", + "help": "نص مجمّع لكل المقاطع على المخطط الزمني. اضغط Backspace / Delete على كلمة أو تحديد لقصّها من الفيديو (بالأحمر). انقر نقرًا مزدوجًا على كلمة لتصحيح نصها: تتبعها الترجمات ولا يتغيّر الفيديو. مرّر المؤشر فوق كلمة معلَّمة لاستعادتها.", "noClips": "لا توجد مقاطع بعد", "noTranscript": "لا يوجد نص بعد", "whisperHint": "يستخدم التفريغ النصي Whisper محليًا — يعمل على حاسوبك، ولا تغادر أي بيانات جهازك.", @@ -277,6 +277,10 @@ "restoreSilence": "استعادة الصمت ({{duration}} ث)", "trimSilence": "قص الصمت ({{duration}} ث)", "restoreWord": "استعادة \"{{word}}\"", + "editWord": "تحرير \"{{word}}\"", + "correctedWord": "مصحّحة — كان النص \"{{original}}\"", + "revertWord": "استعادة \"{{original}}\"", + "blankedWord": "مُفرَّغة", "noAudio": "لا يحتوي هذا الملف على مسار صوتي" }, "captions": { diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 7aa5954ce..bb167861a 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "Failed to save export", "failedToSaveExportedVideo": "Failed to save exported video", "failedToRevealInFolder": "Error revealing in folder: {{error}}", - "previewCompositorUnavailable": "Preview unavailable on this machine" + "previewCompositorUnavailable": "Preview unavailable on this machine", + "wordEditFailed": "Could not change that word" }, "export": { "canceled": "Export canceled", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index b291460d7..06ef40395 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -270,7 +270,7 @@ }, "transcript": { "title": "Current transcription", - "help": "Aggregated transcript of every clip on the timeline. Backspace / Delete a word or selection to mark it as skipped (red). Hover a red span to restore it.", + "help": "Aggregated transcript of every clip on the timeline. Backspace / Delete a word or selection to cut it out of the film (red). Double-click a word to correct its text — the captions follow, the film does not move. Hover a marked word to restore it.", "noClips": "No clips yet", "noTranscript": "No transcript yet", "whisperHint": "Transcribe uses local Whisper — runs on your computer, no data leaves the device.", @@ -283,6 +283,10 @@ "restoreSilence": "Restore silence ({{duration}}s)", "trimSilence": "Trim silence ({{duration}}s)", "restoreWord": "Restore \"{{word}}\"", + "editWord": "Edit \"{{word}}\"", + "correctedWord": "Corrected — the transcriber heard \"{{original}}\"", + "revertWord": "Restore \"{{original}}\"", + "blankedWord": "blanked", "noAudio": "This media has no audio track" }, "captions": { diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 4d10b36a6..891d3d16d 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -12,7 +12,8 @@ "failedToSaveExport": "Error al guardar la exportación", "failedToSaveExportedVideo": "Error al guardar el video exportado", "failedToRevealInFolder": "Error al mostrar en la carpeta: {{error}}", - "previewCompositorUnavailable": "Vista previa no disponible en este equipo" + "previewCompositorUnavailable": "Vista previa no disponible en este equipo", + "wordEditFailed": "No se pudo cambiar esa palabra" }, "export": { "canceled": "Exportación cancelada", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index bb9ee27a2..c47036224 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Transcripción actual", - "help": "Transcripción agregada de todos los clips de la línea de tiempo. Retroceso / Supr sobre una palabra o selección la marca como omitida (en rojo). Pasa el cursor sobre un fragmento rojo para restaurarlo.", + "help": "Transcripción agregada de todos los clips de la línea de tiempo. Retroceso / Supr sobre una palabra o selección la corta del vídeo (en rojo). Haz doble clic en una palabra para corregir su texto: los subtítulos la siguen, el vídeo no cambia. Pasa el cursor sobre una palabra marcada para restaurarla.", "noClips": "Aún no hay clips", "noTranscript": "Aún no hay transcripción", "whisperHint": "La transcripción usa Whisper local: se ejecuta en tu equipo y ningún dato sale del dispositivo.", @@ -277,6 +277,10 @@ "restoreSilence": "Restaurar silencio ({{duration}} s)", "trimSilence": "Recortar silencio ({{duration}} s)", "restoreWord": "Restaurar «{{word}}»", + "editWord": "Editar «{{word}}»", + "correctedWord": "Corregida: la transcripción decía «{{original}}»", + "revertWord": "Restaurar «{{original}}»", + "blankedWord": "vaciada", "noAudio": "Este medio no tiene pista de audio" }, "captions": { diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 7a719ade2..f7a04a264 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -18,7 +18,8 @@ "failedToSaveExport": "Échec de l'enregistrement de l'export", "failedToSaveExportedVideo": "Échec de l'enregistrement de la vidéo exportée", "failedToRevealInFolder": "Erreur lors de l'affichage dans le dossier : {{error}}", - "previewCompositorUnavailable": "Aperçu indisponible sur cette machine" + "previewCompositorUnavailable": "Aperçu indisponible sur cette machine", + "wordEditFailed": "Impossible de modifier ce mot" }, "export": { "canceled": "Export annulé", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 23dd573f0..ddfb2bbdf 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Transcription actuelle", - "help": "Transcription agrégée de tous les clips de la timeline. Retour arrière / Suppr sur un mot ou une sélection le marque comme ignoré (en rouge). Survolez un passage rouge pour le restaurer.", + "help": "Transcription agrégée de tous les clips de la timeline. Retour arrière / Suppr sur un mot ou une sélection le coupe du film (en rouge). Double-cliquez sur un mot pour corriger son texte : les sous-titres suivent, le film ne bouge pas. Survolez un mot marqué pour le rétablir.", "noClips": "Aucun clip pour l'instant", "noTranscript": "Aucune transcription pour l'instant", "whisperHint": "La transcription utilise Whisper en local — tout s'exécute sur votre ordinateur, aucune donnée ne quitte l'appareil.", @@ -277,6 +277,10 @@ "restoreSilence": "Restaurer le silence ({{duration}} s)", "trimSilence": "Couper le silence ({{duration}} s)", "restoreWord": "Restaurer « {{word}} »", + "editWord": "Modifier « {{word}} »", + "correctedWord": "Corrigé — la transcription disait « {{original}} »", + "revertWord": "Rétablir « {{original}} »", + "blankedWord": "vidé", "noAudio": "Ce média n'a pas de piste audio" }, "captions": { diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 70a680a7a..1734c8c82 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "Impossibile salvare l'esportazione", "failedToSaveExportedVideo": "Impossibile salvare il video esportato", "failedToRevealInFolder": "Errore durante la visualizzazione nella cartella: {{error}}", - "previewCompositorUnavailable": "Anteprima non disponibile su questo computer" + "previewCompositorUnavailable": "Anteprima non disponibile su questo computer", + "wordEditFailed": "Impossibile modificare questa parola" }, "export": { "canceled": "Esportazione annullata", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index e10828765..565cf1864 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Trascrizione corrente", - "help": "Trascrizione aggregata di tutti i clip sulla timeline. Backspace / Canc su una parola o selezione la segna come saltata (in rosso). Passa sopra un tratto rosso per ripristinarlo.", + "help": "Trascrizione aggregata di tutti i clip sulla timeline. Backspace / Canc su una parola o selezione la taglia dal video (in rosso). Fai doppio clic su una parola per correggerne il testo: i sottotitoli la seguono, il video non cambia. Passa sopra una parola contrassegnata per ripristinarla.", "noClips": "Ancora nessun clip", "noTranscript": "Ancora nessuna trascrizione", "whisperHint": "La trascrizione usa Whisper in locale: gira sul tuo computer, nessun dato lascia il dispositivo.", @@ -277,6 +277,10 @@ "restoreSilence": "Ripristina silenzio ({{duration}} s)", "trimSilence": "Taglia silenzio ({{duration}} s)", "restoreWord": "Ripristina «{{word}}»", + "editWord": "Modifica «{{word}}»", + "correctedWord": "Corretta — la trascrizione diceva «{{original}}»", + "revertWord": "Ripristina «{{original}}»", + "blankedWord": "svuotata", "noAudio": "Questo contenuto non ha una traccia audio" }, "captions": { diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index bcbc57164..2686aaff8 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -20,7 +20,8 @@ "failedToSaveExportedVideo": "エクスポートした動画の保存に失敗しました", "failedToRevealInFolder": "フォルダの表示に失敗しました: {{error}}", "exportBackgroundLoadFailed": "エクスポートに失敗しました: 背景画像を読み込めませんでした ({{url}})", - "previewCompositorUnavailable": "このマシンではプレビューを表示できません" + "previewCompositorUnavailable": "このマシンではプレビューを表示できません", + "wordEditFailed": "この単語を変更できませんでした" }, "export": { "canceled": "エクスポートがキャンセルされました", diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json index ead358d48..96807f17f 100644 --- a/src/i18n/locales/ja-JP/settings.json +++ b/src/i18n/locales/ja-JP/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "現在の文字起こし", - "help": "タイムライン上のすべてのクリップをまとめた文字起こしです。単語や選択範囲を Backspace / Delete でスキップ（赤色）にできます。赤い部分にカーソルを合わせると元に戻せます。", + "help": "タイムライン上のすべてのクリップをまとめた文字起こしです。単語や選択範囲を Backspace / Delete で映像から切り取れます（赤色）。単語をダブルクリックすると文字を修正できます。字幕は追従し、映像は変わりません。印の付いた単語にカーソルを合わせると元に戻せます。", "noClips": "クリップがまだありません", "noTranscript": "文字起こしがまだありません", "whisperHint": "文字起こしはローカルの Whisper を使用します。お使いのパソコン上で実行され、データが端末外に出ることはありません。", @@ -277,6 +277,10 @@ "restoreSilence": "無音を元に戻す（{{duration}} 秒）", "trimSilence": "無音をトリム（{{duration}} 秒）", "restoreWord": "「{{word}}」を元に戻す", + "editWord": "「{{word}}」を編集", + "correctedWord": "修正済み — 文字起こしでは「{{original}}」でした", + "revertWord": "「{{original}}」に戻す", + "blankedWord": "空欄", "noAudio": "このメディアには音声トラックがありません" }, "captions": { diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 96e6c5339..16400545c 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "내보낸 파일 저장에 실패했습니다", "failedToSaveExportedVideo": "내보낸 비디오 저장에 실패했습니다", "failedToRevealInFolder": "폴더에서 파일 표시 오류: {{error}}", - "previewCompositorUnavailable": "이 컴퓨터에서는 미리보기를 사용할 수 없습니다" + "previewCompositorUnavailable": "이 컴퓨터에서는 미리보기를 사용할 수 없습니다", + "wordEditFailed": "이 단어를 변경할 수 없습니다" }, "export": { "canceled": "내보내기가 취소되었습니다", diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json index 631189192..9410e4dbe 100644 --- a/src/i18n/locales/ko-KR/settings.json +++ b/src/i18n/locales/ko-KR/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "현재 전사", - "help": "타임라인의 모든 클립을 합친 전사입니다. 단어나 선택 영역에서 Backspace / Delete를 누르면 건너뛴 것으로 표시됩니다(빨간색). 빨간 부분에 마우스를 올리면 복원할 수 있습니다.", + "help": "타임라인의 모든 클립을 합친 전사입니다. 단어나 선택 영역에서 Backspace / Delete를 누르면 영상에서 잘라냅니다(빨간색). 단어를 두 번 클릭하면 텍스트를 고칠 수 있습니다. 자막은 따라가고 영상은 그대로입니다. 표시된 단어에 마우스를 올리면 되돌릴 수 있습니다.", "noClips": "아직 클립이 없습니다", "noTranscript": "아직 전사가 없습니다", "whisperHint": "전사는 로컬 Whisper를 사용합니다. 이 컴퓨터에서 실행되며 데이터가 기기를 벗어나지 않습니다.", @@ -277,6 +277,10 @@ "restoreSilence": "무음 복원 ({{duration}}초)", "trimSilence": "무음 자르기 ({{duration}}초)", "restoreWord": "\"{{word}}\" 복원", + "editWord": "\"{{word}}\" 편집", + "correctedWord": "수정됨 — 전사에는 \"{{original}}\"였습니다", + "revertWord": "\"{{original}}\"(으)로 되돌리기", + "blankedWord": "비움", "noAudio": "이 미디어에는 오디오 트랙이 없습니다" }, "captions": { diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index e5f828d3c..9b0054861 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "Falha ao salvar exportação", "failedToSaveExportedVideo": "Falha ao salvar vídeo exportado", "failedToRevealInFolder": "Erro ao mostrar na pasta: {{error}}", - "previewCompositorUnavailable": "Pré-visualização indisponível neste computador" + "previewCompositorUnavailable": "Pré-visualização indisponível neste computador", + "wordEditFailed": "Não foi possível alterar essa palavra" }, "export": { "canceled": "Exportação cancelada", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index de22724be..5b7291b64 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Transcrição atual", - "help": "Transcrição agregada de todos os clipes da linha do tempo. Backspace / Delete em uma palavra ou seleção marca como ignorada (em vermelho). Passe o mouse sobre um trecho vermelho para restaurá-lo.", + "help": "Transcrição agregada de todos os clipes da linha do tempo. Backspace / Delete em uma palavra ou seleção a corta do vídeo (em vermelho). Clique duas vezes em uma palavra para corrigir o texto: as legendas acompanham, o vídeo não muda. Passe o mouse sobre uma palavra marcada para restaurá-la.", "noClips": "Nenhum clipe ainda", "noTranscript": "Nenhuma transcrição ainda", "whisperHint": "A transcrição usa o Whisper local — roda no seu computador, nenhum dado sai do dispositivo.", @@ -277,6 +277,10 @@ "restoreSilence": "Restaurar silêncio ({{duration}} s)", "trimSilence": "Cortar silêncio ({{duration}} s)", "restoreWord": "Restaurar \"{{word}}\"", + "editWord": "Editar \"{{word}}\"", + "correctedWord": "Corrigida — a transcrição dizia \"{{original}}\"", + "revertWord": "Restaurar \"{{original}}\"", + "blankedWord": "apagada", "noAudio": "Esta mídia não tem faixa de áudio" }, "captions": { diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 4abdc63f3..c10afdf23 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "Не удалось сохранить экспорт", "failedToSaveExportedVideo": "Не удалось сохранить экспортированное видео", "failedToRevealInFolder": "Ошибка при показе в папке: {{error}}", - "previewCompositorUnavailable": "Предпросмотр недоступен на этом компьютере" + "previewCompositorUnavailable": "Предпросмотр недоступен на этом компьютере", + "wordEditFailed": "Не удалось изменить это слово" }, "export": { "canceled": "Экспорт отменён", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index ca193c6d8..73da559dd 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Текущая расшифровка", - "help": "Сводная расшифровка всех клипов на таймлайне. Backspace / Delete по слову или выделению помечает его как пропущенное (красным). Наведите курсор на красный фрагмент, чтобы вернуть его.", + "help": "Сводная расшифровка всех клипов на таймлайне. Backspace / Delete по слову или выделению вырезает его из видео (красным). Двойной щелчок по слову исправляет его текст: субтитры следуют за ним, видео не меняется. Наведите курсор на отмеченное слово, чтобы вернуть его.", "noClips": "Клипов пока нет", "noTranscript": "Расшифровки пока нет", "whisperHint": "Расшифровка использует локальный Whisper — работает на вашем компьютере, данные не покидают устройство.", @@ -277,6 +277,10 @@ "restoreSilence": "Вернуть тишину ({{duration}} с)", "trimSilence": "Вырезать тишину ({{duration}} с)", "restoreWord": "Вернуть «{{word}}»", + "editWord": "Изменить «{{word}}»", + "correctedWord": "Исправлено — в расшифровке было «{{original}}»", + "revertWord": "Вернуть «{{original}}»", + "blankedWord": "очищено", "noAudio": "В этом медиафайле нет аудиодорожки" }, "captions": { diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index d4bb46a10..b39f81a2c 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -12,7 +12,8 @@ "failedToSaveExport": "Dışa aktarım kaydedilemedi", "failedToSaveExportedVideo": "Dışa aktarılan video kaydedilemedi", "failedToRevealInFolder": "Klasörde gösterme hatası: {{error}}", - "previewCompositorUnavailable": "Bu makinede önizleme kullanılamıyor" + "previewCompositorUnavailable": "Bu makinede önizleme kullanılamıyor", + "wordEditFailed": "Bu kelime değiştirilemedi" }, "export": { "canceled": "Dışa aktarım iptal edildi", diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json index 485666d42..68156849a 100644 --- a/src/i18n/locales/tr/settings.json +++ b/src/i18n/locales/tr/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Geçerli döküm", - "help": "Zaman çizelgesindeki tüm kliplerin birleşik dökümü. Bir kelimede veya seçimde Backspace / Delete tuşuna basmak onu atlanmış (kırmızı) olarak işaretler. Kırmızı bölümün üzerine gelerek geri alabilirsiniz.", + "help": "Zaman çizelgesindeki tüm kliplerin birleşik dökümü. Bir kelimede veya seçimde Backspace / Delete tuşu onu videodan keser (kırmızı). Bir kelimeye çift tıklayarak metnini düzeltebilirsiniz: altyazılar buna uyar, video değişmez. İşaretli bir kelimenin üzerine gelerek geri alabilirsiniz.", "noClips": "Henüz klip yok", "noTranscript": "Henüz döküm yok", "whisperHint": "Döküm yerel Whisper kullanır — bilgisayarınızda çalışır, hiçbir veri cihazdan çıkmaz.", @@ -277,6 +277,10 @@ "restoreSilence": "Sessizliği geri al ({{duration}} sn)", "trimSilence": "Sessizliği kırp ({{duration}} sn)", "restoreWord": "\"{{word}}\" kelimesini geri al", + "editWord": "\"{{word}}\" kelimesini düzenle", + "correctedWord": "Düzeltildi — dökümde \"{{original}}\" yazıyordu", + "revertWord": "\"{{original}}\" haline getir", + "blankedWord": "boşaltıldı", "noAudio": "Bu medyada ses parçası yok" }, "captions": { diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 6f55d77a5..56b4f6a10 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "Không thể lưu bản xuất", "failedToSaveExportedVideo": "Không thể lưu video đã xuất", "failedToRevealInFolder": "Lỗi khi hiển thị trong thư mục: {{error}}", - "previewCompositorUnavailable": "Không thể xem trước trên máy này" + "previewCompositorUnavailable": "Không thể xem trước trên máy này", + "wordEditFailed": "Không thể thay đổi từ này" }, "export": { "canceled": "Đã hủy xuất", diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json index a2af00397..a440a5cd4 100644 --- a/src/i18n/locales/vi/settings.json +++ b/src/i18n/locales/vi/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Bản chép lời hiện tại", - "help": "Bản chép lời gộp của mọi clip trên dòng thời gian. Nhấn Backspace / Delete trên một từ hoặc vùng chọn để đánh dấu là bỏ qua (màu đỏ). Di chuột lên đoạn màu đỏ để khôi phục.", + "help": "Bản chép lời gộp của mọi clip trên dòng thời gian. Nhấn Backspace / Delete trên một từ hoặc vùng chọn để cắt nó khỏi video (màu đỏ). Nhấp đúp vào một từ để sửa văn bản: phụ đề đi theo, video không đổi. Di chuột lên từ được đánh dấu để khôi phục.", "noClips": "Chưa có clip nào", "noTranscript": "Chưa có bản chép lời", "whisperHint": "Việc chép lời dùng Whisper cục bộ — chạy ngay trên máy của bạn, không dữ liệu nào rời khỏi thiết bị.", @@ -277,6 +277,10 @@ "restoreSilence": "Khôi phục khoảng lặng ({{duration}} giây)", "trimSilence": "Cắt khoảng lặng ({{duration}} giây)", "restoreWord": "Khôi phục \"{{word}}\"", + "editWord": "Sửa \"{{word}}\"", + "correctedWord": "Đã sửa — bản chép lời ghi \"{{original}}\"", + "revertWord": "Khôi phục \"{{original}}\"", + "blankedWord": "đã xoá", "noAudio": "Media này không có bản âm thanh" }, "captions": { diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index dcfeb282b..1d471b3c5 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "保存导出文件失败", "failedToSaveExportedVideo": "保存导出的视频失败", "failedToRevealInFolder": "在文件夹中显示时出错：{{error}}", - "previewCompositorUnavailable": "此设备无法使用预览" + "previewCompositorUnavailable": "此设备无法使用预览", + "wordEditFailed": "无法修改该词" }, "export": { "canceled": "导出已取消", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index bd13392a1..386df3c87 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "当前转录", - "help": "时间轴上所有片段的合并转录。对某个词或选区按退格 / Delete 会将其标记为跳过（红色）。将鼠标悬停在红色片段上可恢复。", + "help": "时间轴上所有片段的合并转录。对某个词或选区按退格 / Delete 会将其从画面中剪掉（红色）。双击某个词可修改文字：字幕随之更新，画面不变。将鼠标悬停在带标记的词上可还原。", "noClips": "暂无片段", "noTranscript": "暂无转录", "whisperHint": "转录使用本地 Whisper — 在您的电脑上运行，数据不会离开本设备。", @@ -277,6 +277,10 @@ "restoreSilence": "恢复静音（{{duration}} 秒）", "trimSilence": "修剪静音（{{duration}} 秒）", "restoreWord": "恢复“{{word}}”", + "editWord": "编辑“{{word}}”", + "correctedWord": "已更正 — 转录原文为“{{original}}”", + "revertWord": "还原为“{{original}}”", + "blankedWord": "已清空", "noAudio": "此媒体没有音频轨道" }, "captions": { diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index 4893e4caf..b772e47db 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -20,7 +20,8 @@ "failedToSaveExport": "儲存匯出檔案失敗", "failedToSaveExportedVideo": "儲存匯出的影片失敗", "failedToRevealInFolder": "在資料夾中顯示時出錯：{{error}}", - "previewCompositorUnavailable": "此裝置無法使用預覽" + "previewCompositorUnavailable": "此裝置無法使用預覽", + "wordEditFailed": "無法修改這個字" }, "export": { "canceled": "匯出已取消", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index f9328155d..2f5c29ac6 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -265,7 +265,7 @@ }, "transcript": { "title": "目前的逐字稿", - "help": "時間軸上所有片段的合併逐字稿。對某個字或選取範圍按 Backspace / Delete 會標記為略過（紅色）。將滑鼠移到紅色片段上即可還原。", + "help": "時間軸上所有片段的合併逐字稿。對某個字或選取範圍按 Backspace / Delete 會將其從影片中剪掉（紅色）。連按兩下某個字即可修改文字：字幕隨之更新，影片不變。將滑鼠移到有標記的字上即可還原。", "noClips": "尚無片段", "noTranscript": "尚無逐字稿", "whisperHint": "逐字稿使用本機 Whisper — 在您的電腦上執行，資料不會離開本裝置。", @@ -278,6 +278,10 @@ "restoreSilence": "還原靜音（{{duration}} 秒）", "trimSilence": "修剪靜音（{{duration}} 秒）", "restoreWord": "還原「{{word}}」", + "editWord": "編輯「{{word}}」", + "correctedWord": "已更正 — 逐字稿原本是「{{original}}」", + "revertWord": "還原為「{{original}}」", + "blankedWord": "已清空", "noAudio": "此媒體沒有音訊軌道" }, "captions": { diff --git a/src/lib/ai-edition/store/documentWriteAudit.test.ts b/src/lib/ai-edition/store/documentWriteAudit.test.ts index 6a76e8013..1d3e928f8 100644 --- a/src/lib/ai-edition/store/documentWriteAudit.test.ts +++ b/src/lib/ai-edition/store/documentWriteAudit.test.ts @@ -131,6 +131,9 @@ const DECLARED: WritePath[] = [ w("src/components/ai-edition/NewEditorShell.tsx", "handleRenameProject", "save", "gesture"), // Ctrl+S / File > Save. w("src/components/ai-edition/NewEditorShell.tsx", "handleSave", "save", "gesture"), + // A word rewritten in the transcript pane. A correction, not a cut: it writes + // `transcript.words[].text` and leaves the timeline alone. + w("src/components/ai-edition/NewEditorShell.tsx", "handleSetWordText", "save", "gesture"), // "Save" chosen on the way out of Ctrl+N and Ctrl+O. w("src/components/ai-edition/NewEditorShell.tsx", "onKey", "save", "gesture"), w("src/components/ai-edition/NewEditorShell.tsx", "onKey", "save", "gesture"), From 492bdbbbd6e12f680e11544afc2d6a083697b022 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 16:12:09 +0200 Subject: [PATCH 06/12] feat(editor): add words to the transcript that nobody said MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The third gesture on the word stream, and the one that had to get past a guard. Put the caret between two words, type, and a field opens beside the word you were on; Enter turns it into a real word, in amber, marked `source: "synth"`. It has no audio, so its own control deletes it rather than trimming — there is nothing for a trim to remove — and Backspace over a run of nothing but inserts does the same. The guard it got past did not work. React 18 builds `onBeforeInput` from the legacy `textInput` event, whose `TextEvent` has no `inputType`, so the block's handler threw `Cannot read properties of undefined (reading 'startsWith')` on every character typed, never reached its own `preventDefault`, and let the character land in the contentEditable — the exact desynchronisation between the DOM and `words` it was written to prevent. Confirmed in the browser before touching it. It now listens to the native `beforeinput`, where the event really is an `InputEvent`, ignoring what the two nested fields raise on their own. An inserted word takes the silence it is dropped into and nothing else: from the word it follows, up to what its text needs to be read at subtitle pace, and never past the word that comes next. Between two words that run into each other it has no duration at all and rides their caption line, which is where it reads correctly anyway. Dropped into a pause it shortens the `[silence]` pill by exactly what it took. That span is also the slot a synthesized voice will have to fit, when there is one. `wordsInRange` had to learn about a word with no span — an overlap test excludes a point at either edge of the range, which silently lost every word inserted at the very start of a clip — and `withSilenceGaps` now breaks the resulting tie on array order rather than leaving it to sort stability nobody had written down. A run also carries inserts forward now, not just corrections. They have no original text to recognise, so time places them: each goes back after whatever the new transcript ends last before it. `carryOverWordEdits` counts what it could not place, as it already did for corrections. Verified end to end in the browser preview: typing leaves the block's own text untouched, the word lands in `words`, in `segment.wordIds`, in the rebuilt segment text and in the legacy mirror, an insert into the silence shrinks the pill from 2.6s to 2.2s, and deleting it puts the pill back. --- src/components/ai-edition/NewEditorShell.tsx | 50 ++- src/components/ai-edition/RightPanes.tsx | 406 ++++++++++++++++-- .../ai-edition/TranscriptPane.gating.test.tsx | 2 + .../TranscriptPane.keyboardCut.test.tsx | 4 + .../TranscriptPane.sharedMedia.test.tsx | 2 + .../TranscriptPane.wordEdit.test.tsx | 2 + .../TranscriptPane.wordInsert.test.tsx | 242 +++++++++++ src/i18n/locales/ar/editor.json | 4 +- src/i18n/locales/ar/settings.json | 5 +- src/i18n/locales/en/editor.json | 4 +- src/i18n/locales/en/settings.json | 5 +- src/i18n/locales/es/editor.json | 4 +- src/i18n/locales/es/settings.json | 5 +- src/i18n/locales/fr/editor.json | 4 +- src/i18n/locales/fr/settings.json | 5 +- src/i18n/locales/it/editor.json | 4 +- src/i18n/locales/it/settings.json | 5 +- src/i18n/locales/ja-JP/editor.json | 4 +- src/i18n/locales/ja-JP/settings.json | 5 +- src/i18n/locales/ko-KR/editor.json | 4 +- src/i18n/locales/ko-KR/settings.json | 5 +- src/i18n/locales/pt-BR/editor.json | 4 +- src/i18n/locales/pt-BR/settings.json | 5 +- src/i18n/locales/ru/editor.json | 4 +- src/i18n/locales/ru/settings.json | 5 +- src/i18n/locales/tr/editor.json | 4 +- src/i18n/locales/tr/settings.json | 5 +- src/i18n/locales/vi/editor.json | 4 +- src/i18n/locales/vi/settings.json | 5 +- src/i18n/locales/zh-CN/editor.json | 4 +- src/i18n/locales/zh-CN/settings.json | 5 +- src/i18n/locales/zh-TW/editor.json | 4 +- src/i18n/locales/zh-TW/settings.json | 5 +- .../ai-edition/document/transcript.test.ts | 186 +++++++- src/lib/ai-edition/document/transcript.ts | 220 +++++++++- .../store/documentWriteAudit.test.ts | 4 + .../timeline/aggregated-transcript.ts | 26 +- 37 files changed, 1184 insertions(+), 77 deletions(-) create mode 100644 src/components/ai-edition/TranscriptPane.wordInsert.test.tsx diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index ddafae140..ee8f66b66 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -13,7 +13,12 @@ import { applyProbedDuration, replaceTimeline as replaceTimelineOp, } from "@/lib/ai-edition/document/timeline"; -import { setDocumentWordText } from "@/lib/ai-edition/document/transcript"; +import { + type InsertSide, + insertDocumentWord, + removeDocumentWords, + setDocumentWordText, +} from "@/lib/ai-edition/document/transcript"; import { isModalOpen } from "@/lib/ai-edition/modalGuard"; import { type AxcutClip, documentSchema } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; @@ -630,6 +635,47 @@ export function NewEditorShell() { [enqueueTimelineWrite, saveDocument, te], ); + // transcript-pane → a word nobody said. It takes the silence it is dropped into and no + // audio at all, so unlike a cut it changes nothing about the film; today it reaches the + // captions and stops there. + const handleInsertWord = useCallback( + (assetId: string, anchorWordId: string, side: InsertSide, text: string) => { + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc) return; + try { + await saveDocument(insertDocumentWord(doc, assetId, anchorWordId, side, text), { + history: true, + }); + } catch (err) { + toast.error(te("errors.wordInsertFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + + // Deleting inserted words. One save for the whole set, so a Backspace over several of + // them is one Ctrl+Z, and the document layer refuses anything that was actually spoken. + const handleRemoveWords = useCallback( + (assetId: string, wordIds: string[]) => { + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc || wordIds.length === 0) return; + try { + await saveDocument(removeDocumentWords(doc, assetId, wordIds), { history: true }); + } catch (err) { + toast.error(te("errors.wordRemoveFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + const handleSelectProject = useCallback( async (id: string) => { try { @@ -1178,6 +1224,8 @@ export function NewEditorShell() { onAddTrimRange: handleAddTrimRange, onRemoveTrimRange: handleRemoveTrimRange, onSetWordText: handleSetWordText, + onInsertWord: handleInsertWord, + onRemoveWords: handleRemoveWords, onTranscribe: handleTranscribe, canTranscribe: hasAsset, isTranscribing: transcriptGate.state === "pending", diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index d01e98753..d2bed7c31 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -21,7 +21,7 @@ import { import { type ChangeEvent, type CSSProperties, - type FormEvent, + Fragment, memo, type ClipboardEvent as ReactClipboardEvent, type KeyboardEvent as ReactKeyboardEvent, @@ -40,6 +40,7 @@ import GradientEditor, { type GradientEditorState } from "@/components/ui/gradie import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { collectNativeFormats } from "@/lib/ai-edition/document/outputFormat"; +import type { InsertSide } from "@/lib/ai-edition/document/transcript"; import type { AxcutAsset, AxcutClip, @@ -58,6 +59,7 @@ import { type ClipSection, type ClipWord, findCueWordId, + isInsertedWord, isSilenceWord, type TrimRun, } from "@/lib/ai-edition/timeline/aggregated-transcript"; @@ -706,6 +708,8 @@ export function TranscriptPane({ onAddTrimRange, onRemoveTrimRange, onSetWordText, + onInsertWord, + onRemoveWords, onTranscribe, canTranscribe, isTranscribing, @@ -728,6 +732,11 @@ export function TranscriptPane({ * `ClipWord.id`: the transcript belongs to the asset, so a correction lands on the * media and shows on every clip that plays it — which is the point. */ onSetWordText: (assetId: string, wordId: string, text: string) => void; + /** Add a word nobody said, beside the word the caret was resting on. Bare id, as above. */ + onInsertWord: (assetId: string, anchorWordId: string, side: InsertSide, text: string) => void; + /** Delete inserted words. Only ever called with `source: "synth"` ids — a transcribed + * word is cut with a trim, never deleted. */ + onRemoveWords: (assetId: string, wordIds: string[]) => void; onTranscribe: () => void; canTranscribe: boolean; isTranscribing: boolean; @@ -839,6 +848,8 @@ export function TranscriptPane({ onAddTrimRange={onAddTrimRange} onRemoveTrimRange={onRemoveTrimRange} onSetWordText={onSetWordText} + onInsertWord={onInsertWord} + onRemoveWords={onRemoveWords} /> ))} @@ -867,6 +878,8 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ onAddTrimRange, onRemoveTrimRange, onSetWordText, + onInsertWord, + onRemoveWords, }: { index: number; section: ClipSection; @@ -876,6 +889,8 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; onRemoveTrimRange: (trimId: string) => void; onSetWordText: (assetId: string, wordId: string, text: string) => void; + onInsertWord: (assetId: string, anchorWordId: string, side: InsertSide, text: string) => void; + onRemoveWords: (assetId: string, wordIds: string[]) => void; }) { const ts = useScopedT("settings"); const { clip, asset, words } = section; @@ -948,6 +963,17 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // Only skip words that are currently kept (don't double-skip). const keptRange = rangeWords.filter((w) => w.kept); if (keptRange.length === 0) return; + // An inserted word has no audio to cut, so Backspace deletes it outright. Only a + // range made entirely of inserts takes this path: mixed with spoken words the trim + // covers them anyway — they sit inside its span and read as cut, which is what the + // keystroke asked for. + if (keptRange.every((w) => isInsertedWord(w.word))) { + onRemoveWords( + clip.assetId, + keptRange.map((w) => w.word.id), + ); + return; + } pendingCaretWordIdRef.current = keptRange[0].id; const startSec = Math.min(...keptRange.map((w) => w.word.startSec)); const endSec = Math.max(...keptRange.map((w) => w.word.endSec)); @@ -958,7 +984,7 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ `Skip ${formatMs(startSec * 1000)}-${formatMs(endSec * 1000)} from ${clip.assetId}.`, ); }, - [busy, clip.assetId, trimTarget, onAddTrimRange], + [busy, clip.assetId, trimTarget, onAddTrimRange, onRemoveWords], ); const removeTrimRun = useCallback( @@ -1021,38 +1047,93 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ [cutNativeSelection], ); - const handleBeforeInput = useCallback( - (event: FormEvent) => { - const inputEvent = event.nativeEvent as InputEvent; - if (inputEvent.inputType.startsWith("delete")) { + // The word an insert will sit beside, and what has been typed into it so far. Held on + // the block rather than the word, because the field belongs BETWEEN two words: the id is + // only how it finds its place in the stream. + const [insertion, setInsertion] = useState<{ + clipWordId: string; + side: InsertSide; + draft: string; + } | null>(null); + const insertionAbandonedRef = useRef(false); + + const openInsertion = useCallback( + (seed: string) => { + if (busy || !seed.trim()) return; + const editor = editorRef.current; + const selection = globalThis.getSelection(); + if (!editor || !selection) return; + if (!editor.contains(selection.anchorNode)) return; + const caret = findInsertionAnchor(editor, selection.anchorNode, selection.anchorOffset); + if (!caret) return; + const anchor = resolveInsertionAnchor(words, caret.clipWordId, caret.side); + if (!anchor) return; + setInsertion({ ...anchor, draft: seed }); + }, + [busy, words], + ); + + const commitInsertion = useCallback(() => { + const pending = insertion; + setInsertion(null); + if (!pending) return; + const text = pending.draft.trim(); + if (!text) return; + const anchor = words.find((w) => w.id === pending.clipWordId); + if (!anchor) return; + onInsertWord(clip.assetId, anchor.word.id, pending.side, text); + }, [insertion, words, onInsertWord, clip.assetId]); + + // Attached to the DOM, not through React's `onBeforeInput`. + // + // React 18 does not build that synthetic event from the native `beforeinput`: it + // derives it from the legacy `textInput`, whose event object is a `TextEvent` and + // carries no `inputType` at all. So the guard that was supposed to keep typed text out + // of the projection threw `Cannot read properties of undefined (reading 'startsWith')` + // on every character, never reached its own `preventDefault`, and let the character + // land in the contentEditable — the exact desynchronisation between the DOM and `words` + // it was written to prevent. Verified in the browser before this was moved. + // + // The native event is a real `InputEvent`, its `inputType` is the thing both branches + // switch on, and preventing it actually stops the browser. + useEffect(() => { + const editor = editorRef.current; + if (!editor) return; + const onBeforeInput = (event: InputEvent) => { + // The word editor and the insertion field are ``s INSIDE this element, so + // their own typing bubbles here natively — React's `stopPropagation` only ever + // stopped the synthetic tree. Their text is theirs. + if (event.target instanceof HTMLInputElement) return; + if (event.inputType.startsWith("delete")) { event.preventDefault(); - cutNativeSelection( - inputEvent.inputType === "deleteContentForward" ? "forward" : "backward", - ); + cutNativeSelection(event.inputType === "deleteContentForward" ? "forward" : "backward"); return; } - // Inserts are blocked to keep the projection stable: every run of text - // here maps back to a `transcript.words` entry by id, and free text has - // no id to land on. Deletion is fine because it goes through - // `cutNativeSelection`, which resolves the selection to word ids first. - // - // This used to defer to `SourceTranscriptModal`, deleted with the v3 - // media pane — it never got past read-only, so it was never the answer - // it was cited as. Editing a word's TEXT therefore has no in-app path - // today. Adding one means a word-level mutation alongside - // `skipWordRange`, reached from here; lifting this guard on its own - // would only desynchronise the DOM from `words`. - if (inputEvent.inputType === "insertText" || inputEvent.inputType === "insertFromPaste") { + // Free text never lands in the block itself: every run of text here maps back to a + // `transcript.words` entry by id, and typed characters have no id. What they open + // instead is a field beside the word the caret was on, whose commit creates a real + // word to hold them. So the gesture is the document one — put the caret somewhere + // and type — without the DOM ever getting ahead of `words`. + if (event.inputType.startsWith("insert")) { event.preventDefault(); + openInsertion(event.data ?? ""); } + }; + editor.addEventListener("beforeinput", onBeforeInput); + return () => editor.removeEventListener("beforeinput", onBeforeInput); + }, [cutNativeSelection, openInsertion]); + + const handlePaste = useCallback( + (event: ReactClipboardEvent) => { + // Handled here rather than through `insertFromPaste`: preventing the paste stops + // that beforeinput from ever firing, and this is the only place the clipboard text + // is still readable. + event.preventDefault(); + openInsertion(event.clipboardData.getData("text/plain")); }, - [cutNativeSelection], + [openInsertion], ); - const handlePaste = useCallback((event: ReactClipboardEvent) => { - event.preventDefault(); - }, []); - const handlePointerUp = useCallback( (event: ReactPointerEvent) => { if (event.button !== 0) return; @@ -1192,7 +1273,6 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ spellCheck={false} aria-label={ts("transcript.editorAria", { filename })} aria-multiline="true" - onBeforeInput={handleBeforeInput} onKeyDown={handleKeyDown} onPaste={handlePaste} onPointerUp={handlePointerUp} @@ -1214,18 +1294,38 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // scrollbar that breaks the cue auto-scroll UX. }} > - {words.map((cw) => ( - - ))} + {words.map((cw) => { + const field = + insertion?.clipWordId === cw.id ? ( + setInsertion({ ...insertion, draft })} + onCommit={commitInsertion} + onCancel={() => { + insertionAbandonedRef.current = true; + setInsertion(null); + }} + abandonedRef={insertionAbandonedRef} + /> + ) : null; + return ( + + {insertion?.side === "before" ? field : null} + + {insertion?.side === "after" ? field : null} + + ); + })} )} @@ -1259,6 +1359,7 @@ const TranscriptWord = memo(function TranscriptWord({ onRestore, onAddTrimRange, onSetWordText, + onRemoveWords, }: { cw: ClipWord; isCue: boolean; @@ -1269,6 +1370,7 @@ const TranscriptWord = memo(function TranscriptWord({ onRestore: (run: TrimRun) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; onSetWordText: (assetId: string, wordId: string, text: string) => void; + onRemoveWords: (assetId: string, wordIds: string[]) => void; }) { const ts = useScopedT("settings"); const [hover, setHover] = useState(false); @@ -1296,6 +1398,12 @@ const TranscriptWord = memo(function TranscriptWord({ onSetWordText(target.assetId, cw.word.id, next); }, [draft, cw.word.text, cw.word.id, onSetWordText, target.assetId]); + const inserted = isInsertedWord(cw.word); + + const removeInserted = useCallback(() => { + onRemoveWords(target.assetId, [cw.word.id]); + }, [onRemoveWords, target.assetId, cw.word.id]); + const revert = useCallback(() => { if (original === undefined) return; // Writing the original back through the same path is what clears the provenance @@ -1414,7 +1522,6 @@ const TranscriptWord = memo(function TranscriptWord({ setDraft(null); } }} - onBeforeInput={(event) => event.stopPropagation()} onPaste={(event) => event.stopPropagation()} onPointerUp={(event) => event.stopPropagation()} style={{ @@ -1436,6 +1543,58 @@ const TranscriptWord = memo(function TranscriptWord({ ); } + // A word nobody said. Amber rather than the accent: this one is not a fix to what was + // heard, it is text with no sound underneath — the caveat is the point. Double-click + // rewrites it like any other word; the cross deletes it, because there is no audio for a + // trim to remove. + if (inserted) { + return ( + setHover(true)} + onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} + > + + {cw.word.text} + + {hover ? ( + + + ) : null}{" "} + + ); + } + // A word the user emptied. It still owns a span of the media, so it keeps a place in // the stream: rendered as its own (empty) text it would be a bare space — invisible, // impossible to click, and therefore impossible to undo. @@ -1568,6 +1727,27 @@ const TranscriptWord = memo(function TranscriptWord({ * bin on a cut word — same size, same place, the accent rather than the danger colour, * since reverting a correction restores something instead of removing it. */ function RevertWordButton({ label, onRevert }: { label: string; onRevert: () => void }) { + return ( + + + ); +} + +/** The one hover control shape the word stream uses, in whichever colour says what it does. + * `contentEditable={false}` keeps it out of the enclosing editable block, and the click is + * stopped so it never reaches the seek handler underneath. */ +function WordChipButton({ + label, + tone, + onPress, + children, +}: { + label: string; + tone: string; + onPress: () => void; + children: ReactNode; +}) { return ( ); } +/** + * The field a typed character opens between two words. It is not a word yet — nothing is + * written until it commits — so it carries no `data-word-id` and no place in `words`. + * + * Every event it raises is stopped at the field, for the same reason the word editor stops + * its own: the block around it reads Backspace as a cut and a click as a seek. + */ +function InsertionField({ + value, + label, + onChange, + onCommit, + onCancel, + abandonedRef, +}: { + value: string; + label: string; + onChange: (value: string) => void; + onCommit: () => void; + onCancel: () => void; + abandonedRef: { current: boolean }; +}) { + return ( + onChange(event.target.value)} + onBlur={() => { + if (abandonedRef.current) { + abandonedRef.current = false; + return; + } + onCommit(); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") { + event.preventDefault(); + onCommit(); + } else if (event.key === "Escape") { + event.preventDefault(); + onCancel(); + } + }} + onBeforeInput={(event) => event.stopPropagation()} + onPaste={(event) => event.stopPropagation()} + onPointerUp={(event) => event.stopPropagation()} + style={{ + display: "inline", + width: `${Math.max(value.length, 3) + 2}ch`, + margin: "0 3px 2px 0", + padding: "0 5px", + border: "1px solid var(--warn)", + borderRadius: 999, + background: "var(--warn-soft)", + color: "var(--fg)", + font: "inherit", + outline: "none", + }} + /> + ); +} + // ─── Caret / selection helpers ──────────────────────────────────── // Ponytail port of axcut's findCollapsedDeletionWordId. The non-collapsed // path uses findWordId directly (a range selection's endpoints already @@ -1693,6 +1940,79 @@ function findCollapsedDeletionWordId( return pool.find((wordNode) => isKept(wordNode.dataset.wordId ?? null))?.dataset.wordId ?? null; } +/** + * Where a typed character goes: beside the word the caret was resting on, never inside it. + * + * A caret in the middle of a word anchors AFTER that word rather than splitting it in two — + * a split would need two words where the transcript has one, and neither half would own the + * audio any more. At the very start of the block there is nothing to sit after, so the + * anchor is the first word and the new one lands before it. + */ +function findInsertionAnchor( + editor: HTMLElement, + node: Node | null, + offset: number, +): { clipWordId: string; side: InsertSide } | null { + const wordNodes = Array.from(editor.querySelectorAll("[data-word-id]")); + if (wordNodes.length === 0 || !node) return null; + + const direct = closestWordElement(node); + if (direct?.dataset.wordId) { + const atStart = node.nodeType === Node.TEXT_NODE && offset <= 0; + return { clipWordId: direct.dataset.wordId, side: atStart ? "before" : "after" }; + } + + // The caret is between the block's own children, and `offset` is a child index — the + // same shape `findCollapsedDeletionWordId` reads when it resolves a cut. Walk back for + // the word to sit after; if there is none, the caret is at the head of the stream and + // the new word goes before the first word ahead of it. + const childNodes = Array.from(node.childNodes); + for (const candidate of childNodes.slice(0, clampRangeOffset(node, offset)).reverse()) { + const wordId = findWordId(candidate) ?? findDescendantWordId(candidate); + if (wordId) return { clipWordId: wordId, side: "after" }; + } + for (const candidate of childNodes.slice(clampRangeOffset(node, offset))) { + const wordId = findWordId(candidate) ?? findDescendantWordId(candidate); + if (wordId) return { clipWordId: wordId, side: "before" }; + } + const first = wordNodes[0]; + return first?.dataset.wordId ? { clipWordId: first.dataset.wordId, side: "before" } : null; +} + +/** + * Pull the DOM's answer back onto a word the TRANSCRIPT has. + * + * `[silence]` pills carry a `data-word-id` like everything else in the stream, but they are + * pseudo-words `withSilenceGaps` invents per clip — there is nothing in `transcript.words` + * for a new word to be inserted next to. So the anchor walks off a silence to the nearest + * real word in the direction the caret was already facing, and only crosses to the other + * side when that direction runs out of stream. + */ +function resolveInsertionAnchor( + words: ClipWord[], + clipWordId: string, + side: InsertSide, +): { clipWordId: string; side: InsertSide } | null { + const from = words.findIndex((w) => w.id === clipWordId); + if (from < 0) return null; + const real = (index: number) => + index >= 0 && index < words.length && !isSilenceWord(words[index].word); + if (side === "after") { + for (let i = from; i >= 0; i--) if (real(i)) return { clipWordId: words[i].id, side: "after" }; + for (let i = 0; i < words.length; i++) { + if (real(i)) return { clipWordId: words[i].id, side: "before" }; + } + return null; + } + for (let i = from; i < words.length; i++) { + if (real(i)) return { clipWordId: words[i].id, side: "before" }; + } + for (let i = words.length - 1; i >= 0; i--) { + if (real(i)) return { clipWordId: words[i].id, side: "after" }; + } + return null; +} + function findDescendantWordId(node: Node): string | null { if (node instanceof HTMLElement && node.dataset.wordId) { return node.dataset.wordId; diff --git a/src/components/ai-edition/TranscriptPane.gating.test.tsx b/src/components/ai-edition/TranscriptPane.gating.test.tsx index 34d97dfd3..bb62d61eb 100644 --- a/src/components/ai-edition/TranscriptPane.gating.test.tsx +++ b/src/components/ai-edition/TranscriptPane.gating.test.tsx @@ -57,6 +57,8 @@ function renderPane( onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={overrides.isTranscribing ?? false} diff --git a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx index 520e4b81b..97ba78739 100644 --- a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx +++ b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx @@ -85,6 +85,8 @@ function renderPane( onAddTrimRange={onAddTrimRange} onRemoveTrimRange={vi.fn()} onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} @@ -229,6 +231,8 @@ describe("keyboard cut with the caret between words", () => { } onRemoveTrimRange={vi.fn()} onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx index c3ad72571..37a03b25a 100644 --- a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx +++ b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx @@ -79,6 +79,8 @@ function renderPane(onSeek: (sec: number) => void = vi.fn()) { onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx index 2514fb23f..3c8fd0b97 100644 --- a/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx +++ b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx @@ -65,6 +65,8 @@ function renderPane(words?: AxcutWord[], busyAssetIds: string[] = []) { onAddTrimRange={onAddTrimRange} onRemoveTrimRange={vi.fn()} onSetWordText={onSetWordText} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx new file mode 100644 index 000000000..7af0cc9db --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx @@ -0,0 +1,242 @@ +// @vitest-environment jsdom +// Typing a word into the transcript that nobody said. +// +// This is the third gesture on the one word stream, and the one that had to get past a +// guard: the block used to swallow every keystroke outright, because free text has no +// `transcript.words` entry to land on. It still never lands in the block — what a typed +// character opens is a field beside the word the caret was on, and only its commit makes a +// word. These tests hold that: the DOM never gets ahead of `words`, and Backspace inside +// the field types instead of cutting the clip out from under it. + +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutClip, AxcutTranscript, AxcutWord } from "@/lib/ai-edition/schema"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 3, + cameraTrack: null, +}; + +const CLIP: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 3, + timelineStartSec: 0, + timelineEndSec: 3, + wordRefs: [], + origin: "user", + reason: "", +}; + +// Contiguous, so no `[silence]` pill sits between them to shift the caret indices. +const WORDS: AxcutWord[] = [ + { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "Bonjour" }, + { id: "w2", segmentId: "s", startSec: 1, endSec: 2, text: "à" }, + { id: "w3", segmentId: "s", startSec: 2, endSec: 3, text: "tous" }, +]; + +function renderPane(words: AxcutWord[] = WORDS, busyAssetIds: string[] = []) { + const onInsertWord = vi.fn(); + const onRemoveWords = vi.fn(); + const onAddTrimRange = vi.fn(); + const transcript: AxcutTranscript = { + assetId: "asset_1", + language: "fr", + segments: [], + words, + }; + const view = render( + + + , + ); + const editor = view.container.querySelector('[role="textbox"]'); + if (!editor) throw new Error("transcript editor not rendered"); + const field = () => view.container.querySelector("input[data-word-inserter]"); + const wordEl = (id: string) => { + const el = view.container.querySelector(`[data-word-id="clip_1:${id}"]`); + if (!el) throw new Error(`word ${id} not rendered`); + return el; + }; + return { ...view, editor, field, wordEl, onInsertWord, onRemoveWords, onAddTrimRange }; +} + +/** Park the caret between words at editor level, the way `restoreCaretBeforeWord` does. */ +function caretBeforeWordAt(editor: HTMLElement, index: number) { + const range = document.createRange(); + range.setStart(editor, index); + range.collapse(true); + const selection = window.getSelection(); + selection?.removeAllRanges(); + selection?.addRange(range); +} + +/** + * A real native `beforeinput`, because that is what the block listens to. + * + * Not `fireEvent.beforeInput`: React 18 builds its `onBeforeInput` from the legacy + * `textInput` event, whose `TextEvent` has no `inputType` — which is exactly why the guard + * moved off React and onto the DOM. Driving the synthetic one here would test a path the + * browser never takes. + */ +function type(editor: HTMLElement, data: string) { + // Through `fireEvent` so the state the listener sets is flushed, but with an event + // built by hand — `fireEvent.beforeInput` does not exist here, and the point is to + // dispatch the real thing. + fireEvent( + editor, + new InputEvent("beforeinput", { + data, + inputType: "insertText", + bubbles: true, + cancelable: true, + }), + ); +} + +afterEach(() => { + cleanup(); + window.getSelection()?.removeAllRanges(); +}); + +describe("typing between two words", () => { + it("opens a field there instead of dropping the keystroke", () => { + const view = renderPane(); + expect(view.field()).toBeNull(); + caretBeforeWordAt(view.editor, 2); // between "à" and "tous" + type(view.editor, "v"); + expect(view.field()).toHaveValue("v"); + }); + + it("never writes the typed text into the block itself", () => { + // The whole reason inserts were blocked: a run of text with no word id behind it + // desynchronises the DOM from `words`. + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.editor.textContent).not.toContain("v "); + expect(view.onInsertWord).not.toHaveBeenCalled(); + }); + + it("commits on Enter, against the word the caret was after", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.change(field, { target: { value: "vraiment" } }); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w2", "after", "vraiment"); + }); + + it("anchors before the first word when the caret is at the very start", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 0); + type(view.editor, "E"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w1", "before", "E"); + }); + + it("abandons on Escape without writing anything", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Escape" }); + fireEvent.blur(field); + expect(view.onInsertWord).not.toHaveBeenCalled(); + expect(view.field()).toBeNull(); + }); + + it("commits on blur", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 1); + type(view.editor, "x"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.change(field, { target: { value: "donc" } }); + fireEvent.blur(field); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w1", "after", "donc"); + }); + + it("does not cut the media when Backspace is pressed inside the field", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Backspace" }); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); + + it("stays shut while this clip's transcript is being regenerated", () => { + const view = renderPane(WORDS, ["asset_1"]); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.field()).toBeNull(); + }); +}); + +describe("a word that was inserted", () => { + const INSERTED: AxcutWord[] = [ + WORDS[0], + { id: "synth_1", segmentId: "s", startSec: 1, endSec: 1, text: "vraiment", source: "synth" }, + WORDS[1], + WORDS[2], + ]; + + it("reads as its own thing, not as a transcribed word", () => { + const view = renderPane(INSERTED); + const el = view.wordEl("synth_1"); + expect(el).toHaveAttribute("data-inserted", "true"); + expect(el.textContent).toContain("vraiment"); + }); + + // There is no audio for a trim to remove, so the gesture that makes a spoken word go + // away cannot be the one that makes this go away. + it("is deleted outright by its own control", () => { + const view = renderPane(INSERTED); + fireEvent.mouseEnter(view.wordEl("synth_1")); + const remove = view.wordEl("synth_1").querySelector("button"); + if (!remove) throw new Error("no delete control"); + fireEvent.click(remove); + expect(view.onRemoveWords).toHaveBeenCalledWith("asset_1", ["synth_1"]); + }); + + it("is deleted, not trimmed, when Backspace lands on it alone", () => { + const view = renderPane(INSERTED); + caretBeforeWordAt(view.editor, 2); // right after the insert + fireEvent.keyDown(view.editor, { key: "Backspace" }); + expect(view.onRemoveWords).toHaveBeenCalledWith("asset_1", ["synth_1"]); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); +}); diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 93f340aaa..afafe6f4b 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "فشل حفظ الفيديو المُصدَّر", "failedToRevealInFolder": "خطأ في الكشف في المجلد: {{error}}", "previewCompositorUnavailable": "المعاينة غير متوفرة على هذا الجهاز", - "wordEditFailed": "تعذّر تغيير هذه الكلمة" + "wordEditFailed": "تعذّر تغيير هذه الكلمة", + "wordInsertFailed": "تعذّرت إضافة هذه الكلمة", + "wordRemoveFailed": "تعذّر حذف هذه الكلمة" }, "export": { "canceled": "تم إلغاء التصدير", diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json index b211f0a7d..21b37731d 100644 --- a/src/i18n/locales/ar/settings.json +++ b/src/i18n/locales/ar/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "النص الحالي", - "help": "نص مجمّع لكل المقاطع على المخطط الزمني. اضغط Backspace / Delete على كلمة أو تحديد لقصّها من الفيديو (بالأحمر). انقر نقرًا مزدوجًا على كلمة لتصحيح نصها: تتبعها الترجمات ولا يتغيّر الفيديو. مرّر المؤشر فوق كلمة معلَّمة لاستعادتها.", + "help": "نص مجمّع لكل المقاطع على المخطط الزمني. اضغط Backspace / Delete على كلمة أو تحديد لقصّها من الفيديو (بالأحمر). انقر نقرًا مزدوجًا على كلمة لتصحيح نصها. اكتب بين كلمتين لإضافة كلمة جديدة باللون الكهرماني: تصل إلى الترجمات ولا تمسّ الفيديو. مرّر المؤشر فوق كلمة معلَّمة للتراجع.", "noClips": "لا توجد مقاطع بعد", "noTranscript": "لا يوجد نص بعد", "whisperHint": "يستخدم التفريغ النصي Whisper محليًا — يعمل على حاسوبك، ولا تغادر أي بيانات جهازك.", @@ -281,6 +281,9 @@ "correctedWord": "مصحّحة — كان النص \"{{original}}\"", "revertWord": "استعادة \"{{original}}\"", "blankedWord": "مُفرَّغة", + "insertAria": "كلمة جديدة", + "insertedWord": "أضفتها بنفسك — لا صوت خلفها", + "removeInserted": "حذف \"{{word}}\"", "noAudio": "لا يحتوي هذا الملف على مسار صوتي" }, "captions": { diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index bb167861a..6128ad898 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "Failed to save exported video", "failedToRevealInFolder": "Error revealing in folder: {{error}}", "previewCompositorUnavailable": "Preview unavailable on this machine", - "wordEditFailed": "Could not change that word" + "wordEditFailed": "Could not change that word", + "wordInsertFailed": "Could not add that word", + "wordRemoveFailed": "Could not delete that word" }, "export": { "canceled": "Export canceled", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 06ef40395..0f79df528 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -270,7 +270,7 @@ }, "transcript": { "title": "Current transcription", - "help": "Aggregated transcript of every clip on the timeline. Backspace / Delete a word or selection to cut it out of the film (red). Double-click a word to correct its text — the captions follow, the film does not move. Hover a marked word to restore it.", + "help": "Aggregated transcript of every clip on the timeline. Backspace / Delete a word or selection to cut it out of the film (red). Double-click a word to correct its text. Type between two words to add one, in amber: it reaches the captions and leaves the film alone. Hover a marked word to undo it.", "noClips": "No clips yet", "noTranscript": "No transcript yet", "whisperHint": "Transcribe uses local Whisper — runs on your computer, no data leaves the device.", @@ -287,6 +287,9 @@ "correctedWord": "Corrected — the transcriber heard \"{{original}}\"", "revertWord": "Restore \"{{original}}\"", "blankedWord": "blanked", + "insertAria": "New word", + "insertedWord": "Added by you — no audio behind it", + "removeInserted": "Delete \"{{word}}\"", "noAudio": "This media has no audio track" }, "captions": { diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 891d3d16d..410802353 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -13,7 +13,9 @@ "failedToSaveExportedVideo": "Error al guardar el video exportado", "failedToRevealInFolder": "Error al mostrar en la carpeta: {{error}}", "previewCompositorUnavailable": "Vista previa no disponible en este equipo", - "wordEditFailed": "No se pudo cambiar esa palabra" + "wordEditFailed": "No se pudo cambiar esa palabra", + "wordInsertFailed": "No se pudo añadir esa palabra", + "wordRemoveFailed": "No se pudo eliminar esa palabra" }, "export": { "canceled": "Exportación cancelada", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index c47036224..ba4f052d1 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Transcripción actual", - "help": "Transcripción agregada de todos los clips de la línea de tiempo. Retroceso / Supr sobre una palabra o selección la corta del vídeo (en rojo). Haz doble clic en una palabra para corregir su texto: los subtítulos la siguen, el vídeo no cambia. Pasa el cursor sobre una palabra marcada para restaurarla.", + "help": "Transcripción agregada de todos los clips de la línea de tiempo. Retroceso / Supr sobre una palabra o selección la corta del vídeo (en rojo). Haz doble clic en una palabra para corregir su texto. Escribe entre dos palabras para añadir una, en ámbar: llega a los subtítulos y no toca el vídeo. Pasa el cursor sobre una palabra marcada para deshacer.", "noClips": "Aún no hay clips", "noTranscript": "Aún no hay transcripción", "whisperHint": "La transcripción usa Whisper local: se ejecuta en tu equipo y ningún dato sale del dispositivo.", @@ -281,6 +281,9 @@ "correctedWord": "Corregida: la transcripción decía «{{original}}»", "revertWord": "Restaurar «{{original}}»", "blankedWord": "vaciada", + "insertAria": "Palabra nueva", + "insertedWord": "Añadida por ti: no hay audio detrás", + "removeInserted": "Eliminar «{{word}}»", "noAudio": "Este medio no tiene pista de audio" }, "captions": { diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index f7a04a264..a2e134fe3 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -19,7 +19,9 @@ "failedToSaveExportedVideo": "Échec de l'enregistrement de la vidéo exportée", "failedToRevealInFolder": "Erreur lors de l'affichage dans le dossier : {{error}}", "previewCompositorUnavailable": "Aperçu indisponible sur cette machine", - "wordEditFailed": "Impossible de modifier ce mot" + "wordEditFailed": "Impossible de modifier ce mot", + "wordInsertFailed": "Impossible d'ajouter ce mot", + "wordRemoveFailed": "Impossible de supprimer ce mot" }, "export": { "canceled": "Export annulé", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index ddfb2bbdf..05f6bff42 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Transcription actuelle", - "help": "Transcription agrégée de tous les clips de la timeline. Retour arrière / Suppr sur un mot ou une sélection le coupe du film (en rouge). Double-cliquez sur un mot pour corriger son texte : les sous-titres suivent, le film ne bouge pas. Survolez un mot marqué pour le rétablir.", + "help": "Transcription agrégée de tous les clips de la timeline. Retour arrière / Suppr sur un mot ou une sélection le coupe du film (en rouge). Double-cliquez sur un mot pour corriger son texte. Tapez entre deux mots pour en ajouter un, en ambre : il va dans les sous-titres et ne touche pas au film. Survolez un mot marqué pour annuler.", "noClips": "Aucun clip pour l'instant", "noTranscript": "Aucune transcription pour l'instant", "whisperHint": "La transcription utilise Whisper en local — tout s'exécute sur votre ordinateur, aucune donnée ne quitte l'appareil.", @@ -281,6 +281,9 @@ "correctedWord": "Corrigé — la transcription disait « {{original}} »", "revertWord": "Rétablir « {{original}} »", "blankedWord": "vidé", + "insertAria": "Nouveau mot", + "insertedWord": "Ajouté par vous — aucun son derrière", + "removeInserted": "Supprimer « {{word}} »", "noAudio": "Ce média n'a pas de piste audio" }, "captions": { diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 1734c8c82..8e6498dee 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "Impossibile salvare il video esportato", "failedToRevealInFolder": "Errore durante la visualizzazione nella cartella: {{error}}", "previewCompositorUnavailable": "Anteprima non disponibile su questo computer", - "wordEditFailed": "Impossibile modificare questa parola" + "wordEditFailed": "Impossibile modificare questa parola", + "wordInsertFailed": "Impossibile aggiungere questa parola", + "wordRemoveFailed": "Impossibile eliminare questa parola" }, "export": { "canceled": "Esportazione annullata", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index 565cf1864..1777770b4 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Trascrizione corrente", - "help": "Trascrizione aggregata di tutti i clip sulla timeline. Backspace / Canc su una parola o selezione la taglia dal video (in rosso). Fai doppio clic su una parola per correggerne il testo: i sottotitoli la seguono, il video non cambia. Passa sopra una parola contrassegnata per ripristinarla.", + "help": "Trascrizione aggregata di tutti i clip sulla timeline. Backspace / Canc su una parola o selezione la taglia dal video (in rosso). Fai doppio clic su una parola per correggerne il testo. Scrivi tra due parole per aggiungerne una, in ambra: finisce nei sottotitoli e non tocca il video. Passa sopra una parola contrassegnata per annullare.", "noClips": "Ancora nessun clip", "noTranscript": "Ancora nessuna trascrizione", "whisperHint": "La trascrizione usa Whisper in locale: gira sul tuo computer, nessun dato lascia il dispositivo.", @@ -281,6 +281,9 @@ "correctedWord": "Corretta — la trascrizione diceva «{{original}}»", "revertWord": "Ripristina «{{original}}»", "blankedWord": "svuotata", + "insertAria": "Nuova parola", + "insertedWord": "Aggiunta da te — nessun audio dietro", + "removeInserted": "Elimina «{{word}}»", "noAudio": "Questo contenuto non ha una traccia audio" }, "captions": { diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 2686aaff8..2238b2ef4 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -21,7 +21,9 @@ "failedToRevealInFolder": "フォルダの表示に失敗しました: {{error}}", "exportBackgroundLoadFailed": "エクスポートに失敗しました: 背景画像を読み込めませんでした ({{url}})", "previewCompositorUnavailable": "このマシンではプレビューを表示できません", - "wordEditFailed": "この単語を変更できませんでした" + "wordEditFailed": "この単語を変更できませんでした", + "wordInsertFailed": "この単語を追加できませんでした", + "wordRemoveFailed": "この単語を削除できませんでした" }, "export": { "canceled": "エクスポートがキャンセルされました", diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json index 96807f17f..201bc054c 100644 --- a/src/i18n/locales/ja-JP/settings.json +++ b/src/i18n/locales/ja-JP/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "現在の文字起こし", - "help": "タイムライン上のすべてのクリップをまとめた文字起こしです。単語や選択範囲を Backspace / Delete で映像から切り取れます（赤色）。単語をダブルクリックすると文字を修正できます。字幕は追従し、映像は変わりません。印の付いた単語にカーソルを合わせると元に戻せます。", + "help": "タイムライン上のすべてのクリップをまとめた文字起こしです。単語や選択範囲を Backspace / Delete で映像から切り取れます（赤色）。単語をダブルクリックすると文字を修正できます。単語と単語の間で入力すると、琥珀色の新しい単語を追加できます。字幕にだけ入り、映像は変わりません。印の付いた単語にカーソルを合わせると取り消せます。", "noClips": "クリップがまだありません", "noTranscript": "文字起こしがまだありません", "whisperHint": "文字起こしはローカルの Whisper を使用します。お使いのパソコン上で実行され、データが端末外に出ることはありません。", @@ -281,6 +281,9 @@ "correctedWord": "修正済み — 文字起こしでは「{{original}}」でした", "revertWord": "「{{original}}」に戻す", "blankedWord": "空欄", + "insertAria": "新しい単語", + "insertedWord": "あなたが追加した単語 — 音声はありません", + "removeInserted": "「{{word}}」を削除", "noAudio": "このメディアには音声トラックがありません" }, "captions": { diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 16400545c..5acb26773 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "내보낸 비디오 저장에 실패했습니다", "failedToRevealInFolder": "폴더에서 파일 표시 오류: {{error}}", "previewCompositorUnavailable": "이 컴퓨터에서는 미리보기를 사용할 수 없습니다", - "wordEditFailed": "이 단어를 변경할 수 없습니다" + "wordEditFailed": "이 단어를 변경할 수 없습니다", + "wordInsertFailed": "이 단어를 추가할 수 없습니다", + "wordRemoveFailed": "이 단어를 삭제할 수 없습니다" }, "export": { "canceled": "내보내기가 취소되었습니다", diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json index 9410e4dbe..f598c862f 100644 --- a/src/i18n/locales/ko-KR/settings.json +++ b/src/i18n/locales/ko-KR/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "현재 전사", - "help": "타임라인의 모든 클립을 합친 전사입니다. 단어나 선택 영역에서 Backspace / Delete를 누르면 영상에서 잘라냅니다(빨간색). 단어를 두 번 클릭하면 텍스트를 고칠 수 있습니다. 자막은 따라가고 영상은 그대로입니다. 표시된 단어에 마우스를 올리면 되돌릴 수 있습니다.", + "help": "타임라인의 모든 클립을 합친 전사입니다. 단어나 선택 영역에서 Backspace / Delete를 누르면 영상에서 잘라냅니다(빨간색). 단어를 두 번 클릭하면 텍스트를 고칠 수 있습니다. 두 단어 사이에 입력하면 호박색 단어가 추가됩니다. 자막에만 들어가고 영상은 그대로입니다. 표시된 단어에 마우스를 올리면 되돌릴 수 있습니다.", "noClips": "아직 클립이 없습니다", "noTranscript": "아직 전사가 없습니다", "whisperHint": "전사는 로컬 Whisper를 사용합니다. 이 컴퓨터에서 실행되며 데이터가 기기를 벗어나지 않습니다.", @@ -281,6 +281,9 @@ "correctedWord": "수정됨 — 전사에는 \"{{original}}\"였습니다", "revertWord": "\"{{original}}\"(으)로 되돌리기", "blankedWord": "비움", + "insertAria": "새 단어", + "insertedWord": "직접 추가한 단어 — 뒤에 오디오가 없습니다", + "removeInserted": "\"{{word}}\" 삭제", "noAudio": "이 미디어에는 오디오 트랙이 없습니다" }, "captions": { diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 9b0054861..1a4b9347c 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "Falha ao salvar vídeo exportado", "failedToRevealInFolder": "Erro ao mostrar na pasta: {{error}}", "previewCompositorUnavailable": "Pré-visualização indisponível neste computador", - "wordEditFailed": "Não foi possível alterar essa palavra" + "wordEditFailed": "Não foi possível alterar essa palavra", + "wordInsertFailed": "Não foi possível adicionar essa palavra", + "wordRemoveFailed": "Não foi possível excluir essa palavra" }, "export": { "canceled": "Exportação cancelada", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 5b7291b64..40f9a8e99 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Transcrição atual", - "help": "Transcrição agregada de todos os clipes da linha do tempo. Backspace / Delete em uma palavra ou seleção a corta do vídeo (em vermelho). Clique duas vezes em uma palavra para corrigir o texto: as legendas acompanham, o vídeo não muda. Passe o mouse sobre uma palavra marcada para restaurá-la.", + "help": "Transcrição agregada de todos os clipes da linha do tempo. Backspace / Delete em uma palavra ou seleção a corta do vídeo (em vermelho). Clique duas vezes em uma palavra para corrigir o texto. Digite entre duas palavras para adicionar uma, em âmbar: ela vai para as legendas e não mexe no vídeo. Passe o mouse sobre uma palavra marcada para desfazer.", "noClips": "Nenhum clipe ainda", "noTranscript": "Nenhuma transcrição ainda", "whisperHint": "A transcrição usa o Whisper local — roda no seu computador, nenhum dado sai do dispositivo.", @@ -281,6 +281,9 @@ "correctedWord": "Corrigida — a transcrição dizia \"{{original}}\"", "revertWord": "Restaurar \"{{original}}\"", "blankedWord": "apagada", + "insertAria": "Nova palavra", + "insertedWord": "Adicionada por você — sem áudio por trás", + "removeInserted": "Excluir \"{{word}}\"", "noAudio": "Esta mídia não tem faixa de áudio" }, "captions": { diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index c10afdf23..8b6aba527 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "Не удалось сохранить экспортированное видео", "failedToRevealInFolder": "Ошибка при показе в папке: {{error}}", "previewCompositorUnavailable": "Предпросмотр недоступен на этом компьютере", - "wordEditFailed": "Не удалось изменить это слово" + "wordEditFailed": "Не удалось изменить это слово", + "wordInsertFailed": "Не удалось добавить слово", + "wordRemoveFailed": "Не удалось удалить слово" }, "export": { "canceled": "Экспорт отменён", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 73da559dd..f39138626 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Текущая расшифровка", - "help": "Сводная расшифровка всех клипов на таймлайне. Backspace / Delete по слову или выделению вырезает его из видео (красным). Двойной щелчок по слову исправляет его текст: субтитры следуют за ним, видео не меняется. Наведите курсор на отмеченное слово, чтобы вернуть его.", + "help": "Сводная расшифровка всех клипов на таймлайне. Backspace / Delete по слову или выделению вырезает его из видео (красным). Двойной щелчок по слову исправляет его текст. Наберите текст между двумя словами, чтобы добавить своё, янтарным: оно попадёт в субтитры и не тронет видео. Наведите курсор на отмеченное слово, чтобы отменить.", "noClips": "Клипов пока нет", "noTranscript": "Расшифровки пока нет", "whisperHint": "Расшифровка использует локальный Whisper — работает на вашем компьютере, данные не покидают устройство.", @@ -281,6 +281,9 @@ "correctedWord": "Исправлено — в расшифровке было «{{original}}»", "revertWord": "Вернуть «{{original}}»", "blankedWord": "очищено", + "insertAria": "Новое слово", + "insertedWord": "Добавлено вами — за ним нет звука", + "removeInserted": "Удалить «{{word}}»", "noAudio": "В этом медиафайле нет аудиодорожки" }, "captions": { diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index b39f81a2c..a91e03fe0 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -13,7 +13,9 @@ "failedToSaveExportedVideo": "Dışa aktarılan video kaydedilemedi", "failedToRevealInFolder": "Klasörde gösterme hatası: {{error}}", "previewCompositorUnavailable": "Bu makinede önizleme kullanılamıyor", - "wordEditFailed": "Bu kelime değiştirilemedi" + "wordEditFailed": "Bu kelime değiştirilemedi", + "wordInsertFailed": "Bu kelime eklenemedi", + "wordRemoveFailed": "Bu kelime silinemedi" }, "export": { "canceled": "Dışa aktarım iptal edildi", diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json index 68156849a..031cdb7d0 100644 --- a/src/i18n/locales/tr/settings.json +++ b/src/i18n/locales/tr/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Geçerli döküm", - "help": "Zaman çizelgesindeki tüm kliplerin birleşik dökümü. Bir kelimede veya seçimde Backspace / Delete tuşu onu videodan keser (kırmızı). Bir kelimeye çift tıklayarak metnini düzeltebilirsiniz: altyazılar buna uyar, video değişmez. İşaretli bir kelimenin üzerine gelerek geri alabilirsiniz.", + "help": "Zaman çizelgesindeki tüm kliplerin birleşik dökümü. Bir kelimede veya seçimde Backspace / Delete tuşu onu videodan keser (kırmızı). Bir kelimeye çift tıklayarak metnini düzeltebilirsiniz. İki kelimenin arasına yazarak kehribar renginde yeni bir kelime ekleyebilirsiniz: altyazılara girer, videoya dokunmaz. İşaretli bir kelimenin üzerine gelerek geri alabilirsiniz.", "noClips": "Henüz klip yok", "noTranscript": "Henüz döküm yok", "whisperHint": "Döküm yerel Whisper kullanır — bilgisayarınızda çalışır, hiçbir veri cihazdan çıkmaz.", @@ -281,6 +281,9 @@ "correctedWord": "Düzeltildi — dökümde \"{{original}}\" yazıyordu", "revertWord": "\"{{original}}\" haline getir", "blankedWord": "boşaltıldı", + "insertAria": "Yeni kelime", + "insertedWord": "Sizin eklediğiniz — arkasında ses yok", + "removeInserted": "\"{{word}}\" kelimesini sil", "noAudio": "Bu medyada ses parçası yok" }, "captions": { diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 56b4f6a10..9d22463f3 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "Không thể lưu video đã xuất", "failedToRevealInFolder": "Lỗi khi hiển thị trong thư mục: {{error}}", "previewCompositorUnavailable": "Không thể xem trước trên máy này", - "wordEditFailed": "Không thể thay đổi từ này" + "wordEditFailed": "Không thể thay đổi từ này", + "wordInsertFailed": "Không thể thêm từ này", + "wordRemoveFailed": "Không thể xoá từ này" }, "export": { "canceled": "Đã hủy xuất", diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json index a440a5cd4..0b39eb91a 100644 --- a/src/i18n/locales/vi/settings.json +++ b/src/i18n/locales/vi/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "Bản chép lời hiện tại", - "help": "Bản chép lời gộp của mọi clip trên dòng thời gian. Nhấn Backspace / Delete trên một từ hoặc vùng chọn để cắt nó khỏi video (màu đỏ). Nhấp đúp vào một từ để sửa văn bản: phụ đề đi theo, video không đổi. Di chuột lên từ được đánh dấu để khôi phục.", + "help": "Bản chép lời gộp của mọi clip trên dòng thời gian. Nhấn Backspace / Delete trên một từ hoặc vùng chọn để cắt nó khỏi video (màu đỏ). Nhấp đúp vào một từ để sửa văn bản. Gõ giữa hai từ để thêm một từ mới, màu hổ phách: nó vào phụ đề và không đụng tới video. Di chuột lên từ được đánh dấu để hoàn tác.", "noClips": "Chưa có clip nào", "noTranscript": "Chưa có bản chép lời", "whisperHint": "Việc chép lời dùng Whisper cục bộ — chạy ngay trên máy của bạn, không dữ liệu nào rời khỏi thiết bị.", @@ -281,6 +281,9 @@ "correctedWord": "Đã sửa — bản chép lời ghi \"{{original}}\"", "revertWord": "Khôi phục \"{{original}}\"", "blankedWord": "đã xoá", + "insertAria": "Từ mới", + "insertedWord": "Bạn thêm vào — không có âm thanh phía sau", + "removeInserted": "Xoá \"{{word}}\"", "noAudio": "Media này không có bản âm thanh" }, "captions": { diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 1d471b3c5..0f1c4f7bc 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "保存导出的视频失败", "failedToRevealInFolder": "在文件夹中显示时出错：{{error}}", "previewCompositorUnavailable": "此设备无法使用预览", - "wordEditFailed": "无法修改该词" + "wordEditFailed": "无法修改该词", + "wordInsertFailed": "无法添加该词", + "wordRemoveFailed": "无法删除该词" }, "export": { "canceled": "导出已取消", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 386df3c87..4acef69fc 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -264,7 +264,7 @@ }, "transcript": { "title": "当前转录", - "help": "时间轴上所有片段的合并转录。对某个词或选区按退格 / Delete 会将其从画面中剪掉（红色）。双击某个词可修改文字：字幕随之更新，画面不变。将鼠标悬停在带标记的词上可还原。", + "help": "时间轴上所有片段的合并转录。对某个词或选区按退格 / Delete 会将其从画面中剪掉（红色）。双击某个词可修改文字。在两个词之间输入即可添加一个琥珀色的新词：它只进入字幕，不改动画面。将鼠标悬停在带标记的词上可撤销。", "noClips": "暂无片段", "noTranscript": "暂无转录", "whisperHint": "转录使用本地 Whisper — 在您的电脑上运行，数据不会离开本设备。", @@ -281,6 +281,9 @@ "correctedWord": "已更正 — 转录原文为“{{original}}”", "revertWord": "还原为“{{original}}”", "blankedWord": "已清空", + "insertAria": "新词", + "insertedWord": "你添加的词 — 背后没有声音", + "removeInserted": "删除“{{word}}”", "noAudio": "此媒体没有音频轨道" }, "captions": { diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index b772e47db..029b6ff12 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -21,7 +21,9 @@ "failedToSaveExportedVideo": "儲存匯出的影片失敗", "failedToRevealInFolder": "在資料夾中顯示時出錯：{{error}}", "previewCompositorUnavailable": "此裝置無法使用預覽", - "wordEditFailed": "無法修改這個字" + "wordEditFailed": "無法修改這個字", + "wordInsertFailed": "無法加入這個字詞", + "wordRemoveFailed": "無法刪除這個字詞" }, "export": { "canceled": "匯出已取消", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 2f5c29ac6..12894e080 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -265,7 +265,7 @@ }, "transcript": { "title": "目前的逐字稿", - "help": "時間軸上所有片段的合併逐字稿。對某個字或選取範圍按 Backspace / Delete 會將其從影片中剪掉（紅色）。連按兩下某個字即可修改文字：字幕隨之更新，影片不變。將滑鼠移到有標記的字上即可還原。", + "help": "時間軸上所有片段的合併逐字稿。對某個字或選取範圍按 Backspace / Delete 會將其從影片中剪掉（紅色）。連按兩下某個字即可修改文字。在兩個字之間輸入即可加入一個琥珀色的新字：它只進入字幕，不會動到影片。將滑鼠移到有標記的字上即可復原。", "noClips": "尚無片段", "noTranscript": "尚無逐字稿", "whisperHint": "逐字稿使用本機 Whisper — 在您的電腦上執行，資料不會離開本裝置。", @@ -282,6 +282,9 @@ "correctedWord": "已更正 — 逐字稿原本是「{{original}}」", "revertWord": "還原為「{{original}}」", "blankedWord": "已清空", + "insertAria": "新字詞", + "insertedWord": "你加入的字詞 — 背後沒有聲音", + "removeInserted": "刪除「{{word}}」", "noAudio": "此媒體沒有音訊軌道" }, "captions": { diff --git a/src/lib/ai-edition/document/transcript.test.ts b/src/lib/ai-edition/document/transcript.test.ts index 1bc1f5451..9e8ace067 100644 --- a/src/lib/ai-edition/document/transcript.test.ts +++ b/src/lib/ai-edition/document/transcript.test.ts @@ -1,6 +1,15 @@ import { describe, expect, it } from "vitest"; import { type AxcutTranscript, createEmptyDocument } from "../schema"; -import { carryOverWordEdits, setDocumentWordText, setWordText, withTranscript } from "./transcript"; +import { + carryOverWordEdits, + insertDocumentWord, + insertWord, + removeDocumentWords, + removeWord, + setDocumentWordText, + setWordText, + withTranscript, +} from "./transcript"; function fixture(language = "en"): AxcutTranscript { return { @@ -483,3 +492,178 @@ describe("carryOverWordEdits", () => { expect(carryOverWordEdits(null, next).transcript).toBe(next); }); }); + +// ─── Inserting a word nobody said ──────────────────────────────── +// The word carries no audio, so what it may occupy is the silence around it and nothing +// else. These pin that boundary: never over a spoken word, never a duration invented out +// of nothing when there is no pause to take. + +describe("insertWord", () => { + // "I"(1–2) "use"(2–3) "OpenScreen"(3–4), then a gap, then segment 2 at 5. + it("takes the silence after the word it follows, up to what its text needs", () => { + const result = insertWord(fixture(), "word_3", "after", "everywhere"); + const inserted = result.words.find((w) => w.source === "synth"); + expect(inserted?.startSec).toBe(4); + // 10 characters at 15/s = 0.67s, and the next word is a full second away. + expect(inserted?.endSec).toBeCloseTo(4 + 10 / 15, 5); + }); + + it("never runs over the word that comes next", () => { + // "use" ends at 3 and "OpenScreen" starts there: a long word gets no room at all. + const inserted = insertWord(fixture(), "word_2", "after", "a very long addition").words.find( + (w) => w.source === "synth", + ); + expect(inserted).toMatchObject({ startSec: 3, endSec: 3 }); + }); + + it("borrows backwards when it goes before the first word", () => { + const inserted = insertWord(fixture(), "word_1", "before", "Well").words.find( + (w) => w.source === "synth", + ); + // "word_1" starts at 1, and nothing precedes it — the floor is the media's own start. + expect(inserted?.endSec).toBe(1); + expect(inserted?.startSec).toBeCloseTo(1 - 0.4, 5); + }); + + it("marks it synthesized, with an id no transcription run can reuse", () => { + const inserted = insertWord(fixture(), "word_3", "after", "indeed").words.find( + (w) => w.source === "synth", + ); + expect(inserted).toMatchObject({ text: "indeed", source: "synth", segmentId: "segment_1" }); + expect(inserted?.id).toMatch(/^synth_\d+$/); + expect(inserted).not.toHaveProperty("originalText"); + }); + + it("numbers past the inserts already there", () => { + const once = insertWord(fixture(), "word_3", "after", "one"); + const twice = insertWord(once, "word_3", "after", "two"); + const ids = twice.words.filter((w) => w.source === "synth").map((w) => w.id); + expect(new Set(ids).size).toBe(2); + expect(ids).toContain("synth_2"); + }); + + it("lands in the segment's reading order, and rebuilds its text", () => { + const transcript = fixture(); + const result = insertWord(transcript, "word_2", "after", "really"); + const segment = result.segments.find((seg) => seg.id === "segment_1"); + expect(segment?.wordIds).toEqual(["word_1", "word_2", "synth_1", "word_3"]); + expect(segment?.text).toBe("I use really OpenScreen"); + // The segment the insert did not land in is carried over untouched, not rebuilt. + expect(result.segments[1]).toBe(transcript.segments[1]); + }); + + it("sits beside its anchor in the words array, which is what orders a zero-length insert", () => { + const result = insertWord(fixture(), "word_2", "after", "really"); + const ids = result.words.map((w) => w.id); + expect(ids.indexOf("synth_1")).toBe(ids.indexOf("word_2") + 1); + }); + + it("refuses empty text and unknown anchors", () => { + expect(() => insertWord(fixture(), "word_2", "after", " ")).toThrow(/empty/); + expect(() => insertWord(fixture(), "nope", "after", "x")).toThrow(/missing/); + }); + + it("keeps the input transcript untouched", () => { + const transcript = fixture(); + const before = JSON.stringify(transcript); + insertWord(transcript, "word_2", "after", "really"); + expect(JSON.stringify(transcript)).toBe(before); + }); +}); + +describe("removeWord", () => { + const withInsert = () => insertWord(fixture(), "word_2", "after", "really"); + + it("takes the word out of the array, the segment, and its text", () => { + const result = removeWord(withInsert(), "synth_1"); + expect(result.words.some((w) => w.id === "synth_1")).toBe(false); + const segment = result.segments.find((seg) => seg.id === "segment_1"); + expect(segment?.wordIds).toEqual(["word_1", "word_2", "word_3"]); + expect(segment?.text).toBe("I use OpenScreen"); + }); + + // Deleting a transcribed word would leave the film saying something the transcript + // denies. The operation for making a spoken word go away is a trim. + it("refuses a word that was actually spoken", () => { + expect(() => removeWord(fixture(), "word_2")).toThrow(/Refusing to remove transcribed word/); + }); + + it("refuses a word that is not there", () => { + expect(() => removeWord(fixture(), "nope")).toThrow(/missing/); + }); +}); + +describe("insertDocumentWord / removeDocumentWords", () => { + it("writes both the per-asset transcript and the legacy mirror", () => { + const result = insertDocumentWord(makeDoc(), "asset_1", "word_2", "after", "really"); + expect(result.transcript?.words.some((w) => w.id === "synth_1")).toBe(true); + expect(result.transcript).toBe(result.transcripts.find((t) => t.assetId === "asset_1")); + }); + + // One save for the whole set: a Backspace over three inserted words must be one Ctrl+Z. + it("removes several inserted words in a single document", () => { + let doc = insertDocumentWord(makeDoc(), "asset_1", "word_2", "after", "one"); + doc = insertDocumentWord(doc, "asset_1", "word_3", "after", "two"); + const result = removeDocumentWords(doc, "asset_1", ["synth_1", "synth_2"]); + expect(result.transcripts[0].words.some((w) => w.source === "synth")).toBe(false); + }); + + it("rejects an asset with no transcript", () => { + expect(() => insertDocumentWord(makeDoc(), "nope", "word_2", "after", "x")).toThrow( + /no transcript/, + ); + }); +}); + +describe("carryOverWordEdits with inserted words", () => { + const withInsert = () => insertWord(fixture(), "word_2", "after", "really"); + + it("puts an insert back after whatever the new run now ends last before it", () => { + // The insert sits at 3s. The new transcript says "I"(1–2) "used"(2–3) "it"(3.5–4). + const next = retranscribed([ + ["n1", "I", 1, 2], + ["n2", "used", 2, 3], + ["n3", "it", 3.5, 4], + ]); + const result = carryOverWordEdits(withInsert(), next); + expect(result).toMatchObject({ carried: 1, dropped: 0 }); + const ids = result.transcript.words.map((w) => w.id); + expect(ids.indexOf("synth_1")).toBe(ids.indexOf("n2") + 1); + expect(result.transcript.words.find((w) => w.id === "synth_1")).toMatchObject({ + text: "really", + source: "synth", + }); + }); + + it("puts it at the head when the new run has nothing before it", () => { + const carried = carryOverWordEdits( + insertWord(fixture(), "word_1", "before", "Well"), + retranscribed([["n1", "I", 1, 2]]), + ); + expect(carried.carried).toBe(1); + expect(carried.transcript.words[0].text).toBe("Well"); + }); + + it("counts an insert it could not place, rather than losing it quietly", () => { + const empty: AxcutTranscript = { assetId: "asset_1", language: "en", segments: [], words: [] }; + expect(carryOverWordEdits(withInsert(), empty)).toMatchObject({ carried: 0, dropped: 1 }); + }); + + it("carries corrections and inserts together", () => { + const both = insertWord( + setWordText(fixture(), "word_3", "OpenScreenApp"), + "word_2", + "after", + "really", + ); + const next = retranscribed([ + ["n1", "I", 1, 2], + ["n2", "use", 2, 3], + ["n3", "OpenScreen", 3, 4], + ]); + const result = carryOverWordEdits(both, next); + expect(result).toMatchObject({ carried: 2, dropped: 0 }); + expect(result.transcript.words.find((w) => w.id === "n3")?.text).toBe("OpenScreenApp"); + expect(result.transcript.words.some((w) => w.text === "really")).toBe(true); + }); +}); diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts index 6728575b1..dec291474 100644 --- a/src/lib/ai-edition/document/transcript.ts +++ b/src/lib/ai-edition/document/transcript.ts @@ -147,12 +147,200 @@ export function setDocumentWordText( return withTranscript(document, setWordText(transcript, wordId, text)); } +/** Where a new word goes relative to the word the caret was resting on. */ +export type InsertSide = "before" | "after"; + +/** + * How long an inserted word needs to be readable on screen. Subtitle practice is roughly + * fifteen characters a second, with a floor so a one-letter word is not a single frame. + * It is only ever a REQUEST — `insertWord` gives the word whatever silence is actually + * free, and no more. + */ +function readingSeconds(text: string): number { + return Math.max(0.4, text.trim().length / 15); +} + +/** `synth_N`, numbered past every id already in the transcript. + * + * The prefix buys uniqueness, not meaning: a transcription run regenerates `word_N` from + * 1, so a synthesized word holding one of those ids would be overwritten by the next run. + * What the word IS lives in `source`, which is what every reader checks. */ +function nextSynthWordId(transcript: AxcutTranscript): string { + let highest = 0; + for (const word of transcript.words) { + const match = /^synth_(\d+)$/.exec(word.id); + if (match) highest = Math.max(highest, Number(match[1])); + } + return `synth_${highest + 1}`; +} + +/** + * Insert a word that no one said. + * + * It carries no audio, so it takes the SILENCE it is dropped into and nothing else: from + * the word it follows up to what its text needs to be read, and never past the word that + * comes next. Dropped between two words that run straight into each other it has no + * duration at all and simply rides their caption line — which is where it reads correctly + * anyway, since there is no pause on screen to fill. + * + * That is the whole of what an inserted word can do today: it reaches the captions and + * stops there. When a voice can be synthesized for it, `source: "synth"` is what marks the + * words that need speaking, and the span computed here is the slot that audio has to fit. + */ +export function insertWord( + transcript: AxcutTranscript, + anchorWordId: string, + side: InsertSide, + text: string, +): AxcutTranscript { + const trimmed = text.trim(); + if (trimmed.length === 0) { + throw new Error("Cannot insert an empty word"); + } + const anchorIndex = transcript.words.findIndex((word) => word.id === anchorWordId); + if (anchorIndex < 0) { + throw new Error(`Cannot insert next to missing transcript word "${anchorWordId}"`); + } + const anchor = transcript.words[anchorIndex]; + const segment = transcript.segments.find((seg) => seg.id === anchor.segmentId); + if (!segment) { + throw new Error( + `Transcript word "${anchorWordId}" references missing segment "${anchor.segmentId}"`, + ); + } + const anchorSlot = segment.wordIds.indexOf(anchorWordId); + if (anchorSlot < 0) { + throw new Error(`Segment "${segment.id}" does not reference anchor word "${anchorWordId}"`); + } + + const wanted = readingSeconds(trimmed); + let startSec: number; + let endSec: number; + if (side === "after") { + startSec = anchor.endSec; + // The next word IN TIME, which is not necessarily the next one in the array — the + // array is insertion order, and only time decides what the new word may overlap. + const nextStart = transcript.words + .filter((word) => word.startSec >= startSec && word.id !== anchorWordId) + .reduce( + (soonest, word) => (soonest === null ? word.startSec : Math.min(soonest, word.startSec)), + null, + ); + endSec = nextStart === null ? startSec + wanted : Math.min(startSec + wanted, nextStart); + } else { + endSec = anchor.startSec; + const previousEnd = transcript.words + .filter((word) => word.endSec <= endSec && word.id !== anchorWordId) + .reduce( + (latest, word) => (latest === null ? word.endSec : Math.max(latest, word.endSec)), + null, + ); + const floor = previousEnd === null ? 0 : previousEnd; + startSec = Math.max(floor, endSec - wanted); + } + + const inserted: AxcutWord = { + id: nextSynthWordId(transcript), + segmentId: segment.id, + startSec, + endSec: Math.max(startSec, endSec), + text: trimmed, + source: "synth", + }; + + // Position in `words` matters as well as the timings: a zero-length insert shares its + // start with the word it sits against, and the reading order of that tie is the array + // order (see `withSilenceGaps`). + const at = side === "after" ? anchorIndex + 1 : anchorIndex; + const words = [...transcript.words.slice(0, at), inserted, ...transcript.words.slice(at)]; + const slot = side === "after" ? anchorSlot + 1 : anchorSlot; + const wordIds = [...segment.wordIds.slice(0, slot), inserted.id, ...segment.wordIds.slice(slot)]; + const byId = new Map(words.map((word) => [word.id, word])); + const segmentText = joinSegmentText(wordIds.map((id) => byId.get(id)?.text ?? "")); + + return { + ...transcript, + words, + segments: transcript.segments.map((seg) => + seg.id === segment.id ? { ...seg, wordIds, text: segmentText } : seg, + ), + }; +} + +/** + * Delete an inserted word. + * + * Only a synthesized one: a transcribed word is the label on a piece of audio, and the + * operation for making that go away is a trim, which removes the sound with it. Deleting + * the label alone would leave the film saying a word the transcript denies. + */ +export function removeWord(transcript: AxcutTranscript, wordId: string): AxcutTranscript { + const target = transcript.words.find((word) => word.id === wordId); + if (!target) { + throw new Error(`Cannot remove missing transcript word "${wordId}"`); + } + if (target.source !== "synth") { + throw new Error( + `Refusing to remove transcribed word "${wordId}": cut it with a trim, or blank its text`, + ); + } + const words = transcript.words.filter((word) => word.id !== wordId); + const byId = new Map(words.map((word) => [word.id, word])); + return { + ...transcript, + words, + segments: transcript.segments.map((segment) => { + if (!segment.wordIds.includes(wordId)) return segment; + const wordIds = segment.wordIds.filter((id) => id !== wordId); + return { + ...segment, + wordIds, + text: joinSegmentText(wordIds.map((id) => byId.get(id)?.text ?? "")), + }; + }), + }; +} + +/** {@link insertWord}, addressed by asset. Goes through `withTranscript` for the same + * reason {@link setDocumentWordText} does. */ +export function insertDocumentWord( + document: AxcutDocument, + assetId: string, + anchorWordId: string, + side: InsertSide, + text: string, +): AxcutDocument { + const transcript = document.transcripts.find((t) => t.assetId === assetId); + if (!transcript) { + throw new Error(`Cannot insert a word into asset "${assetId}": it has no transcript`); + } + return withTranscript(document, insertWord(transcript, anchorWordId, side, text)); +} + +/** {@link removeWord}, addressed by asset and taking the whole set at once: a Backspace + * over several inserted words has to be ONE write, or undoing it takes as many presses as + * there were words. */ +export function removeDocumentWords( + document: AxcutDocument, + assetId: string, + wordIds: readonly string[], +): AxcutDocument { + const transcript = document.transcripts.find((t) => t.assetId === assetId); + if (!transcript) { + throw new Error(`Cannot remove a word from asset "${assetId}": it has no transcript`); + } + return withTranscript( + document, + wordIds.reduce((acc, wordId) => removeWord(acc, wordId), transcript), + ); +} + /** What {@link carryOverWordEdits} managed to save from the previous transcript. */ export interface WordEditCarryOver { transcript: AxcutTranscript; - /** Corrections re-applied to the new transcript. */ + /** Corrections and insertions re-applied to the new transcript. */ carried: number; - /** Corrections the new transcript left no place for. These are lost. */ + /** Edits the new transcript left no place for. These are lost. */ dropped: number; } @@ -176,7 +364,12 @@ export function carryOverWordEdits( const edits = (previous?.words ?? []).filter( (word) => word.source === "user" && word.originalText !== undefined, ); - if (edits.length === 0) return { transcript: next, carried: 0, dropped: 0 }; + const inserts = (previous?.words ?? []) + .filter((word) => word.source === "synth") + .sort((a, b) => a.startSec - b.startSec); + if (edits.length === 0 && inserts.length === 0) { + return { transcript: next, carried: 0, dropped: 0 }; + } // Candidates are read from `next` throughout, never from the transcript being // built up: a word already rewritten by an earlier correction no longer carries @@ -198,5 +391,24 @@ export function carryOverWordEdits( transcript = setWordText(transcript, match.id, edit.text); carried += 1; } - return { transcript, carried, dropped: edits.length - carried }; + + // An inserted word has no original text to recognise, so time is what places it: the + // audio did not change between runs, only how it was heard. Each one goes back after + // whatever the new transcript now ends last before it — including a word re-inserted a + // moment ago, which is what keeps two inserts at the same spot in their old order. + for (const insert of inserts) { + const before = transcript.words + .filter((word) => word.endSec <= insert.startSec) + .reduce( + (latest, word) => (latest === null || word.endSec >= latest.endSec ? word : latest), + null, + ); + const head = transcript.words[0]; + const target = before ?? head ?? null; + if (!target) continue; + transcript = insertWord(transcript, target.id, before ? "after" : "before", insert.text); + carried += 1; + } + + return { transcript, carried, dropped: edits.length + inserts.length - carried }; } diff --git a/src/lib/ai-edition/store/documentWriteAudit.test.ts b/src/lib/ai-edition/store/documentWriteAudit.test.ts index 1d3e928f8..204b92ea9 100644 --- a/src/lib/ai-edition/store/documentWriteAudit.test.ts +++ b/src/lib/ai-edition/store/documentWriteAudit.test.ts @@ -131,6 +131,10 @@ const DECLARED: WritePath[] = [ w("src/components/ai-edition/NewEditorShell.tsx", "handleRenameProject", "save", "gesture"), // Ctrl+S / File > Save. w("src/components/ai-edition/NewEditorShell.tsx", "handleSave", "save", "gesture"), + // A word typed into the transcript pane, and the deletion of one. Both are the user's + // own edits to the transcript; neither touches the timeline. + w("src/components/ai-edition/NewEditorShell.tsx", "handleInsertWord", "save", "gesture"), + w("src/components/ai-edition/NewEditorShell.tsx", "handleRemoveWords", "save", "gesture"), // A word rewritten in the transcript pane. A correction, not a cut: it writes // `transcript.words[].text` and leaves the timeline alone. w("src/components/ai-edition/NewEditorShell.tsx", "handleSetWordText", "save", "gesture"), diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.ts b/src/lib/ai-edition/timeline/aggregated-transcript.ts index 1a35e9e61..0cd068056 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.ts @@ -25,6 +25,13 @@ export function isSilenceWord(word: AxcutWord): boolean { return word.id.startsWith("silence_"); } +/** True for a word the user typed in, which no one said and nothing in the media carries. + * Keyed on `source`, never on the id: the id shape is only there to stop a transcription + * run from reusing it. */ +export function isInsertedWord(word: AxcutWord): boolean { + return word.source === "synth"; +} + /** * Insert a synthetic `[silence]` pseudo-word into every gap of at least * `SILENCE_THRESHOLD_SEC` between consecutive words (and at the clip's @@ -38,7 +45,14 @@ function withSilenceGaps( clipStartSec: number, clipEndSec: number | undefined, ): AxcutWord[] { - const sorted = [...words].sort((a, b) => a.startSec - b.startSec); + // Sorted by time, ties broken by the order the transcript stores them in. The tie is + // not hypothetical: a word inserted between two contiguous words has no duration and + // therefore shares its start with the one it sits against, and only the array says + // which of the two the reader sees first. + const order = new Map(words.map((word, index) => [word.id, index])); + const sorted = [...words].sort( + (a, b) => a.startSec - b.startSec || (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0), + ); const result: AxcutWord[] = []; let cursor = clipStartSec; let n = 0; @@ -114,7 +128,15 @@ export interface ClipSection { } function wordsInRange(transcript: AxcutTranscript, startSec: number, endSec: number): AxcutWord[] { - return transcript.words.filter((w) => w.endSec > startSec && w.startSec < endSec); + return transcript.words.filter((w) => + // An inserted word dropped between two words that run into each other has NO + // duration, and an overlap test excludes a point at either edge of the range — + // which silently lost every word inserted at the very start of a clip. A word with + // no span is in the clip when its moment is. + w.endSec > w.startSec + ? w.endSec > startSec && w.startSec < endSec + : w.startSec >= startSec && w.startSec < endSec, + ); } /** Find the trim range covering this word's center (returns the deepest match). */ From 188f3650ef773bace1573cd2a7837b06698f45d6 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 22:19:19 +0200 Subject: [PATCH 07/12] feat(editor): an added word buys itself time, and the film holds its frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding a word only borrowed whatever silence happened to be free, so a word dropped between two words that run into each other got no time at all. It now creates the time it needs: the clip splits at the word's edge and a held-frame clip carries the deficit, so the timeline grows and everything downstream shifts. Screen and webcam freeze together — both are derived from the one asset source clock the freeze stops advancing — and the decoder is paused for the duration rather than free-running past the held frame into what comes after. That created span is the slot a synthesized voice will speak in. The gesture is gated to dev builds until there is a voice to put in it. A silent freeze frame is not a feature, and captions are not always on, so a release build offers no way to add a word at all — the pane does not even advertise it. Drop the gate in `openInsertion` when TTS lands. Three things the split broke, found by running it rather than by reading it: The captions went DARK over the pause. A line straddling the split was ventilated once per half, each half carrying the whole line, so the caption played, blinked out for exactly the pause the word exists for, then played again from the top. A line covering the held moment now covers the pause too, and spans that meet on the ruler coalesce — measured before and after: two cues of "Bonjour on va parler de vraiment Kubernetes" became one, 0→3.9s. The word rendered TWICE in the pane. The freeze claims it and the half that starts at the same moment matched it as well. The freeze owns it: it is the section the playhead is inside while the pause plays. And one word turned one recording into three headed blocks, each announcing the same filename over a sliver of timecode ("Clip 2 · 0:02.5—0:02.5"). Sections that continue the one before — same media, meeting on both clocks — now flow inline under a single header spanning the whole run. Two clips over one media still get a header each, which is the case the header exists for. The pane also says what its gestures are now: double-click corrects, Backspace cuts, and (in dev) typing between two words adds one. They were invisible until tried. Tests cover the freeze end to end: the document split, playback keeping the created time, the source clock held still through it, the pane showing the word once, the header run, and the caption playing once straight through. --- src/components/ai-edition/RightPanes.tsx | 253 +++++++++++------- .../TranscriptPane.sharedMedia.test.tsx | 82 ++++++ .../TranscriptPane.wordEdit.test.tsx | 9 + .../TranscriptPane.wordInsert.test.tsx | 17 ++ src/i18n/locales/ar/settings.json | 5 +- src/i18n/locales/en/settings.json | 5 +- src/i18n/locales/es/settings.json | 5 +- src/i18n/locales/fr/settings.json | 5 +- src/i18n/locales/it/settings.json | 5 +- src/i18n/locales/ja-JP/settings.json | 5 +- src/i18n/locales/ko-KR/settings.json | 5 +- src/i18n/locales/pt-BR/settings.json | 5 +- src/i18n/locales/ru/settings.json | 5 +- src/i18n/locales/tr/settings.json | 5 +- src/i18n/locales/vi/settings.json | 5 +- src/i18n/locales/zh-CN/settings.json | 5 +- src/i18n/locales/zh-TW/settings.json | 5 +- src/lib/ai-edition/captions/captions.test.ts | 115 ++++++++ src/lib/ai-edition/captions/cues.ts | 30 ++- src/lib/ai-edition/document/timeline.ts | 72 ++++- .../ai-edition/document/transcript.test.ts | 59 ++++ src/lib/ai-edition/document/transcript.ts | 34 ++- src/lib/ai-edition/schema/index.ts | 8 + .../timeline/aggregated-transcript.test.ts | 91 +++++++ .../timeline/aggregated-transcript.ts | 45 +++- .../ai-edition/timeline/timelineMap.test.ts | 61 +++++ src/lib/ai-edition/timeline/timelineMap.ts | 22 +- src/native/useNativePlaybackSync.ts | 28 +- 28 files changed, 878 insertions(+), 113 deletions(-) diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index d2bed7c31..e1f944986 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -782,13 +782,17 @@ export function TranscriptPane({ // engine, nothing attempted) leaves the button worth pressing. const silentMedia = blocked?.reason === "no-audio"; + // The insert gesture is dev-only until TTS (see openInsertion), so the copy follows + // the same gate: release builds must not advertise a dead gesture. + const helpText = + ts("transcript.help") + (import.meta.env.DEV ? ` ${ts("transcript.helpInsert")}` : ""); + const editingHint = ts( + import.meta.env.DEV ? "transcript.editingHintDev" : "transcript.editingHint", + ); + if (clips.length === 0 || !hasAnyTranscript) { return ( - } - helpText={ts("transcript.help")} - > + } helpText={helpText}>
-
-

{ts("transcript.title")}

-
-
- {sections.map((section, idx) => ( - - ))} -
-
+ } helpText={helpText}> + {/* The gestures are invisible until tried: nothing on a plain word stream says + * that double-click corrects and Backspace cuts. One muted line names them; the + * ? popover above carries the long version (amber inserts, hover-bin restore). */} +

+ {editingHint} +

+ {sections.map((section, idx) => ( + + ))} + ); } +/** + * Whether this section merely continues the previous one: same media, and the previous + * clip ends exactly where this one starts, on the source clock and on the ruler alike. + * + * Inserting a word SPLITS the clip it lands in — [before · freeze · after] — so one word + * turned one recording into three headed blocks, each announcing the same filename and a + * sliver of timecode. They are one continuous read and now render as one: the header + * appears on the first section of the run, the rest flow straight on from it. Two clips + * over the same media that are NOT contiguous still get a header each, which is the case + * the header exists for. + */ +function continuesPreviousSection( + previous: ClipSection | undefined, + section: ClipSection, +): boolean { + if (!previous || previous.clip.assetId !== section.clip.assetId) return false; + const EPSILON_SEC = 0.001; + const sourceMeets = + Math.abs((previous.clip.sourceEndSec ?? Number.NaN) - section.clip.sourceStartSec) < + EPSILON_SEC; + const rulerMeets = + Math.abs(previous.clip.timelineEndSec - section.clip.timelineStartSec) < EPSILON_SEC; + return sourceMeets && rulerMeets; +} + +/** The source range the whole run covers, for the one header that fronts it. */ +function runLabelFor(sections: ClipSection[], index: number): { start: number; end: number } { + let last = index; + while ( + last + 1 < sections.length && + continuesPreviousSection(sections[last], sections[last + 1]) + ) { + last += 1; + } + return { + start: sections[index].clip.sourceStartSec, + end: sections[last].clip.sourceEndSec ?? sections[last].clip.sourceStartSec, + }; +} + // One contentEditable block per clip — header (vignette + filename + // range) and a flowing word stream. The stream contains every transcript // word inside the clip's source range, color-coded by whether the word @@ -872,6 +926,8 @@ export function TranscriptPane({ const TranscriptClipBlock = memo(function TranscriptClipBlock({ index, section, + continuation, + runLabel, busy, cueWordId, onSeek, @@ -883,6 +939,10 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ }: { index: number; section: ClipSection; + /** This section reads straight on from the one above — no header, no gap. */ + continuation: boolean; + /** Source range of the whole contiguous run this section fronts. */ + runLabel: { start: number; end: number }; busy: boolean; cueWordId: string | null; onSeek: (sec: number) => void; @@ -901,10 +961,9 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ [clip.assetId, clip.id], ); const filename = asset?.label ?? clip.assetId; - const sourceRangeLabel = - clip.sourceEndSec !== undefined - ? `${formatMs(clip.sourceStartSec * 1000)}—${formatMs(clip.sourceEndSec * 1000)}` - : `${formatMs(clip.sourceStartSec * 1000)}—`; + // The run's range, not this clip's: a split clip's own sliver would read as a + // 0:02.5—0:02.5 recording. + const sourceRangeLabel = `${formatMs(runLabel.start * 1000)}—${formatMs(runLabel.end * 1000)}`; const editorRef = useRef(null); const pendingCaretWordIdRef = useRef(null); @@ -1059,6 +1118,11 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ const openInsertion = useCallback( (seed: string) => { + // ponytail: word insertion ships dev-only until a voice can be synthesized for + // the word — without one it only borrows free silence, and once it creates + // timeline time (the pause gesture) it is a silent freeze frame. Drop this gate + // when TTS lands. + if (!import.meta.env.DEV) return; if (busy || !seed.trim()) return; const editor = editorRef.current; const selection = globalThis.getSelection(); @@ -1176,79 +1240,85 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ return ( - + {continuation ? null : ( 0 ? 16 : 0, + marginBottom: 6, }} > - {index + 1} - - - - {filename} - - - {ts("transcript.clipLabel", { index: index + 1 })} · {sourceRangeLabel} - - - {/* A block whose transcript is being regenerated is read-only — say it, - rather than letting the word stream look live and drop the edits. */} - {busy ? ( - - {ts("transcript.transcribing")} + {index + 1} - ) : null} - + + + {filename} + + + {ts("transcript.clipLabel", { index: index + 1 })} · {sourceRangeLabel} + + + {/* A block whose transcript is being regenerated is read-only — say it, + rather than letting the word stream look live and drop the edits. */} + {busy ? ( + + + {ts("transcript.transcribing")} + + ) : null} + + )} {words.length === 0 ? (

{ ).toEqual(["clip_2:w2"]); }); }); + +// ─── Headers on a clip an inserted word split ──────────────────── +// Inserting a word splits the clip it lands in — [before · freeze · after] — so one word +// turned one recording into three blocks, each announcing the same filename and a sliver +// of timecode ("Clip 2 · 0:02.5—0:02.5"). They are one continuous read: one header, and +// the words flow straight on. The two-copies case above must keep its two headers, which +// is what tells the split apart from a media genuinely placed twice. + +const SPLIT_CLIPS: AxcutClip[] = [ + { + id: "clip_1_fzA", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 6, + timelineStartSec: 0, + timelineEndSec: 6, + wordRefs: [], + origin: "user", + reason: "", + }, + { + id: "clip_1_fz", + assetId: "asset_1", + sourceStartSec: 6, + sourceEndSec: 6, + timelineStartSec: 6, + timelineEndSec: 6.5, + wordRefs: [], + origin: "user", + reason: "Inserted word — held frame", + frozenSec: 0.5, + }, + { + id: "clip_1_fzB", + assetId: "asset_1", + sourceStartSec: 6, + sourceEndSec: 12, + timelineStartSec: 6.5, + timelineEndSec: 12.5, + wordRefs: [], + origin: "user", + reason: "", + }, +]; + +function renderClips(clips: AxcutClip[]) { + return render( + + + , + ); +} + +describe("clip headers", () => { + it("fronts a split clip with one header covering the whole run", () => { + const view = renderClips(SPLIT_CLIPS); + const headers = view.container.querySelectorAll("[data-clip-header]"); + expect(headers).toHaveLength(1); + // The run's range, not the first piece's — and not the freeze's 0:06.0—0:06.0. + expect(headers[0].textContent).toContain("0:00.0—0:12.0"); + }); + + it("still gives two headers to one media placed twice", () => { + const view = renderClips(CLIPS); + expect(view.container.querySelectorAll("[data-clip-header]")).toHaveLength(2); + }); +}); diff --git a/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx index 3c8fd0b97..74d2c021f 100644 --- a/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx +++ b/src/components/ai-edition/TranscriptPane.wordEdit.test.tsx @@ -84,6 +84,15 @@ function renderPane(words?: AxcutWord[], busyAssetIds: string[] = []) { afterEach(cleanup); +describe("telling the user the gestures exist", () => { + it("shows the editing hint line and the ? help when a transcript is on screen", () => { + // The gestures are invisible until tried — the pane must name them itself. + const view = renderPane(); + expect(view.getByText(/Double-click a word to correct it/)).toBeInTheDocument(); + expect(view.getByRole("button", { name: "Help" })).toBeInTheDocument(); + }); +}); + describe("correcting a word", () => { it("opens an editing field on the word a double-click lands on", () => { const view = renderPane(); diff --git a/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx index 7af0cc9db..caed55d61 100644 --- a/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx +++ b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx @@ -134,6 +134,23 @@ describe("typing between two words", () => { expect(view.field()).toHaveValue("v"); }); + it("stays inert outside dev builds — the gesture waits for TTS", () => { + // An inserted word with no voice only borrows free silence, so the gesture ships + // dev-only (see openInsertion). Release builds must drop the keystroke silently, + // the same way they did before the feature existed. + vi.stubEnv("DEV", false); + try { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.field()).toBeNull(); + expect(view.onInsertWord).not.toHaveBeenCalled(); + expect(view.editor.textContent).not.toContain("v "); + } finally { + vi.unstubAllEnvs(); + } + }); + it("never writes the typed text into the block itself", () => { // The whole reason inserts were blocked: a run of text with no word id behind it // desynchronises the DOM from `words`. diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json index 21b37731d..5520c0b50 100644 --- a/src/i18n/locales/ar/settings.json +++ b/src/i18n/locales/ar/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "النص الحالي", - "help": "نص مجمّع لكل المقاطع على المخطط الزمني. اضغط Backspace / Delete على كلمة أو تحديد لقصّها من الفيديو (بالأحمر). انقر نقرًا مزدوجًا على كلمة لتصحيح نصها. اكتب بين كلمتين لإضافة كلمة جديدة باللون الكهرماني: تصل إلى الترجمات ولا تمسّ الفيديو. مرّر المؤشر فوق كلمة معلَّمة للتراجع.", + "editingHint": "انقر نقرًا مزدوجًا على كلمة لتصحيحها، واضغط Backspace لقطعها من الفيلم.", + "editingHintDev": "انقر نقرًا مزدوجًا على كلمة لتصحيحها، واضغط Backspace لقطعها من الفيلم، واكتب بين كلمتين لإضافة كلمة.", + "help": "نص مجمّع لكل المقاطع على المخطط الزمني. اضغط Backspace / Delete على كلمة أو تحديد لقصّها من الفيديو (بالأحمر). انقر نقرًا مزدوجًا على كلمة لتصحيح نصها. مرّر المؤشر فوق كلمة معلَّمة للتراجع.", + "helpInsert": "اكتب بين كلمتين لإضافة كلمة جديدة باللون الكهرماني: تصل إلى الترجمات ولا تمسّ الفيديو.", "noClips": "لا توجد مقاطع بعد", "noTranscript": "لا يوجد نص بعد", "whisperHint": "يستخدم التفريغ النصي Whisper محليًا — يعمل على حاسوبك، ولا تغادر أي بيانات جهازك.", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 0f79df528..110edcdf4 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -270,7 +270,10 @@ }, "transcript": { "title": "Current transcription", - "help": "Aggregated transcript of every clip on the timeline. Backspace / Delete a word or selection to cut it out of the film (red). Double-click a word to correct its text. Type between two words to add one, in amber: it reaches the captions and leaves the film alone. Hover a marked word to undo it.", + "editingHint": "Double-click a word to correct it, Backspace cuts it from the film.", + "editingHintDev": "Double-click a word to correct it, Backspace cuts it from the film, type between two words to add one.", + "help": "Aggregated transcript of every clip on the timeline. Backspace / Delete a word or selection to cut it out of the film (red). Double-click a word to correct its text. Hover a marked word to undo it.", + "helpInsert": "Type between two words to add one, in amber: it reaches the captions and leaves the film alone.", "noClips": "No clips yet", "noTranscript": "No transcript yet", "whisperHint": "Transcribe uses local Whisper — runs on your computer, no data leaves the device.", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index ba4f052d1..595664612 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "Transcripción actual", - "help": "Transcripción agregada de todos los clips de la línea de tiempo. Retroceso / Supr sobre una palabra o selección la corta del vídeo (en rojo). Haz doble clic en una palabra para corregir su texto. Escribe entre dos palabras para añadir una, en ámbar: llega a los subtítulos y no toca el vídeo. Pasa el cursor sobre una palabra marcada para deshacer.", + "editingHint": "Haz doble clic en una palabra para corregirla, Retroceso la corta del vídeo.", + "editingHintDev": "Haz doble clic en una palabra para corregirla, Retroceso la corta del vídeo, escribe entre dos palabras para añadir una.", + "help": "Transcripción agregada de todos los clips de la línea de tiempo. Retroceso / Supr sobre una palabra o selección la corta del vídeo (en rojo). Haz doble clic en una palabra para corregir su texto. Pasa el cursor sobre una palabra marcada para deshacer.", + "helpInsert": "Escribe entre dos palabras para añadir una, en ámbar: llega a los subtítulos y no toca el vídeo.", "noClips": "Aún no hay clips", "noTranscript": "Aún no hay transcripción", "whisperHint": "La transcripción usa Whisper local: se ejecuta en tu equipo y ningún dato sale del dispositivo.", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 05f6bff42..196fcf231 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "Transcription actuelle", - "help": "Transcription agrégée de tous les clips de la timeline. Retour arrière / Suppr sur un mot ou une sélection le coupe du film (en rouge). Double-cliquez sur un mot pour corriger son texte. Tapez entre deux mots pour en ajouter un, en ambre : il va dans les sous-titres et ne touche pas au film. Survolez un mot marqué pour annuler.", + "editingHint": "Double-cliquez sur un mot pour le corriger, Retour arrière le coupe du film.", + "editingHintDev": "Double-cliquez sur un mot pour le corriger, Retour arrière le coupe du film, tapez entre deux mots pour en ajouter un.", + "help": "Transcription agrégée de tous les clips de la timeline. Retour arrière / Suppr sur un mot ou une sélection le coupe du film (en rouge). Double-cliquez sur un mot pour corriger son texte. Survolez un mot marqué pour annuler.", + "helpInsert": "Tapez entre deux mots pour en ajouter un, en ambre : il va dans les sous-titres et ne touche pas au film.", "noClips": "Aucun clip pour l'instant", "noTranscript": "Aucune transcription pour l'instant", "whisperHint": "La transcription utilise Whisper en local — tout s'exécute sur votre ordinateur, aucune donnée ne quitte l'appareil.", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index 1777770b4..3c7b2bdfe 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "Trascrizione corrente", - "help": "Trascrizione aggregata di tutti i clip sulla timeline. Backspace / Canc su una parola o selezione la taglia dal video (in rosso). Fai doppio clic su una parola per correggerne il testo. Scrivi tra due parole per aggiungerne una, in ambra: finisce nei sottotitoli e non tocca il video. Passa sopra una parola contrassegnata per annullare.", + "editingHint": "Fai doppio clic su una parola per correggerla, Backspace la taglia dal video.", + "editingHintDev": "Fai doppio clic su una parola per correggerla, Backspace la taglia dal video, scrivi tra due parole per aggiungerne una.", + "help": "Trascrizione aggregata di tutti i clip sulla timeline. Backspace / Canc su una parola o selezione la taglia dal video (in rosso). Fai doppio clic su una parola per correggerne il testo. Passa sopra una parola contrassegnata per annullare.", + "helpInsert": "Scrivi tra due parole per aggiungerne una, in ambra: finisce nei sottotitoli e non tocca il video.", "noClips": "Ancora nessun clip", "noTranscript": "Ancora nessuna trascrizione", "whisperHint": "La trascrizione usa Whisper in locale: gira sul tuo computer, nessun dato lascia il dispositivo.", diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json index 201bc054c..b7fbec518 100644 --- a/src/i18n/locales/ja-JP/settings.json +++ b/src/i18n/locales/ja-JP/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "現在の文字起こし", - "help": "タイムライン上のすべてのクリップをまとめた文字起こしです。単語や選択範囲を Backspace / Delete で映像から切り取れます（赤色）。単語をダブルクリックすると文字を修正できます。単語と単語の間で入力すると、琥珀色の新しい単語を追加できます。字幕にだけ入り、映像は変わりません。印の付いた単語にカーソルを合わせると取り消せます。", + "editingHint": "ダブルクリックで単語を修正、Backspace で映像からカット。", + "editingHintDev": "ダブルクリックで単語を修正、Backspace で映像からカット、2つの単語の間に入力して新しい単語を追加。", + "help": "タイムライン上のすべてのクリップをまとめた文字起こしです。単語や選択範囲を Backspace / Delete で映像から切り取れます（赤色）。単語をダブルクリックすると文字を修正できます。字幕にだけ入り、映像は変わりません。印の付いた単語にカーソルを合わせると取り消せます。", + "helpInsert": "単語と単語の間で入力すると、琥珀色の新しい単語を追加できます。", "noClips": "クリップがまだありません", "noTranscript": "文字起こしがまだありません", "whisperHint": "文字起こしはローカルの Whisper を使用します。お使いのパソコン上で実行され、データが端末外に出ることはありません。", diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json index f598c862f..1d36f0fff 100644 --- a/src/i18n/locales/ko-KR/settings.json +++ b/src/i18n/locales/ko-KR/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "현재 전사", - "help": "타임라인의 모든 클립을 합친 전사입니다. 단어나 선택 영역에서 Backspace / Delete를 누르면 영상에서 잘라냅니다(빨간색). 단어를 두 번 클릭하면 텍스트를 고칠 수 있습니다. 두 단어 사이에 입력하면 호박색 단어가 추가됩니다. 자막에만 들어가고 영상은 그대로입니다. 표시된 단어에 마우스를 올리면 되돌릴 수 있습니다.", + "editingHint": "단어를 더블 클릭해 수정하고, Backspace로 영상에서 잘라내세요.", + "editingHintDev": "단어를 더블 클릭해 수정하고, Backspace로 영상에서 잘라내고, 두 단어 사이에 입력해 새 단어를 추가하세요.", + "help": "타임라인의 모든 클립을 합친 전사입니다. 단어나 선택 영역에서 Backspace / Delete를 누르면 영상에서 잘라냅니다(빨간색). 단어를 두 번 클릭하면 텍스트를 고칠 수 있습니다. 자막에만 들어가고 영상은 그대로입니다. 표시된 단어에 마우스를 올리면 되돌릴 수 있습니다.", + "helpInsert": "두 단어 사이에 입력하면 호박색 단어가 추가됩니다.", "noClips": "아직 클립이 없습니다", "noTranscript": "아직 전사가 없습니다", "whisperHint": "전사는 로컬 Whisper를 사용합니다. 이 컴퓨터에서 실행되며 데이터가 기기를 벗어나지 않습니다.", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 40f9a8e99..00f91bcfc 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "Transcrição atual", - "help": "Transcrição agregada de todos os clipes da linha do tempo. Backspace / Delete em uma palavra ou seleção a corta do vídeo (em vermelho). Clique duas vezes em uma palavra para corrigir o texto. Digite entre duas palavras para adicionar uma, em âmbar: ela vai para as legendas e não mexe no vídeo. Passe o mouse sobre uma palavra marcada para desfazer.", + "editingHint": "Clique duas vezes em uma palavra para corrigi-la, Backspace a corta do vídeo.", + "editingHintDev": "Clique duas vezes em uma palavra para corrigi-la, Backspace a corta do vídeo, digite entre duas palavras para adicionar uma.", + "help": "Transcrição agregada de todos os clipes da linha do tempo. Backspace / Delete em uma palavra ou seleção a corta do vídeo (em vermelho). Clique duas vezes em uma palavra para corrigir o texto. Passe o mouse sobre uma palavra marcada para desfazer.", + "helpInsert": "Digite entre duas palavras para adicionar uma, em âmbar: ela vai para as legendas e não mexe no vídeo.", "noClips": "Nenhum clipe ainda", "noTranscript": "Nenhuma transcrição ainda", "whisperHint": "A transcrição usa o Whisper local — roda no seu computador, nenhum dado sai do dispositivo.", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index f39138626..204583c43 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "Текущая расшифровка", - "help": "Сводная расшифровка всех клипов на таймлайне. Backspace / Delete по слову или выделению вырезает его из видео (красным). Двойной щелчок по слову исправляет его текст. Наберите текст между двумя словами, чтобы добавить своё, янтарным: оно попадёт в субтитры и не тронет видео. Наведите курсор на отмеченное слово, чтобы отменить.", + "editingHint": "Дважды щёлкните слово, чтобы исправить его, Backspace вырезает его из фильма.", + "editingHintDev": "Дважды щёлкните слово, чтобы исправить его, Backspace вырезает его из фильма, напечатайте между двумя словами, чтобы добавить одно.", + "help": "Сводная расшифровка всех клипов на таймлайне. Backspace / Delete по слову или выделению вырезает его из видео (красным). Двойной щелчок по слову исправляет его текст. Наведите курсор на отмеченное слово, чтобы отменить.", + "helpInsert": "Наберите текст между двумя словами, чтобы добавить своё, янтарным: оно попадёт в субтитры и не тронет видео.", "noClips": "Клипов пока нет", "noTranscript": "Расшифровки пока нет", "whisperHint": "Расшифровка использует локальный Whisper — работает на вашем компьютере, данные не покидают устройство.", diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json index 031cdb7d0..6cf39685b 100644 --- a/src/i18n/locales/tr/settings.json +++ b/src/i18n/locales/tr/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "Geçerli döküm", - "help": "Zaman çizelgesindeki tüm kliplerin birleşik dökümü. Bir kelimede veya seçimde Backspace / Delete tuşu onu videodan keser (kırmızı). Bir kelimeye çift tıklayarak metnini düzeltebilirsiniz. İki kelimenin arasına yazarak kehribar renginde yeni bir kelime ekleyebilirsiniz: altyazılara girer, videoya dokunmaz. İşaretli bir kelimenin üzerine gelerek geri alabilirsiniz.", + "editingHint": "Bir kelimeye çift tıklayarak düzeltin, Backspace onu videodan keser.", + "editingHintDev": "Bir kelimeye çift tıklayarak düzeltin, Backspace onu videodan keser, iki kelimenin arasına yazarak yeni kelime ekleyin.", + "help": "Zaman çizelgesindeki tüm kliplerin birleşik dökümü. Bir kelimede veya seçimde Backspace / Delete tuşu onu videodan keser (kırmızı). Bir kelimeye çift tıklayarak metnini düzeltebilirsiniz. İşaretli bir kelimenin üzerine gelerek geri alabilirsiniz.", + "helpInsert": "İki kelimenin arasına yazarak kehribar renginde yeni bir kelime ekleyebilirsiniz: altyazılara girer, videoya dokunmaz.", "noClips": "Henüz klip yok", "noTranscript": "Henüz döküm yok", "whisperHint": "Döküm yerel Whisper kullanır — bilgisayarınızda çalışır, hiçbir veri cihazdan çıkmaz.", diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json index 0b39eb91a..6bcb6bf24 100644 --- a/src/i18n/locales/vi/settings.json +++ b/src/i18n/locales/vi/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "Bản chép lời hiện tại", - "help": "Bản chép lời gộp của mọi clip trên dòng thời gian. Nhấn Backspace / Delete trên một từ hoặc vùng chọn để cắt nó khỏi video (màu đỏ). Nhấp đúp vào một từ để sửa văn bản. Gõ giữa hai từ để thêm một từ mới, màu hổ phách: nó vào phụ đề và không đụng tới video. Di chuột lên từ được đánh dấu để hoàn tác.", + "editingHint": "Nhấp đúp vào một từ để sửa nó, Backspace cắt nó khỏi video.", + "editingHintDev": "Nhấp đúp vào một từ để sửa nó, Backspace cắt nó khỏi video, gõ giữa hai từ để thêm một từ mới.", + "help": "Bản chép lời gộp của mọi clip trên dòng thời gian. Nhấn Backspace / Delete trên một từ hoặc vùng chọn để cắt nó khỏi video (màu đỏ). Nhấp đúp vào một từ để sửa văn bản. Di chuột lên từ được đánh dấu để hoàn tác.", + "helpInsert": "Gõ giữa hai từ để thêm một từ mới, màu hổ phách: nó vào phụ đề và không đụng tới video.", "noClips": "Chưa có clip nào", "noTranscript": "Chưa có bản chép lời", "whisperHint": "Việc chép lời dùng Whisper cục bộ — chạy ngay trên máy của bạn, không dữ liệu nào rời khỏi thiết bị.", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 4acef69fc..8d7b78eb6 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -264,7 +264,10 @@ }, "transcript": { "title": "当前转录", - "help": "时间轴上所有片段的合并转录。对某个词或选区按退格 / Delete 会将其从画面中剪掉（红色）。双击某个词可修改文字。在两个词之间输入即可添加一个琥珀色的新词：它只进入字幕，不改动画面。将鼠标悬停在带标记的词上可撤销。", + "editingHint": "双击单词即可修正，Backspace 将其从影片中剪掉。", + "editingHintDev": "双击单词即可修正，Backspace 将其从影片中剪掉，在两个单词之间输入即可添加新词。", + "help": "时间轴上所有片段的合并转录。对某个词或选区按退格 / Delete 会将其从画面中剪掉（红色）。双击某个词可修改文字。将鼠标悬停在带标记的词上可撤销。", + "helpInsert": "在两个词之间输入即可添加一个琥珀色的新词：它只进入字幕，不改动画面。", "noClips": "暂无片段", "noTranscript": "暂无转录", "whisperHint": "转录使用本地 Whisper — 在您的电脑上运行，数据不会离开本设备。", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 12894e080..5dcd32110 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -265,7 +265,10 @@ }, "transcript": { "title": "目前的逐字稿", - "help": "時間軸上所有片段的合併逐字稿。對某個字或選取範圍按 Backspace / Delete 會將其從影片中剪掉（紅色）。連按兩下某個字即可修改文字。在兩個字之間輸入即可加入一個琥珀色的新字：它只進入字幕，不會動到影片。將滑鼠移到有標記的字上即可復原。", + "editingHint": "雙擊單字即可修正，Backspace 將其從影片中剪掉。", + "editingHintDev": "雙擊單字即可修正，Backspace 將其從影片中剪掉，在兩個單字之間輸入即可新增新詞。", + "help": "時間軸上所有片段的合併逐字稿。對某個字或選取範圍按 Backspace / Delete 會將其從影片中剪掉（紅色）。連按兩下某個字即可修改文字。將滑鼠移到有標記的字上即可復原。", + "helpInsert": "在兩個字之間輸入即可加入一個琥珀色的新字：它只進入字幕，不會動到影片。", "noClips": "尚無片段", "noTranscript": "尚無逐字稿", "whisperHint": "逐字稿使用本機 Whisper — 在您的電腦上執行，資料不會離開本裝置。", diff --git a/src/lib/ai-edition/captions/captions.test.ts b/src/lib/ai-edition/captions/captions.test.ts index ce3d4a961..4d86af1ef 100644 --- a/src/lib/ai-edition/captions/captions.test.ts +++ b/src/lib/ai-edition/captions/captions.test.ts @@ -650,3 +650,118 @@ describe("translated caption layout", () => { ); }); }); + +// ─── Captions across an inserted word's pause ──────────────────── +// Inserting a word SPLITS the clip it lands in — [before · freeze · after] — and a caption +// line straddling the split was ventilated once per half, each half carrying the whole +// line. On screen: the caption played, blinked out for the pause, then played again from +// the top — dark over the one moment the pause exists for. + +describe("a caption line over a freeze", () => { + /** `clip-1` split at 1.2s, with 0.5s of held frame carrying an inserted word. */ + function splitDoc(): AxcutDocument { + const withInsert = transcript(); + withInsert.segments[0] = { + ...withInsert.segments[0], + text: "hello there really friend", + wordIds: ["w1", "w2", "synth_1", "w3"], + }; + withInsert.words = [ + ...withInsert.words, + { + id: "synth_1", + segmentId: "seg_1", + startSec: 1.2, + endSec: 1.2, + text: "really", + source: "synth", + }, + ]; + return doc({ + transcripts: [withInsert], + timeline: { + ...doc().timeline, + clips: [ + { + id: "clip-1_fzA", + assetId: "asset-1", + sourceStartSec: 0, + sourceEndSec: 1.2, + timelineStartSec: 0, + timelineEndSec: 1.2, + wordRefs: [], + origin: "user", + reason: "", + }, + { + id: "clip-1_fz", + assetId: "asset-1", + sourceStartSec: 1.2, + sourceEndSec: 1.2, + timelineStartSec: 1.2, + timelineEndSec: 1.7, + wordRefs: [], + origin: "user", + reason: "Inserted word — held frame", + frozenSec: 0.5, + }, + { + id: "clip-1_fzB", + assetId: "asset-1", + sourceStartSec: 1.2, + sourceEndSec: 10, + timelineStartSec: 1.7, + timelineEndSec: 10.5, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + }, + }); + } + + it("plays the line once, straight through the pause", () => { + const cues = deriveCaptionCues(splitDoc(), ON, {}); + const first = cues.filter((cue) => cue.text.includes("really")); + expect(first).toHaveLength(1); + // It starts before the freeze and is still up after it — no dark stretch. + expect(first[0].startMs).toBeLessThan(1200); + expect(first[0].endMs).toBeGreaterThan(1700); + }); + + it("still plays a line twice when one media is genuinely placed twice", () => { + // The spans do not touch on the ruler there, so the coalescing must leave them apart. + const twice = doc({ + timeline: { + ...doc().timeline, + clips: [ + { + id: "clip-1", + assetId: "asset-1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }, + { + id: "clip-2", + assetId: "asset-1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 20, + timelineEndSec: 30, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + }, + }); + const cues = deriveCaptionCues(twice, ON, {}); + expect(cues.filter((cue) => cue.text.includes("hello"))).toHaveLength(2); + }); +}); diff --git a/src/lib/ai-edition/captions/cues.ts b/src/lib/ai-edition/captions/cues.ts index c3287ebcb..8154788a1 100644 --- a/src/lib/ai-edition/captions/cues.ts +++ b/src/lib/ai-edition/captions/cues.ts @@ -161,6 +161,17 @@ export function sourceSpanToTimelineSpans( const out: Array<{ startSec: number; endSec: number }> = []; for (const clip of clips) { if (clip.assetId !== assetId) continue; + // A FREEZE clip holds one source moment for created timeline time — an inserted + // word's pause. Its source window is that single point, so the overlap test below + // can never match it, and the line the pause exists for went DARK for its whole + // duration. A line covering the held moment covers the pause too. + if (clip.frozenSec !== undefined) { + const held = clip.sourceStartSec; + if (held >= startSec && held < endSec) { + out.push({ startSec: clip.timelineStartSec, endSec: clip.timelineEndSec }); + } + continue; + } const clipSourceEnd = clip.sourceEndSec ?? Number.POSITIVE_INFINITY; const s = Math.max(startSec, clip.sourceStartSec); const e = Math.min(endSec, clipSourceEnd); @@ -170,7 +181,24 @@ export function sourceSpanToTimelineSpans( endSec: clip.timelineStartSec + (e - clip.sourceStartSec), }); } - return out; + + // Coalesce what meets on the ruler. Splitting a clip to make room for an inserted word + // leaves the line straddling [before · freeze · after]: three spans back to back, each + // carrying the WHOLE line's text, so the caption played, blinked out over the pause, + // then played again from the top. They are one appearance. A line genuinely played + // twice — two clips over one media — does not touch on the ruler and stays two. + const EPSILON_SEC = 0.001; + const ordered = [...out].sort((a, b) => a.startSec - b.startSec); + const merged: Array<{ startSec: number; endSec: number }> = []; + for (const span of ordered) { + const last = merged[merged.length - 1]; + if (last && span.startSec <= last.endSec + EPSILON_SEC) { + last.endSec = Math.max(last.endSec, span.endSec); + continue; + } + merged.push({ ...span }); + } + return merged; } /** diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 232bcb6ee..fa41c2bc6 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -127,6 +127,73 @@ export function resequenceClips(clips: AxcutClip[]): AxcutClip[] { }); } +/** + * Split the clip covering `atSec` (source time, `assetId`'s media) into + * [before · FREEZE · after], where the freeze holds the frame at `atSec` for + * `frozenSec` of timeline time. The pause an inserted word creates: without it the + * word only borrows free silence, and a word dropped between two words that run into + * each other has no time at all. With it the timeline grows, everything downstream + * shifts, and a future TTS voice has a slot to speak in. + * + * No clip covers `atSec` (gap between clips, boundary of the asset) → unchanged, the + * caller decides whether that is acceptable. Trims anchored to the split clip stay on + * their id; `trimAppliesToClip` matches by asset when a trim has no clipId, and both + * halves keep the asset, so a trim that straddles the freeze point still narrows both + * halves exactly as it narrowed the whole clip before. + */ +export function insertFreezeInClips( + clips: AxcutClip[], + assetId: string, + atSec: number, + frozenSec: number, +): AxcutClip[] { + if (frozenSec <= 0) return clips; + const index = clips.findIndex( + (clip) => + clip.assetId === assetId && + (clip.frozenSec ?? 0) === 0 && + (clip.sourceEndSec ?? -1) > atSec + 0.001 && + atSec > clip.sourceStartSec, + ); + if (index < 0) return clips; + const clip = clips[index]; + // `resequenceClips` keeps each clip's OWN timeline length, so the halves must not + // inherit the un-split clip's — that would double the timeline. Lengths here are + // derived from the new source windows; resequence then only relays them. + const beforeLen = atSec - clip.sourceStartSec; + const afterLen = (clip.sourceEndSec ?? 0) - atSec; + const before: AxcutClip = { + ...clip, + id: `${clip.id}_fzA`, + sourceEndSec: atSec, + timelineEndSec: clip.timelineStartSec + beforeLen, + }; + const freeze: AxcutClip = { + id: `${clip.id}_fz`, + assetId: clip.assetId, + sourceStartSec: atSec, + sourceEndSec: atSec, + timelineStartSec: 0, + timelineEndSec: frozenSec, + wordRefs: [], + origin: "user", + reason: "Inserted word — held frame", + frozenSec, + }; + const after: AxcutClip | null = + afterLen > 0.001 + ? { + ...clip, + id: `${clip.id}_fzB`, + sourceStartSec: atSec, + timelineStartSec: 0, + timelineEndSec: afterLen, + } + : null; + const replaced = after ? [before, freeze, after] : [before, freeze]; + return resequenceClips([...clips.slice(0, index), ...replaced, ...clips.slice(index + 1)]); +} + export function subtractInterval(intervals: Interval[], cut: Interval): Interval[] { const output: Interval[] = []; for (const interval of intervals) { @@ -170,7 +237,10 @@ export function resolvePlaybackSegments( for (const clip of ordered) { const sourceEnd = clip.sourceEndSec ?? clip.sourceStartSec; if (sourceEnd <= clip.sourceStartSec) { - // Duration not probed yet — pass through as a single segment, unchanged. + // Either not probed yet, or a FREEZE clip (source window is the point it + // holds; `frozenSec` carries its real length). Both pass through as one + // segment at their timeline length — for a freeze that KEEPS the created + // pause in the compressed stream, which is the whole point. const dur = clip.timelineEndSec - clip.timelineStartSec; result.push({ ...clip, diff --git a/src/lib/ai-edition/document/transcript.test.ts b/src/lib/ai-edition/document/transcript.test.ts index 9e8ace067..03f9e5aae 100644 --- a/src/lib/ai-edition/document/transcript.test.ts +++ b/src/lib/ai-edition/document/transcript.test.ts @@ -615,6 +615,65 @@ describe("insertDocumentWord / removeDocumentWords", () => { }); }); +describe("insertDocumentWord freeze", () => { + /** One clip covering the whole fixture recording — what the editor starts from. */ + function makeDocWithClip() { + const doc = makeDoc(); + return { + ...doc, + timeline: { + ...doc.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user" as const, + reason: "", + }, + ], + }, + }; + } + + it("splits the clip and creates held-frame time when the silence is insufficient", () => { + // "really" after word_2: word_3 starts exactly where word_2 ends, so the word + // gets no silence at all and needs max(0.4, 6/15) = 0.4 s of created time. + const result = insertDocumentWord(makeDocWithClip(), "asset_1", "word_2", "after", "really"); + const clips = result.timeline.clips; + expect(clips).toHaveLength(3); + expect(clips[0]).toMatchObject({ id: "clip_1_fzA", sourceStartSec: 0, sourceEndSec: 3 }); + expect(clips[1]).toMatchObject({ + id: "clip_1_fz", + sourceStartSec: 3, + sourceEndSec: 3, + frozenSec: 0.4, + }); + expect(clips[2]).toMatchObject({ id: "clip_1_fzB", sourceStartSec: 3, sourceEndSec: 10 }); + // The timeline grew by exactly the freeze, laid back-to-back. + expect(clips[1].timelineStartSec).toBe(3); + expect(clips[1].timelineEndSec).toBeCloseTo(3.4, 5); + expect(clips[2].timelineEndSec).toBeCloseTo(10.4, 5); + }); + + it("does not touch the clips when free silence covers the word", () => { + // word_3 ends at 4, word_4 starts at 5: a full second of silence. + const result = insertDocumentWord(makeDocWithClip(), "asset_1", "word_3", "after", "really"); + expect(result.timeline.clips).toHaveLength(1); + expect(result.timeline.clips[0].id).toBe("clip_1"); + }); + + it("leaves the timeline alone when no clip covers the insertion point", () => { + // makeDoc has no clips at all — the word rides the caption line, as before. + const result = insertDocumentWord(makeDoc(), "asset_1", "word_2", "after", "really"); + expect(result.timeline.clips).toHaveLength(0); + }); +}); + describe("carryOverWordEdits with inserted words", () => { const withInsert = () => insertWord(fixture(), "word_2", "after", "really"); diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts index dec291474..ae626ee69 100644 --- a/src/lib/ai-edition/document/transcript.ts +++ b/src/lib/ai-edition/document/transcript.ts @@ -1,4 +1,5 @@ import type { AxcutDocument, AxcutTranscript, AxcutWord } from "../schema"; +import { insertFreezeInClips } from "./timeline"; const CJK_EDGE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u; const CLOSING_PUNCTUATION = /^[,.;:!?%。，、；：！？…）)\]}>》」』】〕]/u; @@ -302,7 +303,15 @@ export function removeWord(transcript: AxcutTranscript, wordId: string): AxcutTr } /** {@link insertWord}, addressed by asset. Goes through `withTranscript` for the same - * reason {@link setDocumentWordText} does. */ + * reason {@link setDocumentWordText} does. + * + * When the free silence the word can borrow is shorter than the text needs to be + * read, the deficit becomes a FREEZE on the timeline (`insertFreezeInClips`): the clip + * is split at the word's edge and a held-frame clip carries the missing time. The + * timeline grows, everything downstream shifts, and the word has a real slot a + * synthesized voice will later speak in. No document-layer gate on this: it is the + * correct document semantics for an inserted word — the product decision of whether + * the gesture exists at all lives in the transcript pane (`openInsertion`). */ export function insertDocumentWord( document: AxcutDocument, assetId: string, @@ -314,7 +323,28 @@ export function insertDocumentWord( if (!transcript) { throw new Error(`Cannot insert a word into asset "${assetId}": it has no transcript`); } - return withTranscript(document, insertWord(transcript, anchorWordId, side, text)); + const next = withTranscript(document, insertWord(transcript, anchorWordId, side, text)); + + // The word `insertWord` just added — the one synth id the old transcript lacked. + const inserted = next.transcripts + .find((t) => t.assetId === assetId) + ?.words.find( + (word) => word.source === "synth" && !transcript.words.some((w) => w.id === word.id), + ); + if (!inserted) return next; + const deficit = readingSeconds(inserted.text) - (inserted.endSec - inserted.startSec); + if (deficit <= 0.05) return next; + // The freeze continues the word's slot: after the word for "after" (silence, then + // held frame), before it for "before" (held frame, then silence) — one contiguous + // stretch of created time the future voice occupies. + const atSec = side === "after" ? inserted.endSec : inserted.startSec; + return { + ...next, + timeline: { + ...next.timeline, + clips: insertFreezeInClips(next.timeline.clips, assetId, atSec, deficit), + }, + }; } /** {@link removeWord}, addressed by asset and taking the whole set at once: a Backspace diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index 9a511fee5..155cddb2b 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -215,6 +215,14 @@ export const clipSchema = z // that as the identity region {x:0,y:0,width:1,height:1} rather than // storing the identity explicitly, so untouched clips stay lean. cropRegion: clipCropRegionSchema.optional(), + // A FREEZE clip: holds the frame at `sourceStartSec` (source window is the + // zero-width point [sourceStartSec, sourceStartSec]) for `frozenSec` of + // TIMELINE time. The pause an inserted word creates so a future TTS voice has + // a slot to speak in — screen and webcam freeze together because both tracks + // are derived from the same asset source clock. Absent on every ordinary clip; + // `resolvePlaybackSegments`'s un-probed passthrough branch must not be confused + // with it (a frozen clip is pushed through unchanged too, see its comment). + frozenSec: z.number().positive().optional(), }) .refine((data) => data.timelineEndSec >= data.timelineStartSec, { message: "timelineEndSec must be greater than or equal to timelineStartSec", diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.test.ts b/src/lib/ai-edition/timeline/aggregated-transcript.test.ts index adc70b6cb..60f1fdea6 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.test.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.test.ts @@ -474,3 +474,94 @@ describe("clipWordId", () => { expect(new Set(scopedIds).size).toBe(scopedIds.length); }); }); + +// ─── Freeze clips in the pane ──────────────────────────────────── +// An inserted word SPLITS the clip it lands in — [before · freeze · after] — and the word +// sits exactly on the split. Both the freeze and the half that starts there matched it, so +// the pane showed the same word twice, in two blocks. + +describe("the section a freeze clip projects", () => { + const TRANSCRIPT: AxcutTranscript = { + assetId: "a1", + language: "fr", + segments: [ + { id: "s1", kind: "speech", startSec: 0, endSec: 4, text: "un deux", wordIds: ["w1", "w2"] }, + ], + words: [ + { id: "w1", segmentId: "s1", startSec: 0, endSec: 2, text: "un" }, + { id: "w2", segmentId: "s1", startSec: 2, endSec: 4, text: "deux" }, + { id: "synth_1", segmentId: "s1", startSec: 2, endSec: 2, text: "vraiment", source: "synth" }, + ], + }; + const ASSET: AxcutAsset = { + id: "a1", + kind: "video", + label: "rec.mp4", + originalPath: "/r.mp4", + durationSec: 4, + cameraTrack: null, + }; + const CLIPS: AxcutClip[] = [ + { + id: "c_fzA", + assetId: "a1", + sourceStartSec: 0, + sourceEndSec: 2, + timelineStartSec: 0, + timelineEndSec: 2, + wordRefs: [], + origin: "user", + reason: "", + }, + { + id: "c_fz", + assetId: "a1", + sourceStartSec: 2, + sourceEndSec: 2, + timelineStartSec: 2, + timelineEndSec: 2.5, + wordRefs: [], + origin: "user", + reason: "", + frozenSec: 0.5, + }, + { + id: "c_fzB", + assetId: "a1", + sourceStartSec: 2, + sourceEndSec: 4, + timelineStartSec: 2.5, + timelineEndSec: 4.5, + wordRefs: [], + origin: "user", + reason: "", + }, + ]; + + it("shows the inserted word the freeze exists for", () => { + const sections = buildAggregatedSections(CLIPS, [TRANSCRIPT], [ASSET], []); + expect(sections[1].words.map((cw) => cw.word.text)).toEqual(["vraiment"]); + }); + + it("shows it exactly once across the whole pane", () => { + const sections = buildAggregatedSections(CLIPS, [TRANSCRIPT], [ASSET], []); + const everywhere = sections.flatMap((section) => + section.words.filter((cw) => cw.word.id === "synth_1"), + ); + expect(everywhere).toHaveLength(1); + }); + + it("leaves the spoken words where they were", () => { + const sections = buildAggregatedSections(CLIPS, [TRANSCRIPT], [ASSET], []); + expect(sections[0].words.map((cw) => cw.word.text)).toEqual(["un"]); + expect(sections[2].words.map((cw) => cw.word.text)).toEqual(["deux"]); + }); + + // The claim is scoped to freezes: with no freeze in the timeline, a word with no + // duration is shown by whichever clip its moment falls in, as before. + it("does not withhold an inserted word when no freeze claims it", () => { + const whole: AxcutClip[] = [{ ...CLIPS[0], id: "c1", sourceEndSec: 4, timelineEndSec: 4 }]; + const sections = buildAggregatedSections(whole, [TRANSCRIPT], [ASSET], []); + expect(sections[0].words.map((cw) => cw.word.text)).toContain("vraiment"); + }); +}); diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.ts b/src/lib/ai-edition/timeline/aggregated-transcript.ts index 0cd068056..de46d071e 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.ts @@ -160,6 +160,32 @@ export function buildClipSection( asset: AxcutAsset | null, trimRanges: AxcutTrimRange[], ): ClipSection { + // A FREEZE clip is the created time behind an inserted word: its source window is + // the single point it holds, so the ordinary range filter matches nothing — and it + // must not. The words it shows are the SYNTH words touching that point: the word + // the freeze exists for. While the playhead runs through the freeze the cue + // resolves against this section and lights the amber word through the whole pause. + if (clip.frozenSec !== undefined) { + const atSec = clip.sourceStartSec; + const words = transcript + ? transcript.words.filter( + (word) => word.source === "synth" && word.startSec <= atSec && word.endSec >= atSec, + ) + : []; + return { + clip, + asset, + transcript, + words: words.map((word) => ({ + id: clipWordId(clip.id, word.id), + word, + kept: true, + trimId: null, + })), + trimRuns: [], + }; + } + // `trimAppliesToClip` — not a bare `assetId` match — is what keeps a cut on the // second of two clips over the same media from also greying out the first one's // words. Same media, same source range: only the clip anchor tells them apart. @@ -244,7 +270,7 @@ export function buildAggregatedSections( ): ClipSection[] { const transcriptById = new Map(transcripts.map((t) => [t.assetId, t])); const assetById = new Map(assets.map((a) => [a.id, a])); - return clips.map((clip) => + const sections = clips.map((clip) => buildClipSection( clip, transcriptById.get(clip.assetId) ?? null, @@ -252,6 +278,23 @@ export function buildAggregatedSections( trimRanges, ), ); + + // A freeze clip claims the inserted word it was created for, and the clip after the + // split starts at the very moment that word sits on — so the word matched BOTH and the + // pane showed it twice, in two blocks. The freeze owns it: it is the section the + // playhead is inside while the pause plays, and the one whose whole reason to exist is + // that word. + const claimed = new Set( + sections + .filter((section) => section.clip.frozenSec !== undefined) + .flatMap((section) => section.words.map((cw) => cw.word.id)), + ); + if (claimed.size === 0) return sections; + return sections.map((section) => + section.clip.frozenSec !== undefined + ? section + : { ...section, words: section.words.filter((cw) => !claimed.has(cw.word.id)) }, + ); } /** Where the playback head currently is, in source time. */ diff --git a/src/lib/ai-edition/timeline/timelineMap.test.ts b/src/lib/ai-edition/timeline/timelineMap.test.ts index ea3f6c61b..ecc81a2d0 100644 --- a/src/lib/ai-edition/timeline/timelineMap.test.ts +++ b/src/lib/ai-edition/timeline/timelineMap.test.ts @@ -17,6 +17,7 @@ import { replacePillSpan, resolveNativePosition, resolvePillIds, + segmentRawSpanSec, } from "./timelineMap"; function clip(overrides: Partial & Pick): AxcutClip { @@ -805,3 +806,63 @@ describe("legacy groupId must never affect identity (regression: test 1)", () => expect(out[0]).not.toHaveProperty("groupId"); }); }); + +// ─── Freeze clips ──────────────────────────────────────────────── +// The pause an inserted word creates. Its source window is the single point it holds, so +// every reader that derives a length from `sourceEndSec - sourceStartSec` gets zero for it +// — and zero is the one answer that makes the playhead skip the pause entirely. + +describe("a freeze clip", () => { + const clips = [ + clip({ id: "a", assetId: "m", sourceStartSec: 0, sourceEndSec: 3 }), + clip({ + id: "fz", + assetId: "m", + sourceStartSec: 3, + sourceEndSec: 3, + timelineStartSec: 3, + timelineEndSec: 3.5, + frozenSec: 0.5, + }), + clip({ + id: "b", + assetId: "m", + sourceStartSec: 3, + sourceEndSec: 6, + timelineStartSec: 3.5, + timelineEndSec: 6.5, + }), + ]; + + it("keeps its created time in the playback segments", () => { + // It carries no source range to compress, so a naive reader drops it and the pause + // vanishes from playback while the ruler still counts it. + const segments = resolvePlaybackSegments(clips, []); + const freeze = segments.find((segment) => segment.id === "fz"); + expect(freeze).toBeDefined(); + expect((freeze?.timelineEndSec ?? 0) - (freeze?.timelineStartSec ?? 0)).toBeCloseTo(0.5, 5); + }); + + it("spans its frozen time on the raw ruler, not its (zero) source length", () => { + const span = segmentRawSpanSec(clips[1], clips); + expect(span.endSec - span.startSec).toBeCloseTo(0.5, 5); + }); + + it("holds the source clock still while the playhead runs through it", () => { + // The raw playhead DOES advance through the pause. Letting that delta reach the + // decoder would push it past the held frame into the content that belongs after. + const segments = resolvePlaybackSegments(clips, []); + for (const rawSec of [3.05, 3.25, 3.45]) { + const position = resolveNativePosition(rawSec, segments, clips); + expect(position?.clip.id).toBe("fz"); + expect(position?.sourceTimeSec).toBe(3); + } + }); + + it("hands the clip after the pause its own source moment again", () => { + const segments = resolvePlaybackSegments(clips, []); + const position = resolveNativePosition(4, segments, clips); + expect(position?.clip.id).toBe("b"); + expect(position?.sourceTimeSec).toBeCloseTo(3.5, 5); + }); +}); diff --git a/src/lib/ai-edition/timeline/timelineMap.ts b/src/lib/ai-edition/timeline/timelineMap.ts index b98601a03..081ede4d7 100644 --- a/src/lib/ai-edition/timeline/timelineMap.ts +++ b/src/lib/ai-edition/timeline/timelineMap.ts @@ -403,7 +403,13 @@ export function segmentRawSpanSec( rawClips: AxcutClip[], ): { startSec: number; endSec: number } { const startSec = getRawVirtualStartTime(segment, rawClips); - const lenSec = (segment.sourceEndSec ?? segment.sourceStartSec) - segment.sourceStartSec; + // A freeze clip's source window is the point it holds — its RAW span is its created + // timeline time, not the (zero) source length, or the playhead could never be + // "inside" it and would skip the pause entirely. + const lenSec = + segment.frozenSec !== undefined + ? segment.frozenSec + : (segment.sourceEndSec ?? segment.sourceStartSec) - segment.sourceStartSec; return { startSec, endSec: startSec + lenSec }; } @@ -690,6 +696,20 @@ export function resolveNativePosition( const seg = visibleSegments[index]; const segSourceEnd = seg.sourceEndSec ?? seg.sourceStartSec; + // Inside a freeze: the source clock does not advance. The whole point of the clip + // is created timeline time over one held frame — clamp the offset to zero rather + // than letting the raw-playhead delta (which DOES advance through the freeze) push + // the decoder past the held frame into content that belongs after the pause. + if (seg.frozenSec !== undefined) { + return { + clip: seg, + clipIndex: index, + sourceTimeSec: seg.sourceStartSec, + }; + } + // No `clampToSegmentStart` any more: this branch used to snap the playhead to the + // next kept segment, and main now returns `positionUnderCut` before reaching here + // (issue #216), so the flag could never be true. const unclamped = seg.sourceStartSec + (rawSec - spans[index].startSec); const maxSource = Math.max(seg.sourceStartSec, segSourceEnd - NATIVE_EOF_MARGIN_SEC); return { diff --git a/src/native/useNativePlaybackSync.ts b/src/native/useNativePlaybackSync.ts index 64e877b63..abe2f1c1a 100644 --- a/src/native/useNativePlaybackSync.ts +++ b/src/native/useNativePlaybackSync.ts @@ -41,6 +41,13 @@ export function useNativePlaybackSync( ); const activeClipId = activePosition?.clip.id ?? null; const sourceTimeSec = activePosition?.sourceTimeSec ?? null; + // A freeze clip holds ONE frame for `frozenSec` of app-clock time. Free-running the + // decoder through it would play the frames after the pause instead; the app clock + // (which does traverse the freeze) then re-seeks on drift and stutters. Pausing the + // decoder for the duration of the freeze is what makes the pause a pause — the + // webcam track freezes with the screen track because both derive from the same + // asset source clock the freeze stops advancing. + const frozen = activePosition?.clip.frozenSec !== undefined; // Reactive "is a native view active?" so activation mid-session re-pushes the // current transport/playhead (time & playing aren't memoised in the store). @@ -49,13 +56,15 @@ export function useNativePlaybackSync( () => getCurrentNativeViewId() !== null, ); - // Play/pause → native free-run. + // Play/pause → native free-run. Inside a freeze the native side is PAUSED however + // the transport is set — the app clock advances through the created time while the + // decoder holds the frame. useEffect(() => { if (!active) { return; } - setNativePlaying(playing); - }, [active, playing]); + setNativePlaying(playing && !frozen); + }, [active, playing, frozen]); // Scrub/step while paused OR periodic resync during playback when drift > 100ms const lastSyncedSourceTimeRef = useRef(null); @@ -68,6 +77,17 @@ export function useNativePlaybackSync( } const now = performance.now(); + // Inside a freeze while playing: the decoder is paused (see the transport + // effect) and parked on the held frame. Refresh the drift refs every run so the + // drift check never sees the (correctly) frozen source clock as divergence and + // fights itself with repeated seeks. + if (playing && frozen) { + setNativeTime(sourceTimeSec); + lastSyncedSourceTimeRef.current = sourceTimeSec; + lastSyncedWallTimeRef.current = now; + return; + } + // When clip changes, let setActiveClip handle the atomic clip-switch-and-seek. if (lastActiveClipIdRef.current !== activeClipId) { lastActiveClipIdRef.current = activeClipId; @@ -95,5 +115,5 @@ export function useNativePlaybackSync( lastSyncedSourceTimeRef.current = sourceTimeSec; lastSyncedWallTimeRef.current = now; } - }, [active, playing, activeClipId, sourceTimeSec]); + }, [active, playing, frozen, activeClipId, sourceTimeSec]); } From 2745da4d446248cfd71b81f2a42bedfa321dcf61 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 22:36:58 +0200 Subject: [PATCH 08/12] revert(editor): an added word no longer splits the clip it lands in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splitting the clip to make room for an inserted word did not survive the rest of the app. A project saved after two inserts came back with one clip named `clip_..._fzA_fzA` — split twice, both freezes and both after-halves gone, the full source range restored onto the mangled id — and zero synthesized words. The inserted text was lost on save, and the held-frame clips that did exist were skipped in playback. That is not a bug to chase. Clip surgery for a caption-only feature puts the timeline's shape under the transcript's control, and every other writer of `timeline.clips` — the duration probe, the recording import, resequencing — is entitled to disagree with it. `frozenSec` goes with it, and so do the readers that had to special-case a clip whose source window is a single point: the playback segments, the raw span, the native position clamp, the decoder pause, the caption ventilation, and the pane's split-clip header run. What stays is the part that was always true on its own: an inserted word is a word in the transcript with `source: "synth"`, it borrows whatever silence is free where it lands, and it reaches the captions. The gesture stays dev-gated — now for a second reason, since without created time an inserted word can only speak inside a pause that already exists. Creating time is still the right answer once there is a voice to put in it. It belongs in a record of its own, beside `trimRanges`, which is the inverse operation and the one shape the timeline already threads everywhere. --- src/components/ai-edition/RightPanes.tsx | 179 ++++++------------ .../TranscriptPane.sharedMedia.test.tsx | 82 -------- src/lib/ai-edition/captions/captions.test.ts | 115 ----------- src/lib/ai-edition/captions/cues.ts | 30 +-- src/lib/ai-edition/document/timeline.ts | 72 +------ .../ai-edition/document/transcript.test.ts | 59 ------ src/lib/ai-edition/document/transcript.ts | 34 +--- src/lib/ai-edition/schema/index.ts | 8 - .../timeline/aggregated-transcript.test.ts | 91 --------- .../timeline/aggregated-transcript.ts | 45 +---- .../ai-edition/timeline/timelineMap.test.ts | 61 ------ src/lib/ai-edition/timeline/timelineMap.ts | 22 +-- src/native/useNativePlaybackSync.ts | 28 +-- 13 files changed, 73 insertions(+), 753 deletions(-) diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index e1f944986..8df7af234 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -855,8 +855,6 @@ export function TranscriptPane({ key={section.clip.id} index={idx} section={section} - continuation={continuesPreviousSection(sections[idx - 1], section)} - runLabel={runLabelFor(sections, idx)} busy={busyAssetIds.includes(section.clip.assetId)} cueWordId={cueWordId} onSeek={onSeek} @@ -871,46 +869,6 @@ export function TranscriptPane({ ); } -/** - * Whether this section merely continues the previous one: same media, and the previous - * clip ends exactly where this one starts, on the source clock and on the ruler alike. - * - * Inserting a word SPLITS the clip it lands in — [before · freeze · after] — so one word - * turned one recording into three headed blocks, each announcing the same filename and a - * sliver of timecode. They are one continuous read and now render as one: the header - * appears on the first section of the run, the rest flow straight on from it. Two clips - * over the same media that are NOT contiguous still get a header each, which is the case - * the header exists for. - */ -function continuesPreviousSection( - previous: ClipSection | undefined, - section: ClipSection, -): boolean { - if (!previous || previous.clip.assetId !== section.clip.assetId) return false; - const EPSILON_SEC = 0.001; - const sourceMeets = - Math.abs((previous.clip.sourceEndSec ?? Number.NaN) - section.clip.sourceStartSec) < - EPSILON_SEC; - const rulerMeets = - Math.abs(previous.clip.timelineEndSec - section.clip.timelineStartSec) < EPSILON_SEC; - return sourceMeets && rulerMeets; -} - -/** The source range the whole run covers, for the one header that fronts it. */ -function runLabelFor(sections: ClipSection[], index: number): { start: number; end: number } { - let last = index; - while ( - last + 1 < sections.length && - continuesPreviousSection(sections[last], sections[last + 1]) - ) { - last += 1; - } - return { - start: sections[index].clip.sourceStartSec, - end: sections[last].clip.sourceEndSec ?? sections[last].clip.sourceStartSec, - }; -} - // One contentEditable block per clip — header (vignette + filename + // range) and a flowing word stream. The stream contains every transcript // word inside the clip's source range, color-coded by whether the word @@ -926,8 +884,6 @@ function runLabelFor(sections: ClipSection[], index: number): { start: number; e const TranscriptClipBlock = memo(function TranscriptClipBlock({ index, section, - continuation, - runLabel, busy, cueWordId, onSeek, @@ -939,10 +895,6 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ }: { index: number; section: ClipSection; - /** This section reads straight on from the one above — no header, no gap. */ - continuation: boolean; - /** Source range of the whole contiguous run this section fronts. */ - runLabel: { start: number; end: number }; busy: boolean; cueWordId: string | null; onSeek: (sec: number) => void; @@ -961,9 +913,10 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ [clip.assetId, clip.id], ); const filename = asset?.label ?? clip.assetId; - // The run's range, not this clip's: a split clip's own sliver would read as a - // 0:02.5—0:02.5 recording. - const sourceRangeLabel = `${formatMs(runLabel.start * 1000)}—${formatMs(runLabel.end * 1000)}`; + const sourceRangeLabel = + clip.sourceEndSec !== undefined + ? `${formatMs(clip.sourceStartSec * 1000)}—${formatMs(clip.sourceEndSec * 1000)}` + : `${formatMs(clip.sourceStartSec * 1000)}—`; const editorRef = useRef(null); const pendingCaretWordIdRef = useRef(null); @@ -1240,85 +1193,79 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ return ( - {continuation ? null : ( + 0 ? 16 : 0, - marginBottom: 6, + justifyContent: "center", + background: "var(--accent-soft)", + color: "var(--accent)", + borderRadius: "var(--r-sm)", + font: "700 12px/1 var(--font-mono)", + flexShrink: 0, }} > + {index + 1} + + + + {filename} + + + {ts("transcript.clipLabel", { index: index + 1 })} · {sourceRangeLabel} + + + {/* A block whose transcript is being regenerated is read-only — say it, + rather than letting the word stream look live and drop the edits. */} + {busy ? ( - {index + 1} - - - - {filename} - - - {ts("transcript.clipLabel", { index: index + 1 })} · {sourceRangeLabel} - + + {ts("transcript.transcribing")} - {/* A block whose transcript is being regenerated is read-only — say it, - rather than letting the word stream look live and drop the edits. */} - {busy ? ( - - - {ts("transcript.transcribing")} - - ) : null} - - )} + ) : null} + {words.length === 0 ? (

{ ).toEqual(["clip_2:w2"]); }); }); - -// ─── Headers on a clip an inserted word split ──────────────────── -// Inserting a word splits the clip it lands in — [before · freeze · after] — so one word -// turned one recording into three blocks, each announcing the same filename and a sliver -// of timecode ("Clip 2 · 0:02.5—0:02.5"). They are one continuous read: one header, and -// the words flow straight on. The two-copies case above must keep its two headers, which -// is what tells the split apart from a media genuinely placed twice. - -const SPLIT_CLIPS: AxcutClip[] = [ - { - id: "clip_1_fzA", - assetId: "asset_1", - sourceStartSec: 0, - sourceEndSec: 6, - timelineStartSec: 0, - timelineEndSec: 6, - wordRefs: [], - origin: "user", - reason: "", - }, - { - id: "clip_1_fz", - assetId: "asset_1", - sourceStartSec: 6, - sourceEndSec: 6, - timelineStartSec: 6, - timelineEndSec: 6.5, - wordRefs: [], - origin: "user", - reason: "Inserted word — held frame", - frozenSec: 0.5, - }, - { - id: "clip_1_fzB", - assetId: "asset_1", - sourceStartSec: 6, - sourceEndSec: 12, - timelineStartSec: 6.5, - timelineEndSec: 12.5, - wordRefs: [], - origin: "user", - reason: "", - }, -]; - -function renderClips(clips: AxcutClip[]) { - return render( - - - , - ); -} - -describe("clip headers", () => { - it("fronts a split clip with one header covering the whole run", () => { - const view = renderClips(SPLIT_CLIPS); - const headers = view.container.querySelectorAll("[data-clip-header]"); - expect(headers).toHaveLength(1); - // The run's range, not the first piece's — and not the freeze's 0:06.0—0:06.0. - expect(headers[0].textContent).toContain("0:00.0—0:12.0"); - }); - - it("still gives two headers to one media placed twice", () => { - const view = renderClips(CLIPS); - expect(view.container.querySelectorAll("[data-clip-header]")).toHaveLength(2); - }); -}); diff --git a/src/lib/ai-edition/captions/captions.test.ts b/src/lib/ai-edition/captions/captions.test.ts index 4d86af1ef..ce3d4a961 100644 --- a/src/lib/ai-edition/captions/captions.test.ts +++ b/src/lib/ai-edition/captions/captions.test.ts @@ -650,118 +650,3 @@ describe("translated caption layout", () => { ); }); }); - -// ─── Captions across an inserted word's pause ──────────────────── -// Inserting a word SPLITS the clip it lands in — [before · freeze · after] — and a caption -// line straddling the split was ventilated once per half, each half carrying the whole -// line. On screen: the caption played, blinked out for the pause, then played again from -// the top — dark over the one moment the pause exists for. - -describe("a caption line over a freeze", () => { - /** `clip-1` split at 1.2s, with 0.5s of held frame carrying an inserted word. */ - function splitDoc(): AxcutDocument { - const withInsert = transcript(); - withInsert.segments[0] = { - ...withInsert.segments[0], - text: "hello there really friend", - wordIds: ["w1", "w2", "synth_1", "w3"], - }; - withInsert.words = [ - ...withInsert.words, - { - id: "synth_1", - segmentId: "seg_1", - startSec: 1.2, - endSec: 1.2, - text: "really", - source: "synth", - }, - ]; - return doc({ - transcripts: [withInsert], - timeline: { - ...doc().timeline, - clips: [ - { - id: "clip-1_fzA", - assetId: "asset-1", - sourceStartSec: 0, - sourceEndSec: 1.2, - timelineStartSec: 0, - timelineEndSec: 1.2, - wordRefs: [], - origin: "user", - reason: "", - }, - { - id: "clip-1_fz", - assetId: "asset-1", - sourceStartSec: 1.2, - sourceEndSec: 1.2, - timelineStartSec: 1.2, - timelineEndSec: 1.7, - wordRefs: [], - origin: "user", - reason: "Inserted word — held frame", - frozenSec: 0.5, - }, - { - id: "clip-1_fzB", - assetId: "asset-1", - sourceStartSec: 1.2, - sourceEndSec: 10, - timelineStartSec: 1.7, - timelineEndSec: 10.5, - wordRefs: [], - origin: "user", - reason: "", - }, - ], - }, - }); - } - - it("plays the line once, straight through the pause", () => { - const cues = deriveCaptionCues(splitDoc(), ON, {}); - const first = cues.filter((cue) => cue.text.includes("really")); - expect(first).toHaveLength(1); - // It starts before the freeze and is still up after it — no dark stretch. - expect(first[0].startMs).toBeLessThan(1200); - expect(first[0].endMs).toBeGreaterThan(1700); - }); - - it("still plays a line twice when one media is genuinely placed twice", () => { - // The spans do not touch on the ruler there, so the coalescing must leave them apart. - const twice = doc({ - timeline: { - ...doc().timeline, - clips: [ - { - id: "clip-1", - assetId: "asset-1", - sourceStartSec: 0, - sourceEndSec: 10, - timelineStartSec: 0, - timelineEndSec: 10, - wordRefs: [], - origin: "user", - reason: "", - }, - { - id: "clip-2", - assetId: "asset-1", - sourceStartSec: 0, - sourceEndSec: 10, - timelineStartSec: 20, - timelineEndSec: 30, - wordRefs: [], - origin: "user", - reason: "", - }, - ], - }, - }); - const cues = deriveCaptionCues(twice, ON, {}); - expect(cues.filter((cue) => cue.text.includes("hello"))).toHaveLength(2); - }); -}); diff --git a/src/lib/ai-edition/captions/cues.ts b/src/lib/ai-edition/captions/cues.ts index 8154788a1..c3287ebcb 100644 --- a/src/lib/ai-edition/captions/cues.ts +++ b/src/lib/ai-edition/captions/cues.ts @@ -161,17 +161,6 @@ export function sourceSpanToTimelineSpans( const out: Array<{ startSec: number; endSec: number }> = []; for (const clip of clips) { if (clip.assetId !== assetId) continue; - // A FREEZE clip holds one source moment for created timeline time — an inserted - // word's pause. Its source window is that single point, so the overlap test below - // can never match it, and the line the pause exists for went DARK for its whole - // duration. A line covering the held moment covers the pause too. - if (clip.frozenSec !== undefined) { - const held = clip.sourceStartSec; - if (held >= startSec && held < endSec) { - out.push({ startSec: clip.timelineStartSec, endSec: clip.timelineEndSec }); - } - continue; - } const clipSourceEnd = clip.sourceEndSec ?? Number.POSITIVE_INFINITY; const s = Math.max(startSec, clip.sourceStartSec); const e = Math.min(endSec, clipSourceEnd); @@ -181,24 +170,7 @@ export function sourceSpanToTimelineSpans( endSec: clip.timelineStartSec + (e - clip.sourceStartSec), }); } - - // Coalesce what meets on the ruler. Splitting a clip to make room for an inserted word - // leaves the line straddling [before · freeze · after]: three spans back to back, each - // carrying the WHOLE line's text, so the caption played, blinked out over the pause, - // then played again from the top. They are one appearance. A line genuinely played - // twice — two clips over one media — does not touch on the ruler and stays two. - const EPSILON_SEC = 0.001; - const ordered = [...out].sort((a, b) => a.startSec - b.startSec); - const merged: Array<{ startSec: number; endSec: number }> = []; - for (const span of ordered) { - const last = merged[merged.length - 1]; - if (last && span.startSec <= last.endSec + EPSILON_SEC) { - last.endSec = Math.max(last.endSec, span.endSec); - continue; - } - merged.push({ ...span }); - } - return merged; + return out; } /** diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index fa41c2bc6..232bcb6ee 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -127,73 +127,6 @@ export function resequenceClips(clips: AxcutClip[]): AxcutClip[] { }); } -/** - * Split the clip covering `atSec` (source time, `assetId`'s media) into - * [before · FREEZE · after], where the freeze holds the frame at `atSec` for - * `frozenSec` of timeline time. The pause an inserted word creates: without it the - * word only borrows free silence, and a word dropped between two words that run into - * each other has no time at all. With it the timeline grows, everything downstream - * shifts, and a future TTS voice has a slot to speak in. - * - * No clip covers `atSec` (gap between clips, boundary of the asset) → unchanged, the - * caller decides whether that is acceptable. Trims anchored to the split clip stay on - * their id; `trimAppliesToClip` matches by asset when a trim has no clipId, and both - * halves keep the asset, so a trim that straddles the freeze point still narrows both - * halves exactly as it narrowed the whole clip before. - */ -export function insertFreezeInClips( - clips: AxcutClip[], - assetId: string, - atSec: number, - frozenSec: number, -): AxcutClip[] { - if (frozenSec <= 0) return clips; - const index = clips.findIndex( - (clip) => - clip.assetId === assetId && - (clip.frozenSec ?? 0) === 0 && - (clip.sourceEndSec ?? -1) > atSec + 0.001 && - atSec > clip.sourceStartSec, - ); - if (index < 0) return clips; - const clip = clips[index]; - // `resequenceClips` keeps each clip's OWN timeline length, so the halves must not - // inherit the un-split clip's — that would double the timeline. Lengths here are - // derived from the new source windows; resequence then only relays them. - const beforeLen = atSec - clip.sourceStartSec; - const afterLen = (clip.sourceEndSec ?? 0) - atSec; - const before: AxcutClip = { - ...clip, - id: `${clip.id}_fzA`, - sourceEndSec: atSec, - timelineEndSec: clip.timelineStartSec + beforeLen, - }; - const freeze: AxcutClip = { - id: `${clip.id}_fz`, - assetId: clip.assetId, - sourceStartSec: atSec, - sourceEndSec: atSec, - timelineStartSec: 0, - timelineEndSec: frozenSec, - wordRefs: [], - origin: "user", - reason: "Inserted word — held frame", - frozenSec, - }; - const after: AxcutClip | null = - afterLen > 0.001 - ? { - ...clip, - id: `${clip.id}_fzB`, - sourceStartSec: atSec, - timelineStartSec: 0, - timelineEndSec: afterLen, - } - : null; - const replaced = after ? [before, freeze, after] : [before, freeze]; - return resequenceClips([...clips.slice(0, index), ...replaced, ...clips.slice(index + 1)]); -} - export function subtractInterval(intervals: Interval[], cut: Interval): Interval[] { const output: Interval[] = []; for (const interval of intervals) { @@ -237,10 +170,7 @@ export function resolvePlaybackSegments( for (const clip of ordered) { const sourceEnd = clip.sourceEndSec ?? clip.sourceStartSec; if (sourceEnd <= clip.sourceStartSec) { - // Either not probed yet, or a FREEZE clip (source window is the point it - // holds; `frozenSec` carries its real length). Both pass through as one - // segment at their timeline length — for a freeze that KEEPS the created - // pause in the compressed stream, which is the whole point. + // Duration not probed yet — pass through as a single segment, unchanged. const dur = clip.timelineEndSec - clip.timelineStartSec; result.push({ ...clip, diff --git a/src/lib/ai-edition/document/transcript.test.ts b/src/lib/ai-edition/document/transcript.test.ts index 03f9e5aae..9e8ace067 100644 --- a/src/lib/ai-edition/document/transcript.test.ts +++ b/src/lib/ai-edition/document/transcript.test.ts @@ -615,65 +615,6 @@ describe("insertDocumentWord / removeDocumentWords", () => { }); }); -describe("insertDocumentWord freeze", () => { - /** One clip covering the whole fixture recording — what the editor starts from. */ - function makeDocWithClip() { - const doc = makeDoc(); - return { - ...doc, - timeline: { - ...doc.timeline, - clips: [ - { - id: "clip_1", - assetId: "asset_1", - sourceStartSec: 0, - sourceEndSec: 10, - timelineStartSec: 0, - timelineEndSec: 10, - wordRefs: [], - origin: "user" as const, - reason: "", - }, - ], - }, - }; - } - - it("splits the clip and creates held-frame time when the silence is insufficient", () => { - // "really" after word_2: word_3 starts exactly where word_2 ends, so the word - // gets no silence at all and needs max(0.4, 6/15) = 0.4 s of created time. - const result = insertDocumentWord(makeDocWithClip(), "asset_1", "word_2", "after", "really"); - const clips = result.timeline.clips; - expect(clips).toHaveLength(3); - expect(clips[0]).toMatchObject({ id: "clip_1_fzA", sourceStartSec: 0, sourceEndSec: 3 }); - expect(clips[1]).toMatchObject({ - id: "clip_1_fz", - sourceStartSec: 3, - sourceEndSec: 3, - frozenSec: 0.4, - }); - expect(clips[2]).toMatchObject({ id: "clip_1_fzB", sourceStartSec: 3, sourceEndSec: 10 }); - // The timeline grew by exactly the freeze, laid back-to-back. - expect(clips[1].timelineStartSec).toBe(3); - expect(clips[1].timelineEndSec).toBeCloseTo(3.4, 5); - expect(clips[2].timelineEndSec).toBeCloseTo(10.4, 5); - }); - - it("does not touch the clips when free silence covers the word", () => { - // word_3 ends at 4, word_4 starts at 5: a full second of silence. - const result = insertDocumentWord(makeDocWithClip(), "asset_1", "word_3", "after", "really"); - expect(result.timeline.clips).toHaveLength(1); - expect(result.timeline.clips[0].id).toBe("clip_1"); - }); - - it("leaves the timeline alone when no clip covers the insertion point", () => { - // makeDoc has no clips at all — the word rides the caption line, as before. - const result = insertDocumentWord(makeDoc(), "asset_1", "word_2", "after", "really"); - expect(result.timeline.clips).toHaveLength(0); - }); -}); - describe("carryOverWordEdits with inserted words", () => { const withInsert = () => insertWord(fixture(), "word_2", "after", "really"); diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts index ae626ee69..dec291474 100644 --- a/src/lib/ai-edition/document/transcript.ts +++ b/src/lib/ai-edition/document/transcript.ts @@ -1,5 +1,4 @@ import type { AxcutDocument, AxcutTranscript, AxcutWord } from "../schema"; -import { insertFreezeInClips } from "./timeline"; const CJK_EDGE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u; const CLOSING_PUNCTUATION = /^[,.;:!?%。，、；：！？…）)\]}>》」』】〕]/u; @@ -303,15 +302,7 @@ export function removeWord(transcript: AxcutTranscript, wordId: string): AxcutTr } /** {@link insertWord}, addressed by asset. Goes through `withTranscript` for the same - * reason {@link setDocumentWordText} does. - * - * When the free silence the word can borrow is shorter than the text needs to be - * read, the deficit becomes a FREEZE on the timeline (`insertFreezeInClips`): the clip - * is split at the word's edge and a held-frame clip carries the missing time. The - * timeline grows, everything downstream shifts, and the word has a real slot a - * synthesized voice will later speak in. No document-layer gate on this: it is the - * correct document semantics for an inserted word — the product decision of whether - * the gesture exists at all lives in the transcript pane (`openInsertion`). */ + * reason {@link setDocumentWordText} does. */ export function insertDocumentWord( document: AxcutDocument, assetId: string, @@ -323,28 +314,7 @@ export function insertDocumentWord( if (!transcript) { throw new Error(`Cannot insert a word into asset "${assetId}": it has no transcript`); } - const next = withTranscript(document, insertWord(transcript, anchorWordId, side, text)); - - // The word `insertWord` just added — the one synth id the old transcript lacked. - const inserted = next.transcripts - .find((t) => t.assetId === assetId) - ?.words.find( - (word) => word.source === "synth" && !transcript.words.some((w) => w.id === word.id), - ); - if (!inserted) return next; - const deficit = readingSeconds(inserted.text) - (inserted.endSec - inserted.startSec); - if (deficit <= 0.05) return next; - // The freeze continues the word's slot: after the word for "after" (silence, then - // held frame), before it for "before" (held frame, then silence) — one contiguous - // stretch of created time the future voice occupies. - const atSec = side === "after" ? inserted.endSec : inserted.startSec; - return { - ...next, - timeline: { - ...next.timeline, - clips: insertFreezeInClips(next.timeline.clips, assetId, atSec, deficit), - }, - }; + return withTranscript(document, insertWord(transcript, anchorWordId, side, text)); } /** {@link removeWord}, addressed by asset and taking the whole set at once: a Backspace diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index 155cddb2b..9a511fee5 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -215,14 +215,6 @@ export const clipSchema = z // that as the identity region {x:0,y:0,width:1,height:1} rather than // storing the identity explicitly, so untouched clips stay lean. cropRegion: clipCropRegionSchema.optional(), - // A FREEZE clip: holds the frame at `sourceStartSec` (source window is the - // zero-width point [sourceStartSec, sourceStartSec]) for `frozenSec` of - // TIMELINE time. The pause an inserted word creates so a future TTS voice has - // a slot to speak in — screen and webcam freeze together because both tracks - // are derived from the same asset source clock. Absent on every ordinary clip; - // `resolvePlaybackSegments`'s un-probed passthrough branch must not be confused - // with it (a frozen clip is pushed through unchanged too, see its comment). - frozenSec: z.number().positive().optional(), }) .refine((data) => data.timelineEndSec >= data.timelineStartSec, { message: "timelineEndSec must be greater than or equal to timelineStartSec", diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.test.ts b/src/lib/ai-edition/timeline/aggregated-transcript.test.ts index 60f1fdea6..adc70b6cb 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.test.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.test.ts @@ -474,94 +474,3 @@ describe("clipWordId", () => { expect(new Set(scopedIds).size).toBe(scopedIds.length); }); }); - -// ─── Freeze clips in the pane ──────────────────────────────────── -// An inserted word SPLITS the clip it lands in — [before · freeze · after] — and the word -// sits exactly on the split. Both the freeze and the half that starts there matched it, so -// the pane showed the same word twice, in two blocks. - -describe("the section a freeze clip projects", () => { - const TRANSCRIPT: AxcutTranscript = { - assetId: "a1", - language: "fr", - segments: [ - { id: "s1", kind: "speech", startSec: 0, endSec: 4, text: "un deux", wordIds: ["w1", "w2"] }, - ], - words: [ - { id: "w1", segmentId: "s1", startSec: 0, endSec: 2, text: "un" }, - { id: "w2", segmentId: "s1", startSec: 2, endSec: 4, text: "deux" }, - { id: "synth_1", segmentId: "s1", startSec: 2, endSec: 2, text: "vraiment", source: "synth" }, - ], - }; - const ASSET: AxcutAsset = { - id: "a1", - kind: "video", - label: "rec.mp4", - originalPath: "/r.mp4", - durationSec: 4, - cameraTrack: null, - }; - const CLIPS: AxcutClip[] = [ - { - id: "c_fzA", - assetId: "a1", - sourceStartSec: 0, - sourceEndSec: 2, - timelineStartSec: 0, - timelineEndSec: 2, - wordRefs: [], - origin: "user", - reason: "", - }, - { - id: "c_fz", - assetId: "a1", - sourceStartSec: 2, - sourceEndSec: 2, - timelineStartSec: 2, - timelineEndSec: 2.5, - wordRefs: [], - origin: "user", - reason: "", - frozenSec: 0.5, - }, - { - id: "c_fzB", - assetId: "a1", - sourceStartSec: 2, - sourceEndSec: 4, - timelineStartSec: 2.5, - timelineEndSec: 4.5, - wordRefs: [], - origin: "user", - reason: "", - }, - ]; - - it("shows the inserted word the freeze exists for", () => { - const sections = buildAggregatedSections(CLIPS, [TRANSCRIPT], [ASSET], []); - expect(sections[1].words.map((cw) => cw.word.text)).toEqual(["vraiment"]); - }); - - it("shows it exactly once across the whole pane", () => { - const sections = buildAggregatedSections(CLIPS, [TRANSCRIPT], [ASSET], []); - const everywhere = sections.flatMap((section) => - section.words.filter((cw) => cw.word.id === "synth_1"), - ); - expect(everywhere).toHaveLength(1); - }); - - it("leaves the spoken words where they were", () => { - const sections = buildAggregatedSections(CLIPS, [TRANSCRIPT], [ASSET], []); - expect(sections[0].words.map((cw) => cw.word.text)).toEqual(["un"]); - expect(sections[2].words.map((cw) => cw.word.text)).toEqual(["deux"]); - }); - - // The claim is scoped to freezes: with no freeze in the timeline, a word with no - // duration is shown by whichever clip its moment falls in, as before. - it("does not withhold an inserted word when no freeze claims it", () => { - const whole: AxcutClip[] = [{ ...CLIPS[0], id: "c1", sourceEndSec: 4, timelineEndSec: 4 }]; - const sections = buildAggregatedSections(whole, [TRANSCRIPT], [ASSET], []); - expect(sections[0].words.map((cw) => cw.word.text)).toContain("vraiment"); - }); -}); diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.ts b/src/lib/ai-edition/timeline/aggregated-transcript.ts index de46d071e..0cd068056 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.ts @@ -160,32 +160,6 @@ export function buildClipSection( asset: AxcutAsset | null, trimRanges: AxcutTrimRange[], ): ClipSection { - // A FREEZE clip is the created time behind an inserted word: its source window is - // the single point it holds, so the ordinary range filter matches nothing — and it - // must not. The words it shows are the SYNTH words touching that point: the word - // the freeze exists for. While the playhead runs through the freeze the cue - // resolves against this section and lights the amber word through the whole pause. - if (clip.frozenSec !== undefined) { - const atSec = clip.sourceStartSec; - const words = transcript - ? transcript.words.filter( - (word) => word.source === "synth" && word.startSec <= atSec && word.endSec >= atSec, - ) - : []; - return { - clip, - asset, - transcript, - words: words.map((word) => ({ - id: clipWordId(clip.id, word.id), - word, - kept: true, - trimId: null, - })), - trimRuns: [], - }; - } - // `trimAppliesToClip` — not a bare `assetId` match — is what keeps a cut on the // second of two clips over the same media from also greying out the first one's // words. Same media, same source range: only the clip anchor tells them apart. @@ -270,7 +244,7 @@ export function buildAggregatedSections( ): ClipSection[] { const transcriptById = new Map(transcripts.map((t) => [t.assetId, t])); const assetById = new Map(assets.map((a) => [a.id, a])); - const sections = clips.map((clip) => + return clips.map((clip) => buildClipSection( clip, transcriptById.get(clip.assetId) ?? null, @@ -278,23 +252,6 @@ export function buildAggregatedSections( trimRanges, ), ); - - // A freeze clip claims the inserted word it was created for, and the clip after the - // split starts at the very moment that word sits on — so the word matched BOTH and the - // pane showed it twice, in two blocks. The freeze owns it: it is the section the - // playhead is inside while the pause plays, and the one whose whole reason to exist is - // that word. - const claimed = new Set( - sections - .filter((section) => section.clip.frozenSec !== undefined) - .flatMap((section) => section.words.map((cw) => cw.word.id)), - ); - if (claimed.size === 0) return sections; - return sections.map((section) => - section.clip.frozenSec !== undefined - ? section - : { ...section, words: section.words.filter((cw) => !claimed.has(cw.word.id)) }, - ); } /** Where the playback head currently is, in source time. */ diff --git a/src/lib/ai-edition/timeline/timelineMap.test.ts b/src/lib/ai-edition/timeline/timelineMap.test.ts index ecc81a2d0..ea3f6c61b 100644 --- a/src/lib/ai-edition/timeline/timelineMap.test.ts +++ b/src/lib/ai-edition/timeline/timelineMap.test.ts @@ -17,7 +17,6 @@ import { replacePillSpan, resolveNativePosition, resolvePillIds, - segmentRawSpanSec, } from "./timelineMap"; function clip(overrides: Partial & Pick): AxcutClip { @@ -806,63 +805,3 @@ describe("legacy groupId must never affect identity (regression: test 1)", () => expect(out[0]).not.toHaveProperty("groupId"); }); }); - -// ─── Freeze clips ──────────────────────────────────────────────── -// The pause an inserted word creates. Its source window is the single point it holds, so -// every reader that derives a length from `sourceEndSec - sourceStartSec` gets zero for it -// — and zero is the one answer that makes the playhead skip the pause entirely. - -describe("a freeze clip", () => { - const clips = [ - clip({ id: "a", assetId: "m", sourceStartSec: 0, sourceEndSec: 3 }), - clip({ - id: "fz", - assetId: "m", - sourceStartSec: 3, - sourceEndSec: 3, - timelineStartSec: 3, - timelineEndSec: 3.5, - frozenSec: 0.5, - }), - clip({ - id: "b", - assetId: "m", - sourceStartSec: 3, - sourceEndSec: 6, - timelineStartSec: 3.5, - timelineEndSec: 6.5, - }), - ]; - - it("keeps its created time in the playback segments", () => { - // It carries no source range to compress, so a naive reader drops it and the pause - // vanishes from playback while the ruler still counts it. - const segments = resolvePlaybackSegments(clips, []); - const freeze = segments.find((segment) => segment.id === "fz"); - expect(freeze).toBeDefined(); - expect((freeze?.timelineEndSec ?? 0) - (freeze?.timelineStartSec ?? 0)).toBeCloseTo(0.5, 5); - }); - - it("spans its frozen time on the raw ruler, not its (zero) source length", () => { - const span = segmentRawSpanSec(clips[1], clips); - expect(span.endSec - span.startSec).toBeCloseTo(0.5, 5); - }); - - it("holds the source clock still while the playhead runs through it", () => { - // The raw playhead DOES advance through the pause. Letting that delta reach the - // decoder would push it past the held frame into the content that belongs after. - const segments = resolvePlaybackSegments(clips, []); - for (const rawSec of [3.05, 3.25, 3.45]) { - const position = resolveNativePosition(rawSec, segments, clips); - expect(position?.clip.id).toBe("fz"); - expect(position?.sourceTimeSec).toBe(3); - } - }); - - it("hands the clip after the pause its own source moment again", () => { - const segments = resolvePlaybackSegments(clips, []); - const position = resolveNativePosition(4, segments, clips); - expect(position?.clip.id).toBe("b"); - expect(position?.sourceTimeSec).toBeCloseTo(3.5, 5); - }); -}); diff --git a/src/lib/ai-edition/timeline/timelineMap.ts b/src/lib/ai-edition/timeline/timelineMap.ts index 081ede4d7..b98601a03 100644 --- a/src/lib/ai-edition/timeline/timelineMap.ts +++ b/src/lib/ai-edition/timeline/timelineMap.ts @@ -403,13 +403,7 @@ export function segmentRawSpanSec( rawClips: AxcutClip[], ): { startSec: number; endSec: number } { const startSec = getRawVirtualStartTime(segment, rawClips); - // A freeze clip's source window is the point it holds — its RAW span is its created - // timeline time, not the (zero) source length, or the playhead could never be - // "inside" it and would skip the pause entirely. - const lenSec = - segment.frozenSec !== undefined - ? segment.frozenSec - : (segment.sourceEndSec ?? segment.sourceStartSec) - segment.sourceStartSec; + const lenSec = (segment.sourceEndSec ?? segment.sourceStartSec) - segment.sourceStartSec; return { startSec, endSec: startSec + lenSec }; } @@ -696,20 +690,6 @@ export function resolveNativePosition( const seg = visibleSegments[index]; const segSourceEnd = seg.sourceEndSec ?? seg.sourceStartSec; - // Inside a freeze: the source clock does not advance. The whole point of the clip - // is created timeline time over one held frame — clamp the offset to zero rather - // than letting the raw-playhead delta (which DOES advance through the freeze) push - // the decoder past the held frame into content that belongs after the pause. - if (seg.frozenSec !== undefined) { - return { - clip: seg, - clipIndex: index, - sourceTimeSec: seg.sourceStartSec, - }; - } - // No `clampToSegmentStart` any more: this branch used to snap the playhead to the - // next kept segment, and main now returns `positionUnderCut` before reaching here - // (issue #216), so the flag could never be true. const unclamped = seg.sourceStartSec + (rawSec - spans[index].startSec); const maxSource = Math.max(seg.sourceStartSec, segSourceEnd - NATIVE_EOF_MARGIN_SEC); return { diff --git a/src/native/useNativePlaybackSync.ts b/src/native/useNativePlaybackSync.ts index abe2f1c1a..64e877b63 100644 --- a/src/native/useNativePlaybackSync.ts +++ b/src/native/useNativePlaybackSync.ts @@ -41,13 +41,6 @@ export function useNativePlaybackSync( ); const activeClipId = activePosition?.clip.id ?? null; const sourceTimeSec = activePosition?.sourceTimeSec ?? null; - // A freeze clip holds ONE frame for `frozenSec` of app-clock time. Free-running the - // decoder through it would play the frames after the pause instead; the app clock - // (which does traverse the freeze) then re-seeks on drift and stutters. Pausing the - // decoder for the duration of the freeze is what makes the pause a pause — the - // webcam track freezes with the screen track because both derive from the same - // asset source clock the freeze stops advancing. - const frozen = activePosition?.clip.frozenSec !== undefined; // Reactive "is a native view active?" so activation mid-session re-pushes the // current transport/playhead (time & playing aren't memoised in the store). @@ -56,15 +49,13 @@ export function useNativePlaybackSync( () => getCurrentNativeViewId() !== null, ); - // Play/pause → native free-run. Inside a freeze the native side is PAUSED however - // the transport is set — the app clock advances through the created time while the - // decoder holds the frame. + // Play/pause → native free-run. useEffect(() => { if (!active) { return; } - setNativePlaying(playing && !frozen); - }, [active, playing, frozen]); + setNativePlaying(playing); + }, [active, playing]); // Scrub/step while paused OR periodic resync during playback when drift > 100ms const lastSyncedSourceTimeRef = useRef(null); @@ -77,17 +68,6 @@ export function useNativePlaybackSync( } const now = performance.now(); - // Inside a freeze while playing: the decoder is paused (see the transport - // effect) and parked on the held frame. Refresh the drift refs every run so the - // drift check never sees the (correctly) frozen source clock as divergence and - // fights itself with repeated seeks. - if (playing && frozen) { - setNativeTime(sourceTimeSec); - lastSyncedSourceTimeRef.current = sourceTimeSec; - lastSyncedWallTimeRef.current = now; - return; - } - // When clip changes, let setActiveClip handle the atomic clip-switch-and-seek. if (lastActiveClipIdRef.current !== activeClipId) { lastActiveClipIdRef.current = activeClipId; @@ -115,5 +95,5 @@ export function useNativePlaybackSync( lastSyncedSourceTimeRef.current = sourceTimeSec; lastSyncedWallTimeRef.current = now; } - }, [active, playing, frozen, activeClipId, sourceTimeSec]); + }, [active, playing, activeClipId, sourceTimeSec]); } From bef60e924c60bb1309eba22fd6f06a05c48f1c92 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 22:51:01 +0200 Subject: [PATCH 09/12] feat(timeline): mark where words were added, without storing anything new MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An added word was visible only in the transcript pane. On the timeline — where you decide what the film does — nothing said a moment carried text with no audio behind it. Each one now gets a thin amber tick on its clip's own track, at the moment it sits on, carrying its text in the tooltip and seeking to it on click. Amber is the colour the pane gives the same word, so the two read as one thing. Derived, never stored: the mark is computed from the transcript's `synth` words on every render, so there is no second record to fall out of step with the first, and nothing for another writer of `timeline.clips` to lose. Positioning is a percentage inside the clip's own box rather than an absolute ruler offset, so a mark travels with its clip through a reorder without arithmetic of its own. --- .../ai-edition/v4/EditorShellV4.module.css | 35 ++++++++++++++ .../v4/V4Timeline.geometry.test.tsx | 3 ++ src/components/ai-edition/v4/V4Timeline.tsx | 47 ++++++++++++++++++- .../v4/V4Timeline.waveform.test.tsx | 2 + src/i18n/locales/ar/timeline.json | 3 +- src/i18n/locales/en/timeline.json | 3 +- src/i18n/locales/es/timeline.json | 3 +- src/i18n/locales/fr/timeline.json | 3 +- src/i18n/locales/it/timeline.json | 3 +- src/i18n/locales/ja-JP/timeline.json | 3 +- src/i18n/locales/ko-KR/timeline.json | 3 +- src/i18n/locales/pt-BR/timeline.json | 3 +- src/i18n/locales/ru/timeline.json | 3 +- src/i18n/locales/tr/timeline.json | 3 +- src/i18n/locales/vi/timeline.json | 3 +- src/i18n/locales/zh-CN/timeline.json | 3 +- src/i18n/locales/zh-TW/timeline.json | 3 +- src/lib/ai-edition/store/useTimeline.ts | 4 ++ 18 files changed, 116 insertions(+), 14 deletions(-) diff --git a/src/components/ai-edition/v4/EditorShellV4.module.css b/src/components/ai-edition/v4/EditorShellV4.module.css index 1e313e34c..fd19b3f57 100644 --- a/src/components/ai-edition/v4/EditorShellV4.module.css +++ b/src/components/ai-edition/v4/EditorShellV4.module.css @@ -1643,6 +1643,41 @@ background: var(--danger-soft); color: var(--danger); } + +/* Where the user has ADDED a word: text with no audio behind it. A thin amber tick + over the waveform, at the moment the word sits on, wide enough to hit and no wider + — the clip underneath still has to be draggable everywhere else. Amber is the + colour the transcript pane gives the same word, so the two read as one thing. */ +.tlClipInsert { + position: absolute; + top: 0; + bottom: 0; + z-index: 2; + width: 9px; + margin-left: -4px; + padding: 0; + border: 0; + background: transparent; + cursor: pointer; +} +.tlClipInsert::before { + content: ""; + position: absolute; + left: 3px; + top: 4px; + bottom: 4px; + width: 3px; + border-radius: 2px; + background: var(--warn); + box-shadow: 0 0 0 1px color-mix(in srgb, #080a0d 45%, transparent); + transition: box-shadow var(--motion-fast) var(--ease); +} +.tlClipInsert:hover::before, +.tlClipInsert:focus-visible::before { + box-shadow: + 0 0 0 1px color-mix(in srgb, #080a0d 45%, transparent), + 0 0 0 4px var(--warn-soft); +} .tlDropHint { position: absolute; inset: 0; diff --git a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx index 8e515568c..f8d0b9b67 100644 --- a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx +++ b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx @@ -75,6 +75,9 @@ function renderTimeline( ) { const tl = { clips, + // Marks for added words are read straight off the transcript (see the pane's + // amber words) — no project here has any. + transcripts: [], assets, annotationRegions: [annotation], speedRegions: [], diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index f6c94e2ca..901863ad0 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -29,7 +29,7 @@ import { useScopedT } from "@/contexts/I18nContext"; import { useAudioPeaks } from "@/hooks/useAudioPeaks"; import { createId } from "@/lib/ai-edition/document/ids"; import { setUiProbeScrubbing } from "@/lib/ai-edition/perf/uiFrameProbe"; -import type { AxcutClip } from "@/lib/ai-edition/schema"; +import type { AxcutClip, AxcutWord } from "@/lib/ai-edition/schema"; import { audioGainScalar } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useTimelineTranscriptGate } from "@/lib/ai-edition/store/transcriptionStore"; @@ -509,6 +509,32 @@ export function V4Timeline({ label: `${(p.member.customScale ?? ZOOM_DEPTH_SCALES[p.member.depth]).toFixed(2)}×`, sourceIds: p.ids, })); + // Where the user has ADDED words. Derived from the transcript on every render and + // stored nowhere: the word carries `source: "synth"` and its own source time, so a mark + // built from it cannot drift from the amber word the transcript pane shows. Grouped by + // clip because each mark is positioned inside its clip's own box — it then travels with + // the clip through a reorder for free, with no ruler arithmetic of its own. + const insertedWordsByClip = useMemo(() => { + const byAsset = new Map(); + for (const transcript of tl.transcripts) { + const added = transcript.words.filter((word) => word.source === "synth"); + if (added.length > 0) byAsset.set(transcript.assetId, added); + } + if (byAsset.size === 0) return new Map>(); + const out = new Map>(); + for (const clip of clips) { + const words = byAsset.get(clip.assetId); + const sourceEnd = clip.sourceEndSec ?? clip.sourceStartSec; + const span = sourceEnd - clip.sourceStartSec; + if (!words || span <= 0) continue; + const marks = words + .filter((word) => word.startSec >= clip.sourceStartSec && word.startSec <= sourceEnd) + .map((word) => ({ word, atPct: ((word.startSec - clip.sourceStartSec) / span) * 100 })); + if (marks.length > 0) out.set(clip.id, marks); + } + return out; + }, [tl.transcripts, clips]); + // trims: content-free (no per-instance text/settings), so touching rows — // inevitable once a trim is ventilated across a clip boundary — are // coalesced into one pill. This is what makes growing a trim across a @@ -1578,6 +1604,25 @@ export function V4Timeline({ {tl.assets.find((a) => a.id === c.assetId)?.label ?? c.assetId} + {(insertedWordsByClip.get(c.id) ?? []).map(({ word, atPct }) => ( +