From 82af80e5fe2ab70d1ca9fab22c8c7ac73a02fb05 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Sat, 22 Aug 2026 11:39:50 +0200 Subject: [PATCH 1/3] fix(stt): offer every whisper.cpp language in the regenerate picker The "Regenerate as" selector only listed 11 hand-picked languages while the shipped whisper small model transcribes ~100. TRANSCRIPT_LANGUAGE_CODES in schema/index.ts is now the single source of truth (mirrors whisper.cpp's own g_lang table) for both the zod schema and the picker, which sorts by name localized via Intl.DisplayNames (falling back to whisper's English name) instead of a hardcoded list of bare codes. --- src/components/ai-edition/Modals.tsx | 100 ++++---- src/lib/ai-edition/schema/index.ts | 225 +++++++++++++++++- .../transcription-and-captions.md | 21 +- 3 files changed, 292 insertions(+), 54 deletions(-) diff --git a/src/components/ai-edition/Modals.tsx b/src/components/ai-edition/Modals.tsx index 9ac9d1079..4a6c740b6 100644 --- a/src/components/ai-edition/Modals.tsx +++ b/src/components/ai-edition/Modals.tsx @@ -17,50 +17,55 @@ import { type ReactNode, type PointerEvent as ReactPointerEvent, useEffect, + useMemo, useRef, useState, } from "react"; import { toFileUrl } from "@/components/video-editor/projectPersistence"; import type { CropRegion } from "@/components/video-editor/types"; -import { useScopedT } from "@/contexts/I18nContext"; +import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { toAxcutTranscriptDsl } from "@/lib/ai-edition/document/transcribe"; -import type { AxcutClip, AxcutTranscript } from "@/lib/ai-edition/schema"; +import { + type AxcutClip, + type AxcutTranscript, + TRANSCRIPT_LANGUAGE_CODES, + TRANSCRIPT_LANGUAGE_NAMES, + type TranscriptLanguageCode, +} from "@/lib/ai-edition/schema"; import { formatSec, formatSeconds } from "@/lib/ai-edition/timeline/format"; import styles from "./NewEditorShell.module.css"; import type { VideoSource } from "./VirtualPreview"; -// ponytail: keep the UI's language list literal in one place. Mirrors -// `transcriptLanguageSchema` in schema/index.ts; if the schema gains a -// language, add it here too. -const REGEN_LANGUAGES = [ - "auto", - "en", - "fr", - "de", - "es", - "it", - "pt", - "nl", - "ja", - "ko", - "zh", -] as const; - -type TranscriptLanguage = (typeof REGEN_LANGUAGES)[number]; - -const LANGUAGE_LABELS: Record = { - auto: "Auto", - en: "EN", - fr: "FR", - de: "DE", - es: "ES", - it: "IT", - pt: "PT", - nl: "NL", - ja: "JA", - ko: "KO", - zh: "ZH", -}; +const languageDisplayNamesCache = new Map(); + +function languageDisplayNamesFor(locale: string): Intl.DisplayNames | null { + const cached = languageDisplayNamesCache.get(locale); + if (cached) return cached; + try { + const names = new Intl.DisplayNames([locale], { type: "language" }); + languageDisplayNamesCache.set(locale, names); + return names; + } catch { + return null; + } +} + +/** + * Localized name for a whisper.cpp language code, e.g. "jw" -> "Javanese" in + * an English UI, "japonais" in a French one. Falls back to whisper's own + * English name (`TRANSCRIPT_LANGUAGE_NAMES`) when the active locale's ICU + * data can't resolve one — `Intl.DisplayNames` echoes the input code back + * rather than throwing when it doesn't recognize it. + */ +function languageLabel(code: Exclude, locale: string): string { + try { + const resolved = languageDisplayNamesFor(locale)?.of(code); + if (resolved && resolved.toLowerCase() !== code.toLowerCase()) return resolved; + } catch { + // Malformed/unsupported subtag for this Intl implementation. + } + return TRANSCRIPT_LANGUAGE_NAMES[code]; +} interface BaseModalProps { open: boolean; @@ -1548,7 +1553,7 @@ export interface SourceTranscriptModalProps extends BaseModalProps { isFailed: boolean; /** Why the last run produced nothing — "no audio track", or the engine's own message. */ failureMessage?: string; - onRegenerate: (language: TranscriptLanguage) => void; + onRegenerate: (language: TranscriptLanguageCode) => void; } export function SourceTranscriptModal({ @@ -1569,17 +1574,30 @@ export function SourceTranscriptModal({ const [isPlaying, setIsPlaying] = useState(false); const [playTime, setPlayTime] = useState(0); const [duration, setDuration] = useState(null); - const [regenLang, setRegenLang] = useState( - (transcript?.language as TranscriptLanguage) ?? "auto", + const { locale } = useI18n(); + const [regenLang, setRegenLang] = useState( + (transcript?.language as TranscriptLanguageCode) ?? "auto", ); // ponytail: sync the language picker to whatever the stored transcript was // generated with. Avoids surprising the user with a different selection on // every open after a regenerate. useEffect(() => { - if (open) setRegenLang((transcript?.language as TranscriptLanguage) ?? "auto"); + if (open) setRegenLang((transcript?.language as TranscriptLanguageCode) ?? "auto"); }, [open, transcript?.language]); + // Auto pinned first; the rest sorted by localized name so a 100-language + // list is scannable instead of ordered by whisper's internal language id. + const regenLanguageOptions = useMemo(() => { + const collator = new Intl.Collator(locale); + const rest = TRANSCRIPT_LANGUAGE_CODES.filter( + (code): code is Exclude => code !== "auto", + ) + .map((code) => ({ code, label: languageLabel(code, locale) })) + .sort((a, b) => collator.compare(a.label, b.label)); + return [{ code: "auto" as const, label: t("mediaStage.auto") }, ...rest]; + }, [locale, t]); + useEffect(() => { if (!open) { setIsPlaying(false); @@ -1900,7 +1918,7 @@ export function SourceTranscriptModal({ aria-label={t("mediaStage.regenerateAs")} value={regenLang} disabled={isTranscribing} - onChange={(e) => setRegenLang(e.target.value as TranscriptLanguage)} + onChange={(e) => setRegenLang(e.target.value as TranscriptLanguageCode)} style={{ width: "100%", padding: "10px 12px", @@ -1911,9 +1929,9 @@ export function SourceTranscriptModal({ font: "500 13px var(--font-body)", }} > - {REGEN_LANGUAGES.map((code) => ( + {regenLanguageOptions.map(({ code, label }) => ( ))} diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index 09743fa9e..8fe33fc1a 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -811,19 +811,229 @@ export const chatInputSchema = z.object({ message: z.string().trim().min(1), }); -export const transcriptLanguageSchema = z.enum([ +// Every code whisper.cpp's multilingual model can resolve, plus "auto" for +// detection. Codes and order mirror whisper.cpp's own `g_lang` table +// (`src/whisper.cpp`, verified against the tag `nix/whisper-stt.nix` pins — +// v1.9.1) — `wparams.language` in +// electron/native/whisper-stt/src/main.cpp forwards the code verbatim, so a +// value outside this list fails to resolve a language id there. The UI's +// "Regenerate as" picker (Modals.tsx) is the only current consumer. +export const TRANSCRIPT_LANGUAGE_CODES = [ "auto", "en", - "fr", + "zh", "de", "es", - "it", + "ru", + "ko", + "fr", + "ja", "pt", + "tr", + "pl", + "ca", "nl", - "ja", - "ko", - "zh", -]); + "ar", + "sv", + "it", + "id", + "hi", + "fi", + "vi", + "he", + "uk", + "el", + "ms", + "cs", + "ro", + "da", + "hu", + "ta", + "no", + "th", + "ur", + "hr", + "bg", + "lt", + "la", + "mi", + "ml", + "cy", + "sk", + "te", + "fa", + "lv", + "bn", + "sr", + "az", + "sl", + "kn", + "et", + "mk", + "br", + "eu", + "is", + "hy", + "ne", + "mn", + "bs", + "kk", + "sq", + "sw", + "gl", + "mr", + "pa", + "si", + "km", + "sn", + "yo", + "so", + "af", + "oc", + "ka", + "be", + "tg", + "sd", + "gu", + "am", + "yi", + "lo", + "uz", + "fo", + "ht", + "ps", + "tk", + "nn", + "mt", + "sa", + "lb", + "my", + "bo", + "tl", + "mg", + "as", + "tt", + "haw", + "ln", + "ha", + "ba", + "jw", + "su", + "yue", +] as const; + +export const transcriptLanguageSchema = z.enum(TRANSCRIPT_LANGUAGE_CODES); + +/** + * English fallback name per code, from whisper.cpp's own `g_lang` table. + * The UI prefers `Intl.DisplayNames` in the active locale and falls back to + * this when that can't resolve a name (e.g. an unusual code on an older ICU). + */ +export const TRANSCRIPT_LANGUAGE_NAMES: Record< + Exclude<(typeof TRANSCRIPT_LANGUAGE_CODES)[number], "auto">, + string +> = { + en: "English", + zh: "Chinese", + de: "German", + es: "Spanish", + ru: "Russian", + ko: "Korean", + fr: "French", + ja: "Japanese", + pt: "Portuguese", + tr: "Turkish", + pl: "Polish", + ca: "Catalan", + nl: "Dutch", + ar: "Arabic", + sv: "Swedish", + it: "Italian", + id: "Indonesian", + hi: "Hindi", + fi: "Finnish", + vi: "Vietnamese", + he: "Hebrew", + uk: "Ukrainian", + el: "Greek", + ms: "Malay", + cs: "Czech", + ro: "Romanian", + da: "Danish", + hu: "Hungarian", + ta: "Tamil", + no: "Norwegian", + th: "Thai", + ur: "Urdu", + hr: "Croatian", + bg: "Bulgarian", + lt: "Lithuanian", + la: "Latin", + mi: "Maori", + ml: "Malayalam", + cy: "Welsh", + sk: "Slovak", + te: "Telugu", + fa: "Persian", + lv: "Latvian", + bn: "Bengali", + sr: "Serbian", + az: "Azerbaijani", + sl: "Slovenian", + kn: "Kannada", + et: "Estonian", + mk: "Macedonian", + br: "Breton", + eu: "Basque", + is: "Icelandic", + hy: "Armenian", + ne: "Nepali", + mn: "Mongolian", + bs: "Bosnian", + kk: "Kazakh", + sq: "Albanian", + sw: "Swahili", + gl: "Galician", + mr: "Marathi", + pa: "Punjabi", + si: "Sinhala", + km: "Khmer", + sn: "Shona", + yo: "Yoruba", + so: "Somali", + af: "Afrikaans", + oc: "Occitan", + ka: "Georgian", + be: "Belarusian", + tg: "Tajik", + sd: "Sindhi", + gu: "Gujarati", + am: "Amharic", + yi: "Yiddish", + lo: "Lao", + uz: "Uzbek", + fo: "Faroese", + ht: "Haitian Creole", + ps: "Pashto", + tk: "Turkmen", + nn: "Norwegian Nynorsk", + mt: "Maltese", + sa: "Sanskrit", + lb: "Luxembourgish", + my: "Myanmar", + bo: "Tibetan", + tl: "Tagalog", + mg: "Malagasy", + as: "Assamese", + tt: "Tatar", + haw: "Hawaiian", + ln: "Lingala", + ha: "Hausa", + ba: "Bashkir", + jw: "Javanese", + su: "Sundanese", + yue: "Cantonese", +}; export type AxcutWord = z.infer; export type AxcutTranscriptSegment = z.infer; @@ -845,6 +1055,7 @@ export type AxcutDocumentInput = z.input; export type CreateProjectInput = z.infer; export type AddAssetInput = z.infer; export type ChatInput = z.infer; +export type TranscriptLanguageCode = z.infer; export function createEmptyDocument( input: CreateProjectInput & { projectId: string; createdAt?: string }, diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index 02bf9ae8b..551aeff47 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -636,16 +636,25 @@ it deletes data ## Known gaps -- **No language selector in the UI.** The renderer always sends - `language: "auto"`. Forcing a language would skip detection on the - first window and slightly improve WER. Needs a UI control wired - through `setTranscript` and a mapping from the UI string to a - whisper.cpp language token. +- **Language selector.** The "Regenerate as" picker + (`SourceTranscriptModal` in + [`src/components/ai-edition/Modals.tsx`](../../src/components/ai-edition/Modals.tsx)) + offers `"auto"` plus every language the `small` multilingual model + resolves. `TRANSCRIPT_LANGUAGE_CODES` / `TRANSCRIPT_LANGUAGE_NAMES` in + [`src/lib/ai-edition/schema/index.ts`](../../src/lib/ai-edition/schema/index.ts) + mirror whisper.cpp's own `g_lang` table verbatim — a code outside that list + fails to resolve a language id in `wparams.language` + (`electron/native/whisper-stt/src/main.cpp`) — and are the single source + the picker's options are built from, so the two cannot drift the way a + hand-duplicated list would. Option labels come from `Intl.DisplayNames` in + the active UI locale, falling back to whisper.cpp's own English name for a + code that locale's ICU data can't resolve. Forcing a language skips + detection on the first window and slightly improves WER. Note that `"auto"` is a *request* value only. The helper used to echo the request straight back into `detected_language`, so with no selector the field was permanently the literal string `"auto"`: the media stage's "detected - language" line ([`src/components/ai-edition/Modals.tsx:1763`](../../src/components/ai-edition/Modals.tsx:1763)) + language" line ([`src/components/ai-edition/Modals.tsx:1891`](../../src/components/ai-edition/Modals.tsx:1891)) displayed it verbatim, and `transcribe.ts` wrote it onto `AxcutTranscript.language`. It now reports `whisper_full_lang_id()` — the detected language under `"auto"`, the forced one otherwise. From 8cc93c49edd503003f69fd6204e02db8bedfd605 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Sat, 22 Aug 2026 12:24:05 +0200 Subject: [PATCH 2/3] fix(stt): wire the language list into the picker that's actually live A subagent review caught that the first commit only widened SourceTranscriptModal, which NewEditorShell never mounts (LeftPanel is always rendered with active="chat"). The picker a real user opens is MediaStage.tsx's own, separate setLang(e.target.value)} + onChange={(e) => setLang(e.target.value as TranscriptLanguageCode)} style={{ flex: 1, minWidth: 0, @@ -387,10 +398,11 @@ export function MediaStage({ outline: "none", }} > - - - - + {regenLanguageOptions.map(({ code, label }) => ( + + ))}