diff --git a/.gitignore b/.gitignore index a1822ee05..24ad425eb 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ workbench/fixtures/ /aur_ci /aur_ci.pub /aur_known_hosts +tmp_handoff.md diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 69476ffc2..95700a6cb 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -364,6 +364,10 @@ pub struct ClipInput { pub webcam_offset_sec: f64, /// `false` évite une ouverture ffmpeg vouée à échouer et réserve du silence à ce clip. pub has_audio: bool, + /// Secondes de sortie pendant lesquelles ce clip TIENT sa dernière image, en silence. + /// Une pause achetée par un mot ajouté (issue #560) : le ruler et la preview la + /// respectaient déjà, l'export l'ignorait, faute d'une fenêtre source non vide. + pub hold_sec: f64, } /// Taille/cadence/codec de sortie voulus par l'app (modale d'export). Tous optionnels : @@ -500,6 +504,7 @@ pub fn export_multi( source_end_sec: c.source_end_sec, webcam_offset_sec: c.webcam_offset_sec, has_audio: c.has_audio, + hold_sec: c.hold_sec, }) .collect(); Ok(AsyncTask::new(ExportMultiTask { @@ -657,6 +662,7 @@ pub fn export_gif( source_end_sec: c.source_end_sec, webcam_offset_sec: c.webcam_offset_sec, has_audio: c.has_audio, + hold_sec: c.hold_sec, }) .collect(); let gif_params = params diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index e51f0f8af..6d7c8d4ec 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -5,7 +5,7 @@ use crate::ffi::*; use crate::regions::SpeedSegment; -use crate::scene::SceneAudio; +use crate::scene::{SceneAudio, SceneAudioTrack}; use anyhow::{bail, Result}; use std::f32::consts::PI; use std::ffi::CString; @@ -1381,6 +1381,142 @@ pub fn assemble_concatenated_pcm( output } +/// Mix imported audio tracks (issue #350) over the assembled programme. +/// +/// Each track is decoded across its trim window — already resampled to 48 kHz +/// stereo by `decode_clip_audio`, the same path a clip's own audio takes — scaled +/// by its per-track gain (the same `10^(dB/20)` law as `finish_audio`), and summed +/// into the programme at `start_sec`. The programme length is NOT extended: a +/// track that runs past the video is truncated to it, so the audio and video +/// streams stay the same length for the muxer. +/// +/// The decode window is capped up front at the room left in the programme after +/// `start_sec`, and a track starting at/after the end is skipped without decoding. +/// `decode_clip_audio` preallocates from the window, so this keeps a long track +/// pinned near a short programme's end from buffering (and clamping away) hours of +/// PCM. `trim_end_sec` must therefore be concrete — the renderer sends +/// `trimEnd ?? durationSec`. +/// +/// A track whose file has no decodable audio is skipped — the same degradation a +/// stream-less clip gets. +pub fn mix_external_tracks(mut programme: PlanarPcm, tracks: &[SceneAudioTrack]) -> PlanarPcm { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if programme_len == 0 { + return programme; + } + for track in tracks { + let offset = (track.start_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; + // A track that starts at or past the programme end contributes nothing — + // skip it before decoding anything. + if offset >= programme_len { + continue; + } + let trim_start = track.trim_start_sec.max(0.0); + let Some(trim_end_full) = track.trim_end_sec else { + // Without a concrete end there is no safe window to decode (see the doc + // comment); the renderer always resolves one, so this only guards a + // hand-written scene. + continue; + }; + // Cap the decode window at the room left in the programme. Everything past + // `offset` that overflows is discarded by `overlay_track_pcm` anyway, so + // decoding it only wastes time and memory — a three-hour track placed at + // second 9 of a ten-second export must not buffer three hours of PCM. + let remaining_sec = (programme_len - offset) as f64 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + let trim_end = trim_end_full.min(trim_start + remaining_sec); + if trim_end <= trim_start { + continue; + } + let decoded = match decode_clip_audio(&track.path, trim_start, trim_end) { + Ok(Some(pcm)) => pcm, + _ => continue, + }; + // The app's own range is -60..+12 dB (the inspector slider); clamping at + // -12 here floored every quiet bed at a tenth of the attenuation asked for. + let gain = 10.0f32.powf(track.gain_db.clamp(-60.0, 12.0) / 20.0); + overlay_track_pcm( + &mut programme, + &decoded, + offset, + gain, + track.fade_in_sec.max(0.0), + track.fade_out_sec.max(0.0), + ); + } + programme +} + +/// Sum one decoded track into the programme at `offset` samples, scaled by `gain`, +/// truncated at the programme's end. Split out of `mix_external_tracks` so the +/// placement/gain/clamp math is testable without ffmpeg, exactly like +/// `mix_aligned_tracks` is split from the decode above. +fn overlay_track_pcm( + programme: &mut PlanarPcm, + decoded: &PlanarPcm, + offset: usize, + gain: f32, + fade_in_sec: f64, + fade_out_sec: f64, +) { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if offset >= programme_len { + return; + } + let room = programme_len - offset; + // The ramps are measured against the DECODED length, not the room left in the + // programme: a track running past the end is cut off there, and a fade-out + // timed to the cut would ramp down over audio the export never reaches. + let decoded_len = decoded.iter().map(Vec::len).max().unwrap_or(0); + let (fade_in, fade_out) = resolve_fade_samples(decoded_len, fade_in_sec, fade_out_sec); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let Some(source) = decoded.get(channel) else { + continue; + }; + let count = source.len().min(room); + let dst = &mut programme[channel]; + for k in 0..count { + dst[offset + k] += source[k] * gain * fade_envelope(k, decoded_len, fade_in, fade_out); + } + } +} + +/// Fade lengths in samples, reduced to fit inside `len`. +/// +/// Fades that do not fit share the window in proportion rather than being clamped +/// independently: clamping each to the length first would turn an asymmetric pair +/// into a symmetric one, losing the shape asked for. Kept identical to the app's +/// `resolveFadeSecs` so the preview and the render agree. +fn resolve_fade_samples(len: usize, fade_in_sec: f64, fade_out_sec: f64) -> (usize, usize) { + if len == 0 { + return (0, 0); + } + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let mut fade_in = fade_in_sec.max(0.0) * rate; + let mut fade_out = fade_out_sec.max(0.0) * rate; + let total = fade_in + fade_out; + if total > len as f64 && total > 0.0 { + let scale = len as f64 / total; + fade_in *= scale; + fade_out *= scale; + } + (fade_in.round() as usize, fade_out.round() as usize) +} + +/// Linear ramp factor at sample `k` of a `len`-sample track. +fn fade_envelope(k: usize, len: usize, fade_in: usize, fade_out: usize) -> f32 { + let mut v = 1.0f32; + if fade_in > 0 && k < fade_in { + v = v.min(k as f32 / fade_in as f32); + } + if fade_out > 0 && len > k { + let remaining = len - k; + if remaining <= fade_out { + v = v.min(remaining as f32 / fade_out as f32); + } + } + v +} + /// Encodeur AAC attaché au muxer avant son header. Les paquets utilisent le même interleaver /// que la vidéo ; les pts restent en unités échantillon jusqu'au rescale vers l'AVStream. pub(crate) struct AacEncoder { @@ -1493,6 +1629,40 @@ impl Drop for AacEncoder { } } +#[cfg(test)] +mod hold_tests { + use super::*; + + /// Les images tenues allongent le CRÉNEAU audio du clip sans allonger son PCM, et + /// `assemble_concatenated_pcm` laisse des zéros dans ce qui dépasse. Le silence d'une + /// pause est donc gratuit : aucun fichier muet à décoder, aucune entrée de mix en plus. + #[test] + fn a_longer_slot_than_pcm_leaves_silence_at_its_tail() { + // 2s de créneau à 1 fps, mais seulement 1s de PCM décodé. + let plan = build_audio_concat_plan(&[2], &[true], 1.0); + let one_sec = AUDIO_OUTPUT_SAMPLE_RATE as usize; + let pcm = vec![Some(vec![vec![0.5f32; one_sec]; AUDIO_OUTPUT_CHANNELS])]; + let out = assemble_concatenated_pcm(&pcm, &plan); + assert_eq!(out[0].len(), 2 * one_sec); + assert!((out[0][0] - 0.5).abs() < 1e-6, "le vrai son est bien là"); + assert_eq!(out[0][2 * one_sec - 1], 0.0, "la queue du créneau est du silence"); + } + + /// Et le son réel n'est PAS étiré pour remplir le créneau : la voix garde son rythme. + #[test] + fn the_clips_own_audio_is_not_stretched_to_fill_the_hold() { + let plan = build_audio_concat_plan(&[4], &[true], 1.0); + let one_sec = AUDIO_OUTPUT_SAMPLE_RATE as usize; + let mut source = vec![0.0f32; one_sec]; + source[0] = 1.0; + let pcm = vec![Some(vec![source.clone(), source])]; + let out = assemble_concatenated_pcm(&pcm, &plan); + // L'impulsion reste au premier échantillon, pas répartie sur quatre secondes. + assert!((out[0][0] - 1.0).abs() < 1e-6); + assert_eq!(out[0][1], 0.0); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1692,6 +1862,68 @@ mod tests { assert_eq!(mixed[1], vec![0.25, -0.5, 0.75]); } + // Imported audio track overlay (issue #350). + #[test] + fn overlay_sums_at_offset_with_gain() { + let mut programme = planar(&[0.1, 0.1, 0.1, 0.1]); + // ×2 gain, placed at sample offset 1. + overlay_track_pcm(&mut programme, &planar(&[0.2, 0.2]), 1, 2.0, 0.0, 0.0); + assert_eq!(programme[0], vec![0.1, 0.5, 0.5, 0.1]); + assert_eq!(programme[1], vec![0.1, 0.5, 0.5, 0.1]); + } + + #[test] + fn overlay_truncates_a_track_that_runs_past_the_programme() { + let mut programme = planar(&[0.0, 0.0, 0.0]); + // A 4-sample track placed at offset 2 has room for only 1 sample. + overlay_track_pcm(&mut programme, &planar(&[1.0, 1.0, 1.0, 1.0]), 2, 1.0, 0.0, 0.0); + assert_eq!(programme[0], vec![0.0, 0.0, 1.0]); + } + + #[test] + fn overlay_past_the_end_is_a_no_op() { + let mut programme = planar(&[0.3, 0.3]); + overlay_track_pcm(&mut programme, &planar(&[1.0]), 5, 1.0, 0.0, 0.0); + assert_eq!(programme[0], vec![0.3, 0.3]); + } + + #[test] + fn mix_external_tracks_skips_empty_windows() { + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 0.0, + gain_db: 0.0, + trim_start_sec: 2.0, + trim_end_sec: Some(1.0), // end <= start: empty window, never decoded + fade_in_sec: 0.0, + fade_out_sec: 0.0, + }]; + // The empty window is skipped before any decode, so the programme is + // untouched even though the path does not exist. + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + + #[test] + fn mix_external_tracks_skips_a_track_that_starts_past_the_programme() { + // 2 samples = ~0.00004 s of programme at 48 kHz; the track starts at 1 s, so + // its offset is past the end. It must be skipped before any decode is + // attempted (the path does not exist), never buffering its window. + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 1.0, + gain_db: 0.0, + trim_start_sec: 0.0, + trim_end_sec: Some(3600.0), + fade_in_sec: 0.0, + fade_out_sec: 0.0, + }]; + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + #[test] fn single_track_is_not_clamped() { // Promesse de non-régression : une source mono-piste ressort telle quelle, y compris @@ -1784,6 +2016,73 @@ mod tests { assert!((loud[0][0] - ceiling).abs() < 1e-6); } + #[test] + fn fades_that_fit_are_left_alone() { + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let (fin, fout) = resolve_fade_samples(rate as usize, 0.1, 0.2); + assert_eq!(fin, (0.1 * rate).round() as usize); + assert_eq!(fout, (0.2 * rate).round() as usize); + } + + #[test] + fn fades_too_long_for_the_track_share_it_in_proportion() { + // 6 s + 4 s of fade on a 2 s track → 1.2 s / 0.8 s, not a clamped 1 s / 1 s. + // Mirrors `resolveFadeSecs` on the app side; the two must agree or the + // preview and the render shape the same track differently. + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let len = (2.0 * rate) as usize; + let (fin, fout) = resolve_fade_samples(len, 6.0, 4.0); + assert_eq!(fin, (1.2 * rate).round() as usize); + assert_eq!(fout, (0.8 * rate).round() as usize); + assert!(fin + fout <= len + 1); + } + + #[test] + fn a_fade_in_longer_than_the_track_still_reaches_full_volume() { + // Left unreduced this holds the gain near zero for the whole track — the + // layer exports silent. + let (fin, fout) = resolve_fade_samples(100, 10.0, 0.0); + assert_eq!((fin, fout), (100, 0)); + assert!((fade_envelope(99, 100, fin, fout) - 0.99).abs() < 1e-3); + } + + #[test] + fn the_envelope_ramps_at_both_edges_and_holds_between() { + assert_eq!(fade_envelope(0, 100, 10, 10), 0.0); + assert!((fade_envelope(5, 100, 10, 10) - 0.5).abs() < 1e-6); + assert_eq!(fade_envelope(50, 100, 10, 10), 1.0); + assert!((fade_envelope(95, 100, 10, 10) - 0.5).abs() < 1e-6); + } + + #[test] + fn overlay_applies_the_fade_over_the_decoded_length() { + // The ramps are measured against the DECODED length, not the room left in + // the programme: a fade-out timed to the programme's end would ramp down + // over audio the export never reaches. + let mut programme = planar(&[0.0, 0.0, 0.0, 0.0]); + let decoded = planar(&[1.0, 1.0, 1.0, 1.0]); + // A 4-sample fade-in at 48 kHz is far below one sample of real time, so + // ask for the whole decoded length in seconds. + let four = 4.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + overlay_track_pcm(&mut programme, &decoded, 0, 1.0, four, 0.0); + assert_eq!(programme[0][0], 0.0); + assert!(programme[0][1] > 0.0 && programme[0][1] < 1.0); + assert!(programme[0][3] > programme[0][1]); + } + + #[test] + fn a_track_gain_below_the_output_bound_is_honoured() { + // The per-track gain range is the inspector's -60..+12, NOT the project + // output trim's ±12: clamping here at -12 floored every quiet bed at a + // tenth of the attenuation asked for. + let mut programme = planar(&[0.0]); + let decoded = planar(&[1.0]); + let gain = 10.0f32.powf(-40.0 / 20.0); + overlay_track_pcm(&mut programme, &decoded, 0, gain, 0.0, 0.0); + assert!((programme[0][0] - gain).abs() < 1e-9); + assert!(programme[0][0] < 10.0f32.powf(-12.0 / 20.0)); + } + #[test] fn output_is_clipped_to_full_scale_and_keeps_its_length() { // The trim can push a hot signal past full scale; the timeline must come back the diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 235a32917..46dcbe290 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -21,7 +21,7 @@ use std::ffi::CString; use std::ptr; use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; @@ -52,6 +52,8 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Secondes de sortie tenues sur la dernière image, en silence (issue #560). + pub hold_sec: f64, } /// Codec cible. Memes variantes que `pipeline_macos::ExportCodec`. @@ -459,6 +461,12 @@ pub fn run_composited_multi( let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene so the + // mix step below owns them. Empty for a project with no imported audio. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour // la raison pour laquelle la preview, elle, reste a 1. @@ -552,7 +560,10 @@ pub fn run_composited_multi( let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index c7cf571bc..e0c4eec34 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -30,7 +30,7 @@ //! décodeurs, symétrique. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; @@ -529,6 +529,8 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Secondes de sortie tenues sur la dernière image, en silence (issue #560). + pub hold_sec: f64, } /// Codec cible pour l'export. Identique à `pipeline_windows::ExportCodec`. @@ -1078,6 +1080,11 @@ pub fn run_composited_multi( // raconte avoir déjà coûté une fois. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); frames = unsafe { crate::timeline_walk::walk_composited_timeline( clips, @@ -1146,7 +1153,10 @@ pub fn run_composited_multi( let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index 5fff82056..674b31984 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -3,7 +3,7 @@ //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; @@ -969,6 +969,8 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Secondes de sortie tenues sur la dernière image, en silence (issue #560). + pub hold_sec: f64, } /// Export **multiclip** : rend la timeline (clips ordonnés, avec trims) en un seul MP4. @@ -1344,6 +1346,11 @@ unsafe fn run_multi_inner( // fenêtrage par clip ; `walk_composited_timeline` s'en charge. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur @@ -1477,7 +1484,7 @@ unsafe fn run_multi_inner( out_fps as f64, ); let assembled_audio = finish_audio( - assemble_concatenated_pcm(&clip_pcm, &audio_plan), + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks), audio_settings, ); audio_encoder.encode(&assembled_audio, octx)?; diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index ff300d475..a53aba3cf 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -19,6 +19,11 @@ pub struct SceneClip { /// Une source sans piste audio décodable garde sa durée via du silence natif. #[serde(default)] pub has_audio: bool, + /// Secondes de sortie pendant lesquelles ce clip TIENT sa dernière image, en silence. + /// `#[serde(default)]` comme `has_audio` juste au-dessus, et pour la même raison : un + /// document écrit avant ce champ se charge sans lui, à 0.0. + #[serde(default)] + pub hold_sec: f64, } #[derive(Debug, Clone, Copy, Deserialize)] @@ -432,6 +437,39 @@ pub struct SceneAudio { pub gain_db: f32, } +/// One imported audio track (issue #350) mixed over the assembled programme — +/// voiceover / BGM / SFX. Deliberately a SEPARATE `Scene` field rather than a +/// member of `SceneAudio`, so `SceneAudio` stays `Copy` and the pipelines keep +/// copying it out of a borrow unchanged. +/// +/// `start_sec` is the track's head on the OUTPUT programme; `trim_start_sec` / +/// `trim_end_sec` window the source file (both source seconds). The renderer +/// resolves `start_sec` from the track's raw timeline position — equal to it when +/// the project has no trims/speed, which is the case this first cut mixes exactly. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneAudioTrack { + pub path: String, + #[serde(default)] + pub start_sec: f64, + #[serde(default)] + pub gain_db: f32, + #[serde(default)] + pub trim_start_sec: f64, + #[serde(default)] + pub trim_end_sec: Option, + /// Ramp lengths at this entry's own edges, in seconds. The app puts them only + /// on the pieces that touch the track's real start and end, so a split or + /// looping track fades once instead of at every cut or repeat. + /// + /// `#[serde(default)]` for the usual reason: a payload from a build that + /// predates the field must degrade to "no fade", not fail the whole scene. + #[serde(default)] + pub fade_in_sec: f64, + #[serde(default)] + pub fade_out_sec: f64, +} + #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneOutput { @@ -500,6 +538,10 @@ pub struct Scene { /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. #[serde(default)] pub audio: SceneAudio, + /// Imported audio tracks mixed over the programme (issue #350). `#[serde(default)]`: + /// absent from every scene written before this, and from a project with none. + #[serde(default)] + pub audio_tracks: Vec, /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, @@ -576,6 +618,26 @@ impl Scene { mod tests { use super::*; + /// Un document écrit avant `holdSec` doit se charger, à 0.0 — comme `hasAudio` avant + /// lui. Sans ce défaut, ouvrir un projet fait par une version antérieure échouerait au + /// parse au lieu de simplement ne rien tenir (issue #560). + #[test] + fn a_clip_without_hold_sec_deserializes_to_zero() { + let json = r##"{"screenPath":"/s.mp4","webcamPath":"/w.mp4","sourceStartSec":0,"sourceEndSec":4,"webcamOffsetSec":0,"hasAudio":true}"##; + let clip: SceneClip = serde_json::from_str(json).expect("clip sans holdSec"); + assert_eq!(clip.hold_sec, 0.0); + } + + #[test] + fn a_clip_carries_its_hold_when_the_document_states_one() { + let json = r##"{"screenPath":"/s.mp4","webcamPath":"/w.mp4","sourceStartSec":2,"sourceEndSec":2,"webcamOffsetSec":0,"hasAudio":true,"holdSec":1.5}"##; + let clip: SceneClip = serde_json::from_str(json).expect("clip tenu"); + assert_eq!(clip.hold_sec, 1.5); + // Fenêtre source vide ET pause positive : c'est exactement la forme que la marche + // laisse désormais passer. + assert!(clip.source_end_sec <= clip.source_start_sec); + } + #[test] fn parses_a_minimal_scene_json() { let json = r##"{ diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index 5d95a81db..92cee3b76 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -17,6 +17,7 @@ use crate::compositor::Compositor; use crate::config::Cfg; use crate::cursor::CursorTrack; use crate::d3d::Gpu; +use crate::ffi::AVFrame; use crate::frame_geometry::webcam_is_real; use crate::pipeline::{ClipSource, Decoder}; use crate::regions::{speed_segments_for_window, SpeedSegment}; @@ -247,7 +248,11 @@ pub(crate) unsafe fn walk_composited_timeline( clip.webcam, ); } - if source_end_sec <= clip.source_start_sec { + // Une fenêtre source vide ne veut plus dire « rien à faire » : un segment TENU + // (issue #560) n'a par construction aucune source à décoder et n'existe que pour + // ses images tenues. Sans cette porte, une pause achetée par un mot ajouté était + // honorée par le ruler et par la preview et n'exportait rien du tout. + if source_end_sec <= clip.source_start_sec && clip.hold_sec <= 0.0 { continue; } @@ -301,6 +306,10 @@ pub(crate) unsafe fn walk_composited_timeline( } let frames_before_clip = frames; + // La dernière paire composée, gardée hors de la boucle : `'clip_frames` peut casser, + // et un clip qui n'existe QUE pour tenir une image n'entre jamais dedans — il faut + // pourtant une image à tenir dans les deux cas. + let mut last_pair: Option<(*mut AVFrame, *mut AVFrame)> = None; 'clip_frames: for segment in &speed_segments { for segment_frame in 0..segment.frame_count { let target_source_time = @@ -322,11 +331,53 @@ pub(crate) unsafe fn walk_composited_timeline( comp.set_cursor_time(Some(target_source_time as f32)); } comp.compose_frame(sf, wf, frames as f32, cfg)?; + last_pair = Some((sf, wf)); on_frame(frames)?; frames += 1; } } + + // Les images tenues, APRÈS les segments de vitesse et volontairement en dehors + // d'eux. Dedans, `stretch_pcm_to_length` étirerait le vrai son du clip à travers le + // gel (WSOLA sur une voix, audible). Dehors, le créneau audio est plus long que le + // PCM et `min(source.len())` laisse des zéros : le silence est gratuit. + // + // `ceil`, pas `round` : arrondir perd jusqu'à une demi-image, prise sur la fin de la + // narration même que la pause existe pour loger. + if clip.hold_sec > 0.0 { + if last_pair.is_none() { + // Clip tenu seul : rien n'a été composé, alors on va chercher une image une + // fois. Un seek qui échoue laisse `last_pair` vide et on n'émet rien plutôt + // que de composer du vide. + if advance_decoder_to(sdec, clip.source_start_sec, 0.0)? + && advance_decoder_to(wdec, clip.source_start_sec, clip.webcam_offset_sec)? + { + let sf = sdec.cur_frame(); + let wf = wdec.cur_frame(); + if !sf.is_null() && !wf.is_null() { + last_pair = Some((sf, wf)); + } + } + } + match last_pair { + Some((sf, wf)) => { + let held = ((clip.hold_sec * out_fps as f64).ceil()) as u64; + // Le temps timeline est ÉPINGLÉ sur l'instant tenu : c'est ce qui fige + // aussi les modificateurs (zoom, curseur) au lieu de les laisser courir. + comp.set_timeline_time(Some(clip.source_start_sec as f32)); + for _ in 0..held { + comp.compose_frame(sf, wf, frames as f32, cfg)?; + on_frame(frames)?; + frames += 1; + } + } + None => eprintln!( + "[pipeline] warning: clip #{}: pause de {:.3}s ignorée, aucune image à tenir (screen=\"{}\")", + clip_index, clip.hold_sec, clip.screen, + ), + } + } on_clip_end( clip_index, source_end_sec, diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 62361572d..22ea8bc3f 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -792,6 +792,7 @@ fn export_linux_mp4() { source_end_sec: 1.0, webcam_offset_sec: 0.0, has_audio: true, + hold_sec: 0.0, }]; let params = ExportParams { width: 640, diff --git a/crates/compositor/tests/export_timing.rs b/crates/compositor/tests/export_timing.rs index f0070794b..11a100626 100644 --- a/crates/compositor/tests/export_timing.rs +++ b/crates/compositor/tests/export_timing.rs @@ -117,6 +117,7 @@ fn whole_clip(dir: &PathBuf) -> ClipSource { source_end_sec: SOURCE_SEC, webcam_offset_sec: 0.0, has_audio: false, + hold_sec: 0.0, } } diff --git a/crates/poc-d3d/src/bench.rs b/crates/poc-d3d/src/bench.rs index aa2de7d60..34460d9aa 100644 --- a/crates/poc-d3d/src/bench.rs +++ b/crates/poc-d3d/src/bench.rs @@ -150,6 +150,7 @@ fn run_bench(args: &[String]) -> Result<()> { source_end_sec: 6.0, // la fixture entière (§ fixture.json : 6 s, 360 frames) webcam_offset_sec: 0.0, has_audio: false, + hold_sec: 0.0, }; let path = format!("{out}/{}_{:?}.mp4", cfg.name, backend).to_lowercase(); let s = pipeline::run_composited_multi( @@ -292,6 +293,7 @@ fn run_gif_bench( source_end_sec: f64::MAX, webcam_offset_sec: 0.0, has_audio: false, + hold_sec: 0.0, }]; for r in 0..repeat { // Each run writes to the same path — the last frame wins. The diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 8f8361e59..f39510487 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -172,6 +172,7 @@ describe("the mutating-tool table", () => { expect([...MUTATING_TOOL_NAMES].sort()).toEqual( [ "addAnnotation", + "addAudio", "addCameraFullscreen", "addSpeed", "addTrim", @@ -184,10 +185,12 @@ describe("the mutating-tool table", () => { "removeTrim", "replaceTimeline", "setAnnotation", + "setAudio", "setCameraFullscreen", "setClipRange", "setSpeed", "setTrim", + "setWordText", "setZoom", ].sort(), ); @@ -2054,3 +2057,289 @@ describe("setZoom answers for the focus it kept", () => { expect(result.resultJson).not.toContain("cursorAnchor"); }); }); + +// Issue #350 / #560 — the audio tools. #561 landed the timeline audio without any +// agent surface, so these cover both that the model can see it and that it cannot +// invent an asset to place. +describe("addAudio / setAudio", () => { + /** The fixture plus one imported audio asset. */ + function withAudioAsset(durationSec: number | null = 30): AxcutDocument { + const doc = fixtureDocument(); + return documentSchema.parse({ + ...doc, + assets: [ + ...doc.assets, + { + id: "audio_1", + kind: "audio", + label: "bed.mp3", + originalPath: "C:/audio/bed.mp3", + ...(durationSec == null ? {} : { durationSec }), + }, + ], + }); + } + + const place = (doc: AxcutDocument, args: Record) => + executeAgentTool(doc, "addAudio", JSON.stringify(args)); + + it("reports imported audio in the snapshot, with the asset kind beside it", () => { + // Without `kind` the model sees an asset it cannot explain and tries to place it + // as footage; without `audioTracks` it cannot see the lanes at all. + const placed = place(withAudioAsset(), { + assetId: "audio_1", + startSec: 2, + endSec: 6, + kind: "voiceover", + }); + expect(placed.ok).toBe(true); + const snapshot = executeAgentTool(placed.document as AxcutDocument, "getCurrentDocument", ""); + const parsed = JSON.parse(snapshot.resultJson); + expect(parsed.assets.find((a: { id: string }) => a.id === "audio_1").kind).toBe("audio"); + expect(parsed.audioTracks).toHaveLength(1); + expect(parsed.audioTracks[0]).toMatchObject({ + assetId: "audio_1", + kind: "voiceover", + startSec: 2, + endSec: 6, + }); + }); + + it("anchors the placed track to the clip under it", () => { + const result = place(withAudioAsset(), { assetId: "audio_1", startSec: 2, endSec: 6 }); + expect(result.ok).toBe(true); + const track = (result.document as AxcutDocument).audioTracks[0]; + // The anchor is what makes it travel with its clip; a bare startMs/endMs would not. + expect(track.clipId).toBe("clip_1"); + expect(track.origin).toBe("agent"); + }); + + it("plays the whole file when endSec is omitted", () => { + const result = place(withAudioAsset(20), { assetId: "audio_1", startSec: 0, offsetSec: 5 }); + expect(result.ok).toBe(true); + const track = (result.document as AxcutDocument).audioTracks[0]; + // 20s file from an in-point of 5s = 15s of span, so the model never computes it. + expect(track.endMs - track.startMs).toBe(15_000); + }); + + it("refuses an offset at or past the end of a known file", () => { + // Otherwise the omitted-end fallback mints a 0.1s track that plays silence, and + // the model reports it as having placed audio. + const result = place(withAudioAsset(20), { assetId: "audio_1", startSec: 0, offsetSec: 20 }); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("offsetSec"); + }); + + it("allows any offset while the duration is unknown", () => { + // A failed probe leaves no duration; refusing on that would block a legitimate call. + expect(place(withAudioAsset(null), { assetId: "audio_1", startSec: 0, offsetSec: 99 }).ok).toBe( + true, + ); + }); + + it("refuses an unknown asset and names the audio the project actually has", () => { + const result = place(withAudioAsset(), { assetId: "nope", startSec: 0, endSec: 4 }); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("audio_1"); + }); + + it("refuses a video asset, pointing at the tool that does place footage", () => { + const result = place(withAudioAsset(), { assetId: "asset_1", startSec: 0, endSec: 4 }); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("replaceTimeline"); + }); + + it("setAudio re-levels and re-lanes the track it names", () => { + const placed = place(withAudioAsset(), { assetId: "audio_1", startSec: 2, endSec: 6 }); + const id = JSON.parse(placed.resultJson).audioId; + const result = executeAgentTool( + placed.document as AxcutDocument, + "setAudio", + JSON.stringify({ audioId: id, gainDb: -6, kind: "voiceover" }), + ); + expect(result.ok).toBe(true); + expect((result.document as AxcutDocument).audioTracks[0]).toMatchObject({ + gainDb: -6, + kind: "voiceover", + }); + }); + + it("setAudio applies the same offset guard as addAudio", () => { + const placed = place(withAudioAsset(20), { assetId: "audio_1", startSec: 2, endSec: 6 }); + const id = JSON.parse(placed.resultJson).audioId; + const result = executeAgentTool( + placed.document as AxcutDocument, + "setAudio", + JSON.stringify({ audioId: id, offsetSec: 25 }), + ); + expect(result.ok).toBe(false); + }); + + it("removeModifier deletes an audio track by id, like every other kind", () => { + const placed = place(withAudioAsset(), { assetId: "audio_1", startSec: 2, endSec: 6 }); + const id = JSON.parse(placed.resultJson).audioId; + const result = executeAgentTool( + placed.document as AxcutDocument, + "removeModifier", + JSON.stringify({ id }), + ); + expect(result.ok).toBe(true); + expect((result.document as AxcutDocument).audioTracks).toEqual([]); + }); +}); + +// ─── Correcting a word from the chat ───────────────────────────── +// The model could READ the transcript and CUT it, and that was all. Asked to fix a +// misheard name it had exactly one tool that touched a word — addTrim — which removes the +// audio with it. These two close that: one read that hands out word ids, one write that +// changes text and nothing else. + +/** A transcript with real words, one of them already corrected by the user. */ +function documentWithWords(): AxcutDocument { + const base = fixtureDocument(); + return { + ...base, + transcripts: [ + { + assetId: "asset_1", + language: "en", + segments: [ + { + id: "seg_1", + kind: "speech", + startSec: 0, + endSec: 3, + text: "I use Cuber Nettes", + wordIds: ["word_1", "word_2", "word_3"], + }, + ], + words: [ + { id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 1, text: "I" }, + { id: "word_2", segmentId: "seg_1", startSec: 1, endSec: 2, text: "use" }, + { + id: "word_3", + segmentId: "seg_1", + startSec: 2, + endSec: 3, + text: "Cuber Nettes", + }, + ], + }, + ], + }; +} + +function run(document: AxcutDocument, name: string, args: unknown) { + return executeAgentTool(document, name, JSON.stringify(args), { editsAllowed: true }); +} + +describe("getTranscriptWords", () => { + it("hands out the ids setWordText takes", () => { + const result = run(documentWithWords(), "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string; text: string }>; + total: number; + }; + expect(result.ok).toBe(true); + expect(payload.total).toBe(3); + expect(payload.words.map((w) => w.id)).toEqual(["word_1", "word_2", "word_3"]); + }); + + // A half-hour transcript is ~70k tokens. Fixing one name should cost one phrase. + it("returns only the words touching the span it is given", () => { + const result = run(documentWithWords(), "getTranscriptWords", { startSec: 2, endSec: 3 }); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string }>; + total: number; + }; + // Touching counts: `word_2` ends exactly where the span begins. Inclusive on + // purpose — a word with no duration at all (one the user typed in) sits on a + // single point, and a strict overlap would drop it from every span it meets. + expect(payload.words.map((w) => w.id)).toEqual(["word_2", "word_3"]); + // `total` still reports the whole transcript, so a filtered read never reads as + // the entire thing. + expect(payload.total).toBe(3); + }); + + it("says nothing about provenance for a plainly transcribed word", () => { + const result = run(documentWithWords(), "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { words: Array> }; + expect(payload.words[0]).not.toHaveProperty("source"); + expect(payload.words[0]).not.toHaveProperty("originalText"); + }); + + it("names what the transcriber had heard, once a word is corrected", () => { + const corrected = run(documentWithWords(), "setWordText", { + wordId: "word_3", + text: "Kubernetes", + }); + const result = run(corrected.document as AxcutDocument, "getTranscriptWords", {}); + const payload = JSON.parse(result.resultJson) as { + words: Array<{ id: string; source?: string; originalText?: string }>; + }; + expect(payload.words.find((w) => w.id === "word_3")).toMatchObject({ + source: "user", + originalText: "Cuber Nettes", + }); + }); + + it("refuses an asset with no transcript instead of answering with nothing", () => { + const result = run({ ...fixtureDocument(), transcripts: [] }, "getTranscriptWords", {}); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("No transcript"); + }); +}); + +describe("setWordText", () => { + it("changes the text and leaves the timeline alone", () => { + const before = documentWithWords(); + const result = run(before, "setWordText", { wordId: "word_3", text: "Kubernetes" }); + expect(result.ok).toBe(true); + const next = result.document as AxcutDocument; + expect(next.transcripts[0].words.find((w) => w.id === "word_3")?.text).toBe("Kubernetes"); + expect(next.timeline).toEqual(before.timeline); + expect(next.transcripts[0].segments[0].text).toBe("I use Kubernetes"); + }); + + // The document carries the transcript twice; a write that reaches only one leaves the + // legacy mirror serving the old text forever. + it("writes the legacy mirror too", () => { + const result = run(documentWithWords(), "setWordText", { + wordId: "word_3", + text: "Kubernetes", + }); + const next = result.document as AxcutDocument; + expect(next.transcript).toBe(next.transcripts.find((t) => t.assetId === "asset_1")); + }); + + it("empties a word without cutting the speech around it", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "" }); + const next = result.document as AxcutDocument; + expect(next.transcripts[0].words.find((w) => w.id === "word_2")?.text).toBe(""); + expect(next.transcripts[0].segments[0].text).toBe("I Cuber Nettes"); + expect(JSON.parse(result.resultJson)).toMatchObject({ blanked: true }); + }); + + it("points an unknown id at the read that hands them out", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "seg_1", text: "x" }); + expect(result.ok).toBe(false); + // `seg_1` is a real id — of a SEGMENT. The two namespaces are the trap. + expect(result.resultJson).toContain("getTranscriptWords"); + }); + + it("refuses a write that would change nothing", () => { + const result = run(documentWithWords(), "setWordText", { wordId: "word_2", text: "use" }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + }); + + it("is a consented edit, not a read", () => { + const result = executeAgentTool( + documentWithWords(), + "setWordText", + JSON.stringify({ wordId: "word_3", text: "Kubernetes" }), + { editsAllowed: false }, + ); + expect(result.document).toBeUndefined(); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 5a0966398..469b5d313 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -16,6 +16,12 @@ // described are gone (see `MUTATING_TOOL_NAMES`). import { z } from "zod"; +import { + collapseTracksToPills, + patchAudioTrack, + placeAudioTrackInDocument, + trackGroupId, +} from "../../src/lib/ai-edition/document/audioTracks"; import { createId } from "../../src/lib/ai-edition/document/ids"; import { moveClip, @@ -26,6 +32,7 @@ import { replaceTimeline, setClipSourceRange, } from "../../src/lib/ai-edition/document/timeline"; +import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript"; import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; import { @@ -337,6 +344,11 @@ function droppedByEdit(before: AxcutDocument, after: AxcutDocument) { // private — callers only ever need the composed `*Args`.) const secondsSchema = z.number().finite().nonnegative(); +/** Span given to an agent-placed audio track when the asset has no probed duration + * yet. Short on purpose: a wrong guess the user has to lengthen beats one that + * silently covers the whole programme. */ +const DEFAULT_AGENT_AUDIO_SEC = 10; + export const addTrimArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, @@ -479,6 +491,26 @@ export const setAnnotationArgs = z.object({ text: z.string().optional(), }); +export const addAudioArgs = z.object({ + assetId: z.string().min(1), + startSec: secondsSchema, + endSec: secondsSchema.optional(), + kind: z.enum(["voiceover", "music"]).default("music"), + offsetSec: secondsSchema.default(0), + gainDb: z.number().min(-60).max(12).default(0), +}); + +export const setAudioArgs = z.object({ + audioId: z.string().min(1), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), + kind: z.enum(["voiceover", "music"]).optional(), + offsetSec: secondsSchema.optional(), + gainDb: z.number().min(-60).max(12).optional(), + muted: z.boolean().optional(), + loop: z.boolean().optional(), +}); + export const addCameraFullscreenArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, @@ -490,6 +522,18 @@ export const setCameraFullscreenArgs = z.object({ endSec: secondsSchema.optional(), }); +export const getTranscriptWordsArgs = z.object({ + assetId: z.string().min(1).optional(), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), +}); + +export const setWordTextArgs = z.object({ + wordId: z.string().min(1), + text: z.string(), + assetId: z.string().min(1).optional(), +}); + export const removeTrimArgs = z.object({ trimRangeId: z.string().min(1), }); @@ -525,7 +569,9 @@ export const removeClipArgs = z.object({ export const OPENSCREEN_TOOL_NAMES = [ "getCurrentDocument", "getTranscript", + "getTranscriptWords", "getCursorTrack", + "setWordText", "addTrim", "addTrims", "setTrim", @@ -541,6 +587,8 @@ export const OPENSCREEN_TOOL_NAMES = [ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addAudio", + "setAudio", "removeTrim", "removeModifier", "removeClip", @@ -592,6 +640,9 @@ export const PHANTOM_TOOL_NAMES = [ * remaining surfaces (descriptions, built tools, executor cases) to each other. */ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ + // Writes the transcript, not the timeline — but it writes the document, so it is a + // consented edit like any other. + "setWordText", "addTrim", "addTrims", "addZooms", @@ -607,6 +658,8 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addAudio", + "setAudio", "removeTrim", "removeModifier", "removeClip", @@ -665,7 +718,9 @@ export function documentSnapshotForModel( const autoFocusAll = legacy?.autoFocusAll === true; return { timeBaseNote: - "clips and trims are in source-time seconds; zooms, speedRegions, annotations and cameraFullscreenRegions are in virtual (edited-timeline) seconds.", + "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds.", + audioNote: + "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.", zoomNote: `renderedScale is what the viewer sees (depth is an ordinal, not a factor: ${ZOOM_DEPTH_LEGEND}). ` + "When a zoom carries customScale it wins over depth and depthIsOverridden is true — " + @@ -683,6 +738,10 @@ export function documentSnapshotForModel( assets: document.assets.map((a) => ({ id: a.id, label: a.label, + // "audio" is an imported voiceover / music file: it is never a clip, it is + // played by an audio track. Without this the model sees an asset it cannot + // explain and tries to place it on the timeline as footage. + kind: a.kind, durationSec: a.durationSec ?? null, hasCameraTrack: a.cameraTrack != null, cameraVisible: a.cameraTrack?.visible ?? false, @@ -755,6 +814,22 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + // Imported audio, collapsed to the pills the ruler draws — a track ventilated + // across a clip boundary is several fragments the user sees as one thing, and + // the model has to name what the user sees. + audioTracks: collapseTracksToPills(document.audioTracks).map((t) => ({ + id: trackGroupId(t), + startSec: roundSec(t.startMs), + endSec: roundSec(t.endMs), + assetId: t.assetId, + // Which lane it sits on. Also decides whether it is transcribed at all. + kind: t.kind, + // Where in the FILE the track starts playing, in that file's own seconds. + offsetSec: roundSec(t.offsetMs), + gainDb: t.gainDb, + muted: t.muted, + loop: t.loop, + })), hasTranscript: document.transcripts.length > 0 || document.transcript !== null, }; } @@ -1203,6 +1278,96 @@ export function executeAgentTool( }; } + // The word-level read. `getTranscript` answers in SEGMENTS, whose ids belong to a + // different namespace than the words — so on its own it cannot address anything + // `setWordText` takes. This is the one that can. It is separate rather than folded + // in because a whole transcript is already ~70k tokens and most turns never touch a + // word; the span filter is there so fixing one name costs one phrase, not the film. + case "getTranscriptWords": { + const parsed = getTranscriptWordsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const assetId = + parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id; + const transcript = + document.transcripts.find((t) => t.assetId === assetId) ?? + (document.transcript?.assetId === assetId ? document.transcript : null); + if (!transcript) { + return failure(`No transcript for asset ${assetId ?? "(none)"}.`); + } + const from = parsed.data.startSec ?? Number.NEGATIVE_INFINITY; + const to = parsed.data.endSec ?? Number.POSITIVE_INFINITY; + const words = transcript.words + .filter((word) => word.endSec >= from && word.startSec <= to) + .map((word) => ({ + id: word.id, + text: word.text, + startSec: word.startSec, + endSec: word.endSec, + // Only the words that are NOT plain transcription say so, so the common + // case costs nothing to read. + ...(word.source ? { source: word.source } : {}), + ...(word.originalText !== undefined ? { originalText: word.originalText } : {}), + })); + return { + ok: true, + resultJson: JSON.stringify({ + assetId, + language: transcript.language, + total: transcript.words.length, + returned: words.length, + words, + }), + }; + } + + // Correcting what the transcriber HEARD. This writes text and nothing else: the + // captions follow it, the film does not move. The tool for making a spoken word go + // away is addTrim, which removes its audio with it. + case "setWordText": { + const parsed = setWordTextArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const assetId = + parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id; + if (!assetId) return failure("Project has no assets — nothing to correct."); + const { wordId, text } = parsed.data; + const transcript = document.transcripts.find((t) => t.assetId === assetId); + const before = transcript?.words.find((word) => word.id === wordId); + if (!before) { + return failure( + `No word ${wordId} in the transcript for asset ${assetId}. ` + + `Call getTranscriptWords to read the ids.`, + ); + } + if (before.text === text) { + return failure(`Word ${wordId} already reads "${text}" — nothing to change.`); + } + let next: AxcutDocument; + try { + next = setDocumentWordText(document, assetId, wordId, text); + } catch (error) { + return failure(error instanceof Error ? error.message : String(error)); + } + const after = next.transcripts + .find((t) => t.assetId === assetId) + ?.words.find((word) => word.id === wordId); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + wordId, + assetId, + text: after?.text ?? text, + was: before.text, + // Absent once the word is back to what the transcriber said — the pair is + // cleared on that round trip, and the model should be able to see it. + originalText: after?.originalText, + blanked: text.trim().length === 0, + }), + summary: + text.trim().length === 0 ? `blanked "${before.text}"` : `"${before.text}" → "${text}"`, + }; + } + case "addTrim": { const parsed = addTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -1843,6 +2008,165 @@ export function executeAgentTool( }; } + case "addAudio": { + const parsed = addAudioArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { assetId, kind, offsetSec, gainDb } = parsed.data; + const asset = document.assets.find((a) => a.id === assetId); + // Two distinct refusals, because they need two different corrections: an + // unknown id is a hallucinated asset, a video id is the model reaching for + // footage. Naming the audio the project HAS is what stops the retry loop. + if (!asset) { + const available = document.assets.filter((a) => a.kind === "audio"); + return failure( + `Unknown asset: ${assetId}.` + + (available.length + ? ` Imported audio in this project: ${available.map((a) => `${a.id} (${a.label})`).join(", ")}.` + : " This project has no imported audio; a file can only be imported or recorded from the editor, not from here."), + ); + } + if (asset.kind !== "audio") { + return failure( + `Asset ${assetId} is video, not audio. addAudio plays an imported audio file over the recording; to place footage use replaceTimeline.`, + ); + } + const durationSec = asset.durationSec ?? 0; + // "Start the file at offsetSec" is only answerable when there is file left + // there. Past the end it yields a track that plays silence, which the model + // then reports as having placed audio. Unknown duration is not a refusal: an + // import whose probe failed carries 0 until the renderer re-probes it. + if (durationSec > 0 && offsetSec >= durationSec) { + return failure( + `offsetSec ${offsetSec}s is at or past the end of ${assetId} (${durationSec}s), so the track would play nothing. Pick an offset inside the file.`, + ); + } + // No endSec means "as long as the file is" — the natural span, and the one + // the editor's own add uses, so the model never has to compute it. + const startSec = parsed.data.startSec; + const endSec = + parsed.data.endSec ?? + startSec + Math.max(0.1, (durationSec || DEFAULT_AGENT_AUDIO_SEC) - offsetSec); + const startMs = toMs(Math.min(startSec, endSec)); + const endMs = toMs(Math.max(startSec, endSec)); + const trackId = createId("audio"); + const withTrack = placeAudioTrackInDocument( + document, + { + id: trackId, + trackId, + startMs, + endMs, + assetId, + kind, + durationSec, + offsetMs: toMs(offsetSec), + gainDb, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: asset.label, + origin: "agent", + } as AxcutDocument["audioTracks"][number], + () => createId("audio"), + "create", + ); + if (withTrack === document) { + return coversNoClip("audio", startMs / 1000, endMs / 1000, document); + } + const placed = withTrack.audioTracks.filter((t) => trackGroupId(t) === trackId); + const next: AxcutDocument = withTrack; + const landing = landingOf(placed, document); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + audioId: trackId, + ...landingReport(landing, startMs / 1000, endMs / 1000), + }), + summary: + `added ${kind} "${asset.label}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + + case "setAudio": { + const parsed = setAudioArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { audioId } = parsed.data; + const pill = collapseTracksToPills(document.audioTracks).find( + (t) => trackGroupId(t) === audioId, + ); + if (!pill) return failure(`Unknown audio track: ${audioId}`); + + if (parsed.data.offsetSec !== undefined) { + const asset = document.assets.find((a) => a.id === pill.assetId); + const durationSec = asset?.durationSec ?? 0; + if (durationSec > 0 && parsed.data.offsetSec >= durationSec) { + return failure( + `offsetSec ${parsed.data.offsetSec}s is at or past the end of ${pill.assetId} (${durationSec}s), so the track would play nothing.`, + ); + } + } + + // Payload first, through the helper that keeps every fragment of the group in + // agreement — gain, mute, loop and the offset are all track-wide, and a patch + // that reached only one fragment would split the pill in two. + let next = patchAudioTrack(document, audioId, { + ...(parsed.data.gainDb !== undefined ? { gainDb: parsed.data.gainDb } : {}), + ...(parsed.data.muted !== undefined ? { muted: parsed.data.muted } : {}), + ...(parsed.data.loop !== undefined ? { loop: parsed.data.loop } : {}), + ...(parsed.data.offsetSec !== undefined ? { offsetMs: toMs(parsed.data.offsetSec) } : {}), + }); + + // A span or lane change re-anchors: drop the group and lay it down again, so + // the fragments are re-cut against the clips the new span covers rather than + // patched in place against the old ones. + const wantsRespan = + parsed.data.startSec !== undefined || + parsed.data.endSec !== undefined || + parsed.data.kind !== undefined; + if (wantsRespan) { + const current = + collapseTracksToPills(next.audioTracks).find((t) => trackGroupId(t) === audioId) ?? pill; + const { startMs, endMs } = resolveSpanMs(current, parsed.data.startSec, parsed.data.endSec); + // A `kind` flip re-clamps against the DESTINATION lane's neighbours, not the + // one it is leaving — moving a take onto the music row must respect what is + // already on the music row (issue #560). + const moved = placeAudioTrackInDocument( + next, + { + ...current, + id: audioId, + trackId: audioId, + startMs, + endMs, + ...(parsed.data.kind !== undefined ? { kind: parsed.data.kind } : {}), + }, + () => createId("audio"), + "move", + ); + if (moved === next) { + return coversNoClip("audio", startMs / 1000, endMs / 1000, document); + } + next = moved; + } + + const after = collapseTracksToPills(next.audioTracks).find( + (t) => trackGroupId(t) === audioId, + ); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + audioId, + startSec: roundSec(after?.startMs ?? pill.startMs), + endSec: roundSec(after?.endMs ?? pill.endMs), + }), + summary: `updated audio ${audioId} ${formatSec(roundSec(after?.startMs ?? pill.startMs))} – ${formatSec(roundSec(after?.endMs ?? pill.endMs))}`, + }; + } + case "removeTrim": { const parsed = removeTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -1872,6 +2196,7 @@ export function executeAgentTool( else if (document.annotations.some((a) => a.id === id)) kind = "annotation"; else if (speedRegions.some((s) => s.id === id)) kind = "speed"; else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen"; + else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio"; if (!kind) { return failure( `No zoom / speed / annotation / full-camera modifier with id ${id}. ` + diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 7729bd624..5aebb9a33 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -57,7 +57,9 @@ const PHANTOM_TOOLS: readonly string[] = PHANTOM_TOOL_NAMES; const ARGS: Record = { getCurrentDocument: {}, getTranscript: {}, + getTranscriptWords: {}, getCursorTrack: {}, + setWordText: { wordId: "word_1", text: "Hullo" }, addTrim: { startSec: 1, endSec: 2 }, addTrims: { ranges: [{ startSec: 1, endSec: 2 }] }, setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 }, @@ -73,6 +75,10 @@ const ARGS: Record = { setAnnotation: { annotationId: "ann_nope" }, addCameraFullscreen: { startSec: 1, endSec: 2 }, setCameraFullscreen: { cameraFullscreenId: "cam_nope" }, + // The fixture has no `kind: "audio"` asset, so these exercise the refusal branch — + // the honest one to pin: the agent can place imported audio, never import it. + addAudio: { assetId: "audio_nope", startSec: 1, endSec: 2 }, + setAudio: { audioId: "audio_nope" }, removeTrim: { trimRangeId: "trim_1" }, removeModifier: { id: "nope" }, removeClip: { clipId: "clip_1" }, @@ -109,9 +115,18 @@ function fixtureDocument(): AxcutDocument { assetId: "asset_1", language: "en", segments: [ - { id: "seg_1", kind: "speech", startSec: 0, endSec: 5, text: "Hello", wordIds: [] }, + { + id: "seg_1", + kind: "speech", + startSec: 0, + endSec: 5, + text: "Hello", + // A real word, so `setWordText` lands on its WRITE branch in the table + // below — a tool refused for an unknown id would look non-mutating. + wordIds: ["word_1"], + }, ], - words: [], + words: [{ id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 5, text: "Hello" }], }, ], timeline: { @@ -351,6 +366,7 @@ describe("one description of the tools, not two", () => { expect(OPENSCREEN_TOOLS.filter((n) => !isMutatingTool(n))).toEqual([ "getCurrentDocument", "getTranscript", + "getTranscriptWords", "getCursorTrack", ]); }); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index d804a3b1a..7ddc55bfc 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -27,6 +27,7 @@ import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { addAnnotationArgs, + addAudioArgs, addCameraFullscreenArgs, addSpeedArgs, addTrimArgs, @@ -37,6 +38,7 @@ import { executeAgentTool, getCursorTrackArgs, getTranscriptArgs, + getTranscriptWordsArgs, isMutatingTool, moveClipArgs, removeClipArgs, @@ -45,10 +47,12 @@ import { replaceTimelineArgs, resolveCursorAssetId, setAnnotationArgs, + setAudioArgs, setCameraFullscreenArgs, setClipRangeArgs, setSpeedArgs, setTrimArgs, + setWordTextArgs, setZoomArgs, } from "../agent-tools"; import { @@ -115,6 +119,7 @@ const BASE_SYSTEM_PROMPT = [ "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, + "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", "- Deleting is a first-class action, not a workaround: removeTrim, removeModifier, removeClip. Never fake a deletion by re-adding an element or zeroing it out (span 0, speed 1×) — that leaves it in the document and misreports what you did.", "If nothing in the list does what was asked, say so; do not approximate it with a bigger tool.", @@ -143,6 +148,10 @@ export const TOOL_DESCRIPTIONS: Record = { "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", + getTranscriptWords: + 'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.', + setWordText: + "Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use addTrim, which cuts the audio with it.", addTrim: "Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).", addTrims: @@ -170,10 +179,14 @@ export const TOOL_DESCRIPTIONS: Record = { "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", + addAudio: + "Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.", + setAudio: + "Move, resize, re-level, re-lane, mute, loop or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.", removeTrim: "Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.", removeModifier: - "Delete a modifier (zoom / speed / annotation / camera-fullscreen) by id; the kind is resolved from the id. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", + "Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio) by id; the kind is resolved from the id. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", removeClip: "Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.", }; @@ -323,7 +336,9 @@ export function buildTools( return [ build("getCurrentDocument", z.object({})), build("getTranscript", getTranscriptArgs), + build("getTranscriptWords", getTranscriptWordsArgs), build("getCursorTrack", getCursorTrackArgs), + build("setWordText", setWordTextArgs), build("addTrim", addTrimArgs), build("addTrims", addTrimsArgs), build("setTrim", setTrimArgs), @@ -339,6 +354,8 @@ export function buildTools( build("setAnnotation", setAnnotationArgs), build("addCameraFullscreen", addCameraFullscreenArgs), build("setCameraFullscreen", setCameraFullscreenArgs), + build("addAudio", addAudioArgs), + build("setAudio", setAudioArgs), build("removeTrim", removeTrimArgs), build("removeModifier", removeModifierArgs), build("removeClip", removeClipArgs), diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 6cbdb97c9..bf3d12289 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -280,6 +280,63 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(first.project.primaryAssetId); expect(after.assets).toHaveLength(2); }); + + // Issue #350 — external audio import (voiceover / BGM / SFX). + it("appends an audio asset without claiming the primary slot", async () => { + const doc = await service.createProject("P"); + const updated = await service.addAsset(doc.project.id, { + path: "/tmp/voiceover.mp3", + kind: "audio", + }); + expect(updated.assets).toHaveLength(1); + expect(updated.assets[0]?.kind).toBe("audio"); + // An audio-only file must never become the project's primary asset, even + // when it is the first file added to an otherwise-empty project. + expect(updated.project.primaryAssetId).toBeUndefined(); + }); + + it("keeps the existing video primary when an audio track is added", async () => { + const doc = await service.createProject("P"); + const withVideo = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const primary = withVideo.project.primaryAssetId; + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/bgm.wav", + kind: "audio", + }); + expect(withAudio.project.primaryAssetId).toBe(primary); + expect(withAudio.assets).toHaveLength(2); + }); + + it("rejects unsupported audio extensions", async () => { + const doc = await service.createProject("P"); + await expect( + service.addAsset(doc.project.id, { path: "/tmp/clip.mp4", kind: "audio" }), + ).rejects.toBeInstanceOf(ProjectFileError); + }); + + it("accepts a recorded .webm take as audio", async () => { + // MediaRecorder writes a voiceover as webm/opus — the same extension a + // screen recording uses. The caller has already declared the kind here, + // so this gate must take it; only the import PICKER, which has nothing + // but the extension to go on, still refuses .webm as audio. + const doc = await service.createProject("P"); + const next = await service.addAsset(doc.project.id, { + path: "/tmp/voiceover-2026.webm", + kind: "audio", + }); + expect(next.assets.at(-1)).toMatchObject({ kind: "audio" }); + // ...and it must not have claimed the primary (video-only) slot. + expect(next.project.primaryAssetId).toBeUndefined(); + }); + + it("accepts a video extension under the default kind but not as audio", async () => { + const doc = await service.createProject("P"); + // The same extension routing works in reverse: an .mp3 is fine as audio + // but rejected as video (covered above), and an .mp4 is the opposite. + await expect( + service.addAsset(doc.project.id, { path: "/tmp/a.mp3", kind: "audio" }), + ).resolves.toBeDefined(); + }); }); describe("removeAsset", () => { @@ -363,6 +420,57 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(b.assets[1]?.id); }); + // Issue #350 — an audio overlay can never be primary. + it("passes primary to the next VIDEO asset, never to an audio asset", async () => { + const doc = await service.createProject("P"); + const video = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + await service.addAsset(doc.project.id, { path: "/tmp/music.mp3", kind: "audio" }); + const primaryId = video.project.primaryAssetId; + expect(primaryId).toBeTruthy(); + // Removing the only video leaves just the audio asset; primary must clear, + // not fall to the audio one. + const after = await service.removeAsset(doc.project.id, primaryId ?? ""); + expect(after.project.primaryAssetId).toBeUndefined(); + expect(after.assets).toHaveLength(1); + expect(after.assets[0]?.kind).toBe("audio"); + }); + + it("drops audioTracks that referenced a removed audio asset", async () => { + const doc = await service.createProject("P"); + await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/music.mp3", + kind: "audio", + }); + const audioId = withAudio.assets.find((a) => a.kind === "audio")?.id ?? ""; + expect(audioId).toBeTruthy(); + const withTrack = await service.saveProject({ + ...withAudio, + audioTracks: [ + { + id: "trk_1", + assetId: audioId, + kind: "music", + startMs: 0, + endMs: 10_000, + durationSec: 10, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "music", + origin: "user", + }, + ], + }); + expect(withTrack.audioTracks).toHaveLength(1); + const after = await service.removeAsset(doc.project.id, audioId); + expect(after.audioTracks).toEqual([]); + expect(after.assets.some((a) => a.id === audioId)).toBe(false); + }); + it("resequences other assets and rederives their anchored regions", async () => { const created = await service.createProject("P"); const withA = await service.addAsset(created.project.id, { path: "/tmp/a.mp4" }); diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 3c93e3bc0..d3eaa01d3 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -13,6 +13,7 @@ import fs, { type FileHandle } from "node:fs/promises"; import path from "node:path"; import { createId } from "../../src/lib/ai-edition/document/ids"; +import { parseStoredDocument, reconcileInsertions } from "../../src/lib/ai-edition/document/load"; import { removeClip } from "../../src/lib/ai-edition/document/timeline"; import { type AxcutAsset, @@ -38,6 +39,9 @@ export interface ProjectSummary { export interface AddAssetInput { path: string; label?: string; + // "audio" imports an external voiceover / BGM / SFX file (issue #350). + // Defaults to "video" when omitted, so existing callers are unaffected. + kind?: "video" | "audio"; } export class DocumentNotFoundError extends Error { @@ -72,6 +76,35 @@ function isSupportedVideoPath(filePath: string): boolean { return SUPPORTED_VIDEO_EXTENSIONS.has(ext); } +// Imported audio (issue #350). Decoding is handled downstream by the same +// WebCodecs / ffmpeg paths that read a video's audio track, so this list is the +// container formats decodeAudioData and the compositor can open. +// What may be filed as an AUDIO asset. Deliberately WIDER than the import +// picker's list in `electron/ipc/handlers.ts`: this gate runs when the caller +// has already declared `kind: "audio"`, so it only has to reject files that +// could not carry audio at all, whereas the picker has to guess from the +// extension alone and must not offer a video as audio. +// +// `.webm` is exactly that difference. An in-editor voiceover take is written by +// MediaRecorder as webm/opus — the same extension a screen recording uses — so +// the picker rightly refuses it while this gate must accept it. +const SUPPORTED_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".oga", + ".opus", + ".webm", +]); + +function isSupportedAudioPath(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase(); + return SUPPORTED_AUDIO_EXTENSIONS.has(ext); +} + function safeProjectId(raw: string): string { // ponytail: project ids are uuid-prefixed strings (e.g. "proj_"). Reject // anything that smells like path traversal before we ever touch the disk. @@ -90,7 +123,7 @@ function safeProjectId(raw: string): string { // `getProject` spells the same two steps out inline because it relinks moved // media between them; keep the order (upgrade, then validate) in step. function parseLoadedDocument(raw: string): AxcutDocument { - return documentSchema.parse(migrateRawDocumentToCurrent(JSON.parse(raw))); + return parseStoredDocument(JSON.parse(raw)); } /** @@ -241,7 +274,9 @@ export class DocumentService { // back, and it is not persisted from here: the renderer saves the document // it was given, as it does for any other load-time repair. const migrated = migrateRawDocumentToCurrent(JSON.parse(raw)); - return documentSchema.parse(await relinkProjectMedia(migrated, this.mediaRegistryDir)); + return reconcileInsertions( + documentSchema.parse(await relinkProjectMedia(migrated, this.mediaRegistryDir)), + ); } async createProject(title: string): Promise { @@ -283,7 +318,15 @@ export class DocumentService { if (!input.path) { throw new ProjectFileError("Asset path is required.", projectId); } - if (!isSupportedVideoPath(input.path)) { + const kind = input.kind ?? "video"; + if (kind === "audio") { + if (!isSupportedAudioPath(input.path)) { + throw new ProjectFileError( + `Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`, + projectId, + ); + } + } else if (!isSupportedVideoPath(input.path)) { throw new ProjectFileError( `Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`, projectId, @@ -300,18 +343,24 @@ export class DocumentService { } const asset: AxcutAsset = { id: createId("asset"), - kind: "video", + kind, label: input.label?.trim() || path.basename(absolutePath), originalPath: absolutePath, sizeBytes, cameraTrack: null, }; + // An audio import is an overlay, never the thing the timeline is built + // around, so it must not claim the empty primaryAssetId slot — otherwise the + // first file dropped into a fresh project (a BGM track) would become its + // primary asset and the editor would try to lay out clips from a file with + // no video. + const claimsPrimary = kind !== "audio" && !doc.project.primaryAssetId; const next: AxcutDocument = { ...doc, assets: [...doc.assets, asset], project: { ...doc.project, - ...(doc.project.primaryAssetId ? {} : { primaryAssetId: asset.id }), + ...(claimsPrimary ? { primaryAssetId: asset.id } : {}), updatedAt: new Date().toISOString(), }, }; @@ -324,9 +373,13 @@ export class DocumentService { throw new ProjectFileError(`Asset ${assetId} not found in project ${projectId}.`, projectId); } const assets = doc.assets.filter((a) => a.id !== assetId); + // Primary is the thing the timeline is built around, so it must fall to the + // next VIDEO asset — never an audio overlay (issue #350), which can't be + // primary (see addAsset). Falling back to `assets[0]` would hand primary to + // an audio asset when the removed one was the last video. const primaryAssetId = doc.project.primaryAssetId === assetId - ? (assets[0]?.id ?? undefined) + ? (assets.find((a) => a.kind !== "audio")?.id ?? undefined) : doc.project.primaryAssetId; const withoutAssetClips = doc.timeline.clips .filter((clip) => clip.assetId === assetId) @@ -334,6 +387,9 @@ export class DocumentService { const next: AxcutDocument = { ...withoutAssetClips, assets, + // Drop imported audio tracks that referenced the removed asset — they + // would otherwise dangle, pointing at an asset the document no longer has. + audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId), timeline: { ...withoutAssetClips.timeline, trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId), diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index e140a4e37..6de7a3cc4 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -289,6 +289,22 @@ interface Window { name?: string; canceled?: boolean; }>; + // Import an external audio file from the timeline toolbar (issue #350). + openAudioFilePicker: () => Promise<{ + success: boolean; + path?: string; + name?: string; + canceled?: boolean; + message?: string; + }>; + // Persist an in-editor voiceover take (raw MediaRecorder bytes) under the + // recordings dir, so it outlives the session like every other asset. + saveRecordedVoiceover: (data: ArrayBuffer) => Promise<{ + success: boolean; + path?: string; + message?: string; + error?: string; + }>; setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>; setCurrentRecordingSession: ( session: import("../src/lib/recordingSession").RecordingSession | null, diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index aa2014670..f404daa84 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -107,6 +107,9 @@ const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([ ".ts", ]); const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); +// See the save-recorded-voiceover handler: an upper bound on renderer-supplied +// bytes written to disk, well past any plausible take. +const MAX_RECORDED_VOICEOVER_BYTES = 512 * 1024 * 1024; const nativeMacCaptureEvents = new EventEmitter(); // Enumeration walks every display and window and grabs a thumbnail of each, so it @@ -186,6 +189,34 @@ function hasAllowedImportVideoExtension(filePath: string): boolean { return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); } +// Imported audio (issue #350). Kept separate from the video set so the two +// pickers stay honest — an audio picker must not approve a video path and vice +// versa. A SUBSET of SUPPORTED_AUDIO_EXTENSIONS in the document service, which +// also accepts `.webm`: that gate is told the kind by its caller, while this one +// only has the extension to go on and `.webm` is far more often a video. +const ALLOWED_IMPORT_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".opus", +]); + +function hasAllowedImportAudioExtension(filePath: string): boolean { + return ALLOWED_IMPORT_AUDIO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} + +// Video OR audio. The type-specific pickers stay honest (see the audio set's +// comment), but the generic media READS — peaks, binary, file-info, chunk — serve +// whichever kind the document points at, so they must accept both. Gating them on +// video alone dropped every imported audio path once `approvedPaths` was empty +// (a project reopen), and the waveform was lost for good (issue #350). +function hasAllowedImportMediaExtension(filePath: string): boolean { + return hasAllowedImportVideoExtension(filePath) || hasAllowedImportAudioExtension(filePath); +} + function runProcess( command: string, args: string[], @@ -282,8 +313,13 @@ async function prepareSupplementalPreviewAudioTrack(videoPath: string) { return { success: true, path: pathToFileURL(outputPath).toString() }; } -async function approveReadableVideoPath( - filePath?: string | null, +// Shared core behind the media path approvers. `hasAllowedExtension` is the ONLY +// thing that differs between video and audio imports, so it is the single knob: +// an already-approved path passes regardless, otherwise the extension gate, +// optional trusted-dir confinement, and a stat check decide whether to approve. +async function approveReadableMediaPath( + filePath: string | null | undefined, + hasAllowedExtension: (p: string) => boolean, trustedDirs?: string[], ): Promise { const normalizedPath = normalizeVideoSourcePath(filePath); @@ -295,7 +331,7 @@ async function approveReadableVideoPath( return normalizedPath; } - if (!hasAllowedImportVideoExtension(normalizedPath)) { + if (!hasAllowedExtension(normalizedPath)) { return null; } @@ -322,6 +358,29 @@ async function approveReadableVideoPath( return normalizedPath; } +function approveReadableVideoPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportVideoExtension, trustedDirs); +} + +function approveReadableAudioPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportAudioExtension, trustedDirs); +} + +// For the generic media reads that accept either kind — NOT for the pickers, +// which must stay type-specific (see `hasAllowedImportMediaExtension`). +function approveReadableAvPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportMediaExtension, trustedDirs); +} + function resolveRecordingOutputPath(fileName: string): string { const trimmed = fileName.trim(); if (!trimmed) { @@ -3590,6 +3649,8 @@ export function registerIpcHandlers( } }); + // The media tab imports VIDEO (it arranges clips). Audio is imported from the + // timeline toolbar instead (issue #350) — see `open-audio-file-picker` below. ipcMain.handle("open-video-file-picker", async () => { try { const dialogOptions = buildDialogOptions( @@ -3636,6 +3697,84 @@ export function registerIpcHandlers( } }); + // Import an external audio file (voiceover / BGM / SFX) — issue #350. Driven by + // the timeline's "Add audio" tool: audio is a timeline overlay (like an + // annotation), not a media-tab clip, so it has its own audio-only picker and the + // renderer adds it as a kind:"audio" asset + track at the playhead. + ipcMain.handle("open-audio-file-picker", async () => { + try { + const dialogOptions = buildDialogOptions( + { + title: mainT("dialogs", "fileDialogs.selectAudio"), + defaultPath: RECORDINGS_DIR, + filters: [ + { + name: mainT("dialogs", "fileDialogs.audioFiles"), + extensions: ["mp3", "wav", "m4a", "aac", "flac", "ogg", "opus"], + }, + { name: mainT("dialogs", "fileDialogs.allFiles"), extensions: ["*"] }, + ], + properties: ["openFile"], + }, + getMainWindow(), + ); + const result = await dialog.showOpenDialog(dialogOptions); + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + + const normalizedPath = await approveReadableAudioPath(result.filePaths[0]); + if (!normalizedPath) { + return { + success: false, + message: "Selected file is not a supported readable audio file", + }; + } + + return { + success: true, + path: normalizedPath, + }; + } catch (error) { + console.error("Failed to open audio file picker:", error); + return { + success: false, + message: "Failed to open audio file picker", + error: String(error), + }; + } + }); + + // In-editor voiceover recording: the renderer hands over the raw MediaRecorder + // blob (webm/opus) and gets back the path it landed at, under the recordings + // dir so it lives with the project's other media and survives relaunches. + ipcMain.handle("save-recorded-voiceover", async (_event, data: ArrayBuffer) => { + try { + if (!(data instanceof ArrayBuffer) || data.byteLength === 0) { + return { success: false, message: "Empty recording" }; + } + // A cap, because this writes renderer-supplied bytes straight to disk. An + // hour of Opus is a few tens of MB, so 512 MB is far past any real take + // and still refuses a runaway or malformed payload before it is buffered. + if (data.byteLength > MAX_RECORDED_VOICEOVER_BYTES) { + return { success: false, message: "Recording too large" }; + } + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + const fileName = `voiceover-${new Date().toISOString().replace(/[:.]/g, "-")}.webm`; + const target = path.join(RECORDINGS_DIR, fileName); + await fs.writeFile(target, Buffer.from(data)); + return { success: true, path: target }; + } catch (error) { + console.error("Failed to save recorded voiceover:", error); + return { + success: false, + message: "Failed to save recorded voiceover", + error: String(error), + }; + } + }); + ipcMain.handle("reveal-in-folder", async (_, filePath: string) => { try { // showItemInFolder returns nothing, it throws on error @@ -3661,7 +3800,7 @@ export function registerIpcHandlers( ipcMain.handle("read-binary-file", async (_, filePath: string) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, @@ -3691,7 +3830,7 @@ export function registerIpcHandlers( // recording above that can never be loaded whole — see read-file-chunk). ipcMain.handle("get-readable-file-info", async (_, filePath: string) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, @@ -3727,7 +3866,7 @@ export function registerIpcHandlers( async (_, filePath: string, durationSec: number): Promise => { try { // Same approval gate as every other read of a renderer-supplied path. - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, message: "File path is not approved" }; } @@ -3751,7 +3890,7 @@ export function registerIpcHandlers( // do (2 GiB cap) and a 16 GB machine cannot hold for multi-GB recordings. ipcMain.handle("read-file-chunk", async (_, filePath: string, offset: number, length: number) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 5d4992c10..7b01e0b2f 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -489,6 +489,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { request.payload.projectId, request.payload.path, request.payload.label, + request.payload.kind, ), ); case "document.removeAsset": diff --git a/electron/native-bridge/services/aiEditionService.ts b/electron/native-bridge/services/aiEditionService.ts index 0fbbccc9c..90088781a 100644 --- a/electron/native-bridge/services/aiEditionService.ts +++ b/electron/native-bridge/services/aiEditionService.ts @@ -151,9 +151,17 @@ export class AiEditionService { } } - async addAsset(projectId: string, path: string, label?: string): Promise { - const document = await this.options.documents.addAsset(projectId, { path, label }); - const assetId = document.project.primaryAssetId ?? document.assets.at(-1)?.id ?? ""; + async addAsset( + projectId: string, + path: string, + label?: string, + kind?: "video" | "audio", + ): Promise { + const document = await this.options.documents.addAsset(projectId, { path, label, kind }); + // The just-added asset is always the last one; primaryAssetId is only a + // fallback for the video case and would point at the wrong asset for an + // audio import (which never claims primary), so prefer the tail. + const assetId = document.assets.at(-1)?.id ?? document.project.primaryAssetId ?? ""; return { assetId, document }; } diff --git a/electron/preload.ts b/electron/preload.ts index 6aff16407..7873bef90 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -276,6 +276,12 @@ contextBridge.exposeInMainWorld("electronAPI", { openVideoFilePicker: () => { return ipcRenderer.invoke("open-video-file-picker"); }, + openAudioFilePicker: () => { + return ipcRenderer.invoke("open-audio-file-picker"); + }, + saveRecordedVoiceover: (data: ArrayBuffer) => { + return ipcRenderer.invoke("save-recorded-voiceover", data); + }, setCurrentVideoPath: (path: string) => { return ipcRenderer.invoke("set-current-video-path", path); }, diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts new file mode 100644 index 000000000..24e9694c8 --- /dev/null +++ b/electron/stt/extractAudio.test.ts @@ -0,0 +1,141 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const spawnMock = vi.fn(); +const resolveFfmpegMock = vi.fn<() => string | null>(); + +vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) })); +vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() })); + +const { extractMono16kPcm, FfmpegUnavailableError, NoAudioTrackError } = await import( + "./extractAudio" +); +const { STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import("./transcriptionContract"); + +/** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */ +function fakeChild() { + const child = new EventEmitter() as EventEmitter & { + stdout: PassThrough; + stderr: PassThrough; + kill: ReturnType; + }; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = vi.fn(); + return child; +} + +/** The little-endian float32 bytes ffmpeg would emit for `values`. */ +function f32le(values: number[]): Buffer { + const buf = Buffer.alloc(values.length * 4); + values.forEach((v, i) => buf.writeFloatLE(v, i * 4)); + return buf; +} + +beforeEach(() => { + vi.clearAllMocks(); + resolveFfmpegMock.mockReturnValue("/usr/bin/ffmpeg"); +}); + +describe("extractMono16kPcm", () => { + it("asks ffmpeg for exactly what whisper wants", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/a.mp3"); + child.stdout.end(f32le([0.5])); + child.emit("close", 0); + await promise; + + const args = spawnMock.mock.calls[0][1] as string[]; + // Mono, 16 kHz, float32 little-endian, no video. Anything else and whisper is + // reading the samples wrong rather than failing loudly. + expect(args).toContain("-vn"); + expect(args.join(" ")).toContain("-ac 1"); + expect(args.join(" ")).toContain("-ar 16000"); + expect(args.join(" ")).toContain("-f f32le"); + }); + + it("decodes the samples ffmpeg writes", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/a.mp3"); + child.stdout.end(f32le([0, 0.5, -0.25])); + child.emit("close", 0); + + const out = await promise; + expect(Array.from(out)).toEqual([0, 0.5, -0.25]); + }); + + it("carries a float split across two chunks instead of dropping it", async () => { + // THE defect worth a test here: stdout chunk boundaries do not respect sample + // boundaries. Dropping the partial tail would shift every following sample and + // detune the whole track — audible, and invisible in a length check. + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/a.mp3"); + const bytes = f32le([0.25, -0.75, 1]); + child.stdout.write(bytes.subarray(0, 6)); // one whole float + half of the next + child.stdout.write(bytes.subarray(6)); + child.stdout.end(); + child.emit("close", 0); + + const out = await promise; + expect(Array.from(out)).toEqual([0.25, -0.75, 1]); + }); + + it("reports a file with no audio track", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/silent.mp4"); + child.stderr.end("Stream map '0:a' matches no streams"); + child.stdout.end(); + child.emit("close", 1); + + await expect(promise).rejects.toBeInstanceOf(NoAudioTrackError); + }); + + it("keeps the samples when ffmpeg exits non-zero AFTER writing audio", async () => { + // A truncated file still yields usable audio; throwing it away would lose a + // transcript over a trailing byte. + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/tmp/truncated.mp3"); + child.stdout.end(f32le([0.1, 0.2])); + child.emit("close", 1); + + expect(Array.from(await promise)).toEqual([expect.closeTo(0.1, 6), expect.closeTo(0.2, 6)]); + }); + + it("refuses with the marker the renderer falls back on when ffmpeg is missing", async () => { + // The string is the contract across the IPC boundary, which drops the class. + resolveFfmpegMock.mockReturnValue(null); + await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toBeInstanceOf(FfmpegUnavailableError); + await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow( + STT_NATIVE_EXTRACTION_UNAVAILABLE, + ); + expect(spawnMock).not.toHaveBeenCalled(); + }); + + it("kills the child when the caller aborts", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const controller = new AbortController(); + const promise = extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal }); + controller.abort(); + + await expect(promise).rejects.toMatchObject({ name: "AbortError" }); + // Not merely stopping to await: ffmpeg would keep decoding a long file for + // minutes, which is the same leak the STT cancel path exists to prevent. + expect(child.kill).toHaveBeenCalled(); + }); + + it("does not spawn at all when the signal is already aborted", async () => { + const controller = new AbortController(); + controller.abort(); + await expect( + extractMono16kPcm("/tmp/a.mp3", { signal: controller.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(spawnMock).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts new file mode 100644 index 000000000..08168d21e --- /dev/null +++ b/electron/stt/extractAudio.ts @@ -0,0 +1,152 @@ +// Native mono-16k extraction for transcription, in the main process. +// +// The renderer used to do this: `extractMono16kFromVideoUrl` read the whole media +// into a `File`, took an `arrayBuffer()`, handed a `slice(0)` copy to +// `decodeAudioData`, and resampled the result to mono 16k — all of it on the UI +// thread, all of it before whisper ever saw a sample. On a four-minute bed that is +// ~86 MB of decoded float32 plus two copies of the encoded bytes, and it froze the +// editor at open. The inference itself was never the problem: it runs in +// `whisper-stt-server`, in its own process, on the GPU. +// +// So this is the same remedy `useAudioPeaks` already got (see +// `electron/media/audioPeaks.ts`, and the note there about WHICH ffmpeg is packaged +// on Windows): let ffmpeg do it, in the main process, streaming. It costs +// `durationSec * 16000 * 4` bytes — 15.7 MB for that same four-minute bed — and the +// renderer never allocates any of it. +// +// It is deliberately NOT cached on disk, unlike peaks. Peaks are re-read on every +// project open; extraction feeds one transcription, whose RESULT is what gets +// persisted (`document.transcripts[]`). Caching the PCM would trade disk for work +// that is already never repeated. + +import { spawn } from "node:child_process"; +import { resolveFfmpeg } from "../media/audioPeaks"; +import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; + +/** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */ +const SAMPLE_RATE = 16_000; + +/** Past this, it is not a recording — it is a wedged ffmpeg. Matches the peaks path. */ +const EXTRACT_TIMEOUT_MS = 60_000; + +/** + * Thrown when no ffmpeg can be resolved, so the caller can fall back to the + * renderer pipeline rather than failing the transcription outright. A distinct type + * because "there is no ffmpeg here" and "this file has no audio" want opposite + * responses: fall back, versus report a permanent failure for this asset. + */ +export class FfmpegUnavailableError extends Error { + constructor() { + super(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary for native audio extraction`); + this.name = "FfmpegUnavailableError"; + } +} + +/** A media with no decodable audio track. Permanent for that file. */ +export class NoAudioTrackError extends Error { + constructor(filePath: string, detail: string) { + super(`No decodable audio in ${filePath}${detail ? `: ${detail}` : ""}`); + this.name = "NoAudioTrackError"; + } +} + +/** + * Decode `filePath` to mono 16 kHz float samples. + * + * Streams `f32le` straight off ffmpeg's stdout, so the only full-size allocation is + * the result itself. Chunk boundaries do not respect sample boundaries — a 4-byte + * float can straddle two `data` events — so a partial tail is carried into the next + * chunk rather than dropped, which would shift every following sample and detune the + * whole track. + */ +export async function extractMono16kPcm( + filePath: string, + options: { signal?: AbortSignal } = {}, +): Promise { + const ffmpeg = resolveFfmpeg(); + if (!ffmpeg) throw new FfmpegUnavailableError(); + if (options.signal?.aborted) throw new DOMException("Aborted", "AbortError"); + + const child = spawn( + ffmpeg, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + filePath, + "-vn", + "-ac", + "1", + "-ar", + String(SAMPLE_RATE), + "-f", + "f32le", + "-", + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + return new Promise((resolve, reject) => { + const chunks: Float32Array[] = []; + let total = 0; + /** Bytes of a float that arrived split across two chunks. */ + let carry: Buffer | null = null; + let stderr = ""; + let settled = false; + + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + fn(); + }; + + const timer = setTimeout(() => { + child.kill("SIGKILL"); + finish(() => + reject(new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`)), + ); + }, EXTRACT_TIMEOUT_MS); + + const onAbort = () => { + child.kill("SIGKILL"); + finish(() => reject(new DOMException("Aborted", "AbortError"))); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + + child.stdout.on("data", (c: Buffer) => { + const buf = carry ? Buffer.concat([carry, c]) : c; + const usable = buf.length - (buf.length % 4); + if (usable > 0) { + // Copy rather than view: a Buffer's memory is rarely 4-byte aligned, and + // `byteOffset` is almost never 0. + const view = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + usable)); + chunks.push(view); + total += view.length; + } + carry = usable < buf.length ? Buffer.from(buf.subarray(usable)) : null; + }); + child.stderr.on("data", (c: Buffer) => { + stderr = (stderr + c.toString()).slice(-2048); + }); + child.once("error", (err) => finish(() => reject(err))); + child.once("close", (code) => { + // A file with no audio track exits non-zero, and so does a corrupt one. The + // caller treats both the same way — this asset will not transcribe — so they + // share an error type; `stderr` carries which it was. + if (code !== 0 && total === 0) { + finish(() => reject(new NoAudioTrackError(filePath, stderr.trim()))); + return; + } + const out = new Float32Array(total); + let at = 0; + for (const part of chunks) { + out.set(part, at); + at += part.length; + } + finish(() => resolve(out)); + }); + }); +} diff --git a/electron/stt/index.ts b/electron/stt/index.ts index c250ea4ac..1004726f5 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { app, type IpcMain } from "electron"; import { planChunks } from "./chunking"; +import { extractMono16kPcm } from "./extractAudio"; import { ensureModels, modelPaths } from "./modelManager"; import type { SttPhraseSegment, @@ -237,12 +238,26 @@ export class SttManager { } /** Transcribe a whole recording, chunk by chunk, reporting progress as it goes. */ + /** Decode `sourcePath` into the samples `transcribe` needs. */ + private async extract(req: SttTranscribeRequest): Promise { + if (!req.sourcePath) { + throw new Error("stt:transcribe needs either `samples` or `sourcePath`"); + } + return extractMono16kPcm(req.sourcePath); + } + async transcribe(req: SttTranscribeRequest): Promise { await this.init(); const epoch = this.cancelEpoch; - const totalSec = req.samples.length / SAMPLE_RATE; - const chunks = planChunks(req.samples, SAMPLE_RATE); + // Extraction is part of the run, and on a long file it is the part the user used + // to watch the editor freeze through. Doing it here means the renderer hands over + // a path and gets segments back, holding none of the audio. No new status phase: + // the caller already reports "extracting-audio" around this call, and the work + // simply moved to the other side of the IPC. + const samples = req.samples ?? (await this.extract(req)); + const totalSec = samples.length / SAMPLE_RATE; + const chunks = planChunks(samples, SAMPLE_RATE); this.emit({ phase: "transcribe", completedSec: 0, totalSec }); const segments: SttPhraseSegment[] = []; @@ -285,7 +300,7 @@ export class SttManager { if (this.cancelEpoch !== epoch) throw cancelledError(); const offsetSec = chunk.startSample / SAMPLE_RATE; const result = await this.transcribeChunk( - req.samples.subarray(chunk.startSample, chunk.endSample), + samples.subarray(chunk.startSample, chunk.endSample), language, ).catch((error) => { if (error instanceof Error && error.name === "AbortError") throw error; diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index cbce5a141..d2e67ae11 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -106,7 +106,24 @@ export interface SttStatusEvent { /** IPC request: renderer → main. */ export interface SttTranscribeRequest { - samples: Float32Array; + /** + * Mono-16k samples the CALLER decoded. Optional since native extraction landed: + * pass `sourcePath` instead and the main process decodes with ffmpeg, off the UI + * thread and without the renderer ever holding the audio. Kept for the caption + * path, which already has samples in hand and has no file to point at. + * + * Exactly one of `samples` / `sourcePath` is required. + */ + samples?: Float32Array; + /** + * A media file for the main process to decode itself (ffmpeg -> mono 16k f32). + * Preferred: the renderer's own pipeline read the whole file, copied it twice and + * resampled it on the UI thread, which is what froze the editor at open. + * + * The caller falls back to its own decode when this cannot be honoured — see + * `FfmpegUnavailableError`. + */ + sourcePath?: string; /** * ISO 639-1 language code (e.g. "en", "fr"). Omit / `"auto"` to let Whisper detect. * The spec locks language detection on by default; we only honour an explicit value. @@ -114,6 +131,17 @@ export interface SttTranscribeRequest { language?: string; } +/** + * Marker carried in the error message when the main process cannot decode a + * `sourcePath` because no ffmpeg is resolvable on this install. + * + * A string rather than an error class because this crosses `ipcRenderer.invoke`, + * which reconstructs a plain `Error` from the message and drops the prototype and + * the `name`. Exported so neither side spells it out by hand — a fallback keyed on + * a literal typed twice is a fallback that silently stops working. + */ +export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable"; + /** IPC response: main → renderer. */ export interface SttTranscribeResponse { segments: SttPhraseSegment[]; diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 9a7f1b590..5da415596 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -95,6 +95,7 @@ function buildNativeClipList(axcutDocument: AxcutDocument): CompositorClipInput[ sourceEndSec, webcamOffsetSec: camera.offsetSec, hasAudio: true, + holdSec: clip.heldSec ?? 0, }, ]; }); diff --git a/src/components/ai-edition/CaptionsPane.gating.test.tsx b/src/components/ai-edition/CaptionsPane.gating.test.tsx index cae76b59a..506404f2a 100644 --- a/src/components/ai-edition/CaptionsPane.gating.test.tsx +++ b/src/components/ai-edition/CaptionsPane.gating.test.tsx @@ -1,9 +1,12 @@ // @vitest-environment jsdom -// Captions are a view of the transcript, so the pane's "Transcribe video" -// button is a retry, not a first step — the background pass has already tried. -// On a media with no audio track that retry can only fail again, so the button -// has to be dead and the pane has to say what is wrong instead of inviting a -// pointless click. +// Captions are a view of the transcript, and since issue #560 they are reached from +// the transcript tab rather than owning one. So this pane no longer STARTS a +// transcription — the transcript tab's empty state carries the single gate. Two +// buttons for one background pass is what made people believe captions were +// transcribed separately. +// +// What the pane still owes the reader is a status: whether a pass is already +// running, and why there will never be one on a media with no audio track. import "@testing-library/jest-dom"; import { cleanup, render, screen } from "@testing-library/react"; @@ -76,6 +79,14 @@ function load(document: AxcutDocument) { }); } +function mount() { + render( + + + , + ); +} + beforeEach(() => { useTranscriptionStore.getState().reset(); useProjectStore.getState().clear(); @@ -86,43 +97,35 @@ afterEach(() => { }); describe("captions pane gating", () => { - it("offers the retry while the media might still yield a transcript", () => { + it("does not offer a second way to start a transcription", () => { load(documentWith(ASSET)); - render( - - - , - ); - expect(screen.getByRole("button", { name: "Transcribe video" })).toBeEnabled(); + mount(); + expect(screen.queryByRole("button", { name: "Transcribe video" })).toBeNull(); + expect( + screen.getByText("Captions are read from the media transcript.", { exact: false }), + ).toBeInTheDocument(); }); - it("shows the queued background run instead of an idle button", () => { + it("reports a background run that is already going", () => { load(documentWith(ASSET)); useTranscriptionStore.setState({ projectId: "proj_1", jobs: { asset_1: { status: "running", language: "auto", manual: false } }, }); - render( - - - , - ); - expect(screen.getByRole("button", { name: "Transcribing…" })).toBeDisabled(); + mount(); + expect(screen.getByText("Transcribing…")).toBeInTheDocument(); + // Still not a control: a running pass is news, not something to press. + expect(screen.queryByRole("button", { name: "Transcribing…" })).toBeNull(); }); - it("kills the retry on a media with no audio track and explains it", () => { + it("explains a media with no audio track, where no pass will ever help", () => { load( documentWith({ ...ASSET, transcriptionFailure: { kind: "no-audio", message: "No audio track found in this video." }, }), ); - render( - - - , - ); - expect(screen.getByRole("button", { name: "Transcribe video" })).toBeDisabled(); + mount(); expect( screen.getByText("This media has no audio track — there is nothing to transcribe."), ).toBeInTheDocument(); diff --git a/src/components/ai-edition/CaptionsPane.tsx b/src/components/ai-edition/CaptionsPane.tsx index 6d6a776c9..a0ba29a76 100644 --- a/src/components/ai-edition/CaptionsPane.tsx +++ b/src/components/ai-edition/CaptionsPane.tsx @@ -19,10 +19,7 @@ import { untranslatedUnits, } from "@/lib/ai-edition/captions"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; -import { - useTimelineTranscriptGate, - useTranscriptionStore, -} from "@/lib/ai-edition/store/transcriptionStore"; +import { useTimelineTranscriptGate } from "@/lib/ai-edition/store/transcriptionStore"; import { useCaptions } from "@/lib/ai-edition/store/useCaptions"; import { nativeBridgeClient } from "@/native"; import { ColorField } from "./ColorField"; @@ -90,14 +87,14 @@ export function CaptionsPane() { // Captions are a view of the transcript, and the transcript arrives on its // own (transcriptionStore's background pass). The pane reads that state // straight from the store rather than being handed a busy flag: it is the - // same answer everywhere, and "Transcribe" here is only ever a retry. + // same answer everywhere, and this pane only ever reports on the pass — + // starting one is the transcript tab's job. // // Resolved over the timeline's assets, not the primary one: `hasTranscript` // below is already timeline-scoped (useCaptions), and mixing the two scopes // is what let a silent primary asset dead-end this button for a project whose // actual footage had speech. const gate = useTimelineTranscriptGate(); - const requestTimelineTranscripts = useTranscriptionStore((s) => s.requestTimelineTranscripts); const isTranscribing = gate.state === "pending"; const silentMedia = gate.state === "blocked" && gate.reason === "no-audio"; const engineError = gate.state === "blocked" && gate.reason === "failed" ? gate.message : null; @@ -223,17 +220,26 @@ export function CaptionsPane() { {engineError}

) : null} - + {/* No transcribe button here. This pane is reached from the transcript + tab, whose empty state carries the one gate — and two buttons for + one background pass is what made people believe captions were + transcribed separately from the transcript (issue #560). What is + worth saying here is whether a run is already going. */} + {isTranscribing ? ( +

+ + {t("captions.transcribing")} +

+ ) : null} ) : (

resolveNativePosition(currentTimeSec, nativeClips, document?.timeline.clips ?? []), + () => + resolveNativePosition( + currentTimeSec, + nativeClips, + document?.timeline.clips ?? [], + document?.timeline.insertRanges ?? [], + ), [nativeClips, currentTimeSec, document], ); const activeClip = activePosition?.clip ?? null; diff --git a/src/components/ai-edition/NewEditorShell.module.css b/src/components/ai-edition/NewEditorShell.module.css index 8078ca27b..e99482f7f 100644 --- a/src/components/ai-edition/NewEditorShell.module.css +++ b/src/components/ai-edition/NewEditorShell.module.css @@ -2122,6 +2122,66 @@ .iconBtn:hover { background: var(--surface-3); color: var(--fg); } .iconBtn:disabled { opacity: 0.5; cursor: not-allowed; } +/* A labelled sibling of .iconBtn. Captions lost their own tab (issue #560), so the + control that replaces it cannot be an icon you must hover to identify: the whole + point of folding it in here was that people find it. */ +.paneHeadBtn { + display: inline-flex; + align-items: center; + gap: 6px; + height: 28px; + padding: 0 10px; + border-radius: var(--r-sm); + background: transparent; + border: 1px solid var(--border); + color: var(--fg-2); + font: 500 12px/1 var(--font-body); + white-space: nowrap; + cursor: pointer; +} +.paneHeadBtn:hover { background: var(--surface-3); color: var(--fg); } + +/* Two mutually exclusive readings of the same tab, so: one track, one lit half. + Not tabs — tabs would say the two transcripts sit side by side, and only one of + them exists as far as everything downstream is concerned (issue #560). */ +.laneSwitch { + display: flex; + gap: 2px; + margin: 0 var(--sp-4) 6px; + padding: 2px; + border: 1px solid var(--border); + border-radius: var(--r-sm); + background: var(--surface-2); +} +.laneSwitchBtn { + flex: 1; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + min-height: 26px; + padding: 0 8px; + border: 0; + border-radius: calc(var(--r-sm) - 2px); + background: transparent; + color: var(--muted); + font: 500 12px/1 var(--font-body); + white-space: nowrap; + cursor: pointer; +} +.laneSwitchBtn:hover { color: var(--fg-2); } +.laneSwitchBtn.isActive { + background: var(--surface); + color: var(--fg); + box-shadow: var(--elev-1, 0 1px 2px rgb(0 0 0 / 0.12)); +} +/* The choice reaches past this tab — it decides the text burnt into the export. */ +.laneSwitchNote { + margin: 0 var(--sp-4) 12px; + font: 400 11.5px/1.45 var(--font-body); + color: var(--meta); +} + .backBtn { display: inline-flex; align-items: center; diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index b371d100c..2d1f5fbb1 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -5,16 +5,25 @@ import { toFileUrl } from "@/components/video-editor/projectPersistence"; import { useEditorDialogActions } from "@/contexts/EditorDialogsContext"; import { useScopedT } from "@/contexts/I18nContext"; import { useShortcuts } from "@/contexts/ShortcutsContext"; -import { - migrateProjectDataToAxcutDocument, - migrateRawDocumentToCurrent, -} from "@/lib/ai-edition/document/migrate"; +import { createId } from "@/lib/ai-edition/document/ids"; +import { parseStoredDocument } from "@/lib/ai-edition/document/load"; +import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate"; import { applyProbedDuration, replaceTimeline as replaceTimelineOp, } from "@/lib/ai-edition/document/timeline"; +import { + type InsertSide, + insertDocumentWord, + removeDocumentWords, + setDocumentWordText, +} from "@/lib/ai-edition/document/transcript"; import { isModalOpen } from "@/lib/ai-edition/modalGuard"; -import { type AxcutClip, documentSchema } from "@/lib/ai-edition/schema"; +import { + type AxcutAudioTrack, + type AxcutClip, + type AxcutInsertRange, +} from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useAssetTranscriptions, @@ -26,6 +35,10 @@ import { useUndoRedoShortcuts } from "@/lib/ai-edition/store/undo"; import { useSequentialTimelineOps } from "@/lib/ai-edition/store/useSequentialTimelineOps"; import { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { newRegionDurationSec } from "@/lib/ai-edition/timeline/newRegionDuration"; +import { + dropTrimPillsByIds, + ventilateTimelineSpanToTrims, +} from "@/lib/ai-edition/timeline/trim-mapping"; import { matchesShortcut } from "@/lib/shortcuts"; import { nativeBridgeClient } from "@/native"; import type { AiEditionProjectSummary } from "@/native/contracts"; @@ -42,8 +55,8 @@ import { type UnsavedChoice, } from "./Modals"; import { Preview } from "./Preview"; -import type { TrimTarget } from "./RightPanes"; import { importPendingRecording } from "./recordingImport"; +import { AddAudioLayerDialog } from "./v4/AddAudioLayerDialog"; import v4 from "./v4/EditorShellV4.module.css"; import { type EditorMode, EditorTopBar } from "./v4/EditorTopBar"; import { type Facet, FloatingInspector } from "./v4/FloatingInspector"; @@ -73,18 +86,24 @@ interface SeekTarget { * handlers that need it. The store write cadence is unchanged — `currentTimeSec` * is still the source of truth, still updated every frame. */ +// Stable empty list: a fresh `[]` each render would churn the preview's audio +// element set on every playhead tick. +const NO_AUDIO_TRACKS: AxcutAudioTrack[] = []; + function NativePlaybackSync({ visibleClips, clips, + insertRanges, }: { visibleClips: AxcutClip[]; clips: AxcutClip[]; + insertRanges: readonly AxcutInsertRange[]; }) { const playing = useProjectStore((s) => s.playing); const currentTimeSec = useProjectStore((s) => s.currentTimeSec); // visibleClips = trim-compressed native stream; `clips` = RAW layout currentTimeSec // is measured against. resolveNativePosition needs both (see timelineMap). - useNativePlaybackSync(playing, currentTimeSec, visibleClips, clips); + useNativePlaybackSync(playing, currentTimeSec, visibleClips, clips, insertRanges); return null; } @@ -185,7 +204,11 @@ export function NewEditorShell() { // don't race each other's save and overwrite one another in the // store. The hook reads the doc inside the chain (after awaiting the // previous save) — see its source for the race this fixes. - const { apply: applyTimelineOp, enqueue: enqueueTimelineWrite } = useSequentialTimelineOps({ + // Only `enqueue` now: the two trim handlers were the last callers of `apply`, and both + // read the document inside the chain so a cut cannot be overwritten by a word edit + // landing between the read and the save. The `add_trim_range` / `remove_trim_range` ops + // stay for the agent, which addresses clips rather than moments. + const { enqueue: enqueueTimelineWrite } = useSequentialTimelineOps({ fallbackDocument: document, saveDocument, }); @@ -427,7 +450,16 @@ export function NewEditorShell() { const handleDropAsset = useCallback( (assetId: string) => enqueueTimelineWrite(() => { - const at = useProjectStore.getState().document?.timeline.clips.length ?? 0; + const doc = useProjectStore.getState().document; + // An audio asset has no video, so it must never become a clip (issue + // #350) — it goes on the audio lane as a track. Adding it "to the + // timeline" reuses its existing track if it already has one (importing + // already placed one) so the same file can't stack up duplicate lanes. + if (doc?.assets.find((a) => a.id === assetId)?.kind === "audio") { + if (doc.audioTracks.some((t) => t.assetId === assetId)) return Promise.resolve(); + return tl.addAudioTrack(assetId).then(() => undefined); + } + const at = doc?.timeline.clips.length ?? 0; return tl.insertClipAt(assetId, at); }).catch((error) => { toast.error(te("mediaStage.couldNotAddAsset"), { @@ -548,7 +580,7 @@ export function NewEditorShell() { const isAxcutDocument = typeof raw === "object" && raw !== null && "schemaVersion" in raw && "timeline" in raw; const doc = isAxcutDocument - ? documentSchema.parse(migrateRawDocumentToCurrent(raw)) // disk-load: upgrade v3/v4 → v5, then validate + ? parseStoredDocument(raw) // disk-load: upgrade, validate, reconcile clip geometry : migrateProjectDataToAxcutDocument(raw as EditorProjectData); const saved = await nativeBridgeClient.aiEdition.save(doc); if (saved.success && saved.document) { @@ -571,37 +603,142 @@ export function NewEditorShell() { // axcut's `queueAddTrimRange` / `queueRemoveTrimRange` callbacks in // apps/web/src/App.tsx. The serialised save + inside-the-chain doc // read is owned by `useSequentialTimelineOps` above. - const handleAddTrimRange = useCallback( - (target: TrimTarget, startSec: number, endSec: number, reason: string) => { - // `clipId` is what keeps the cut on the block the user typed in: with two clips - // over the same media, an asset-only trim showed up on both (see `trimAppliesToClip`). - void applyTimelineOp( - { - type: "add_trim_range", - assetId: target.assetId, - clipId: target.clipId, - startSec, - endSec, + // transcript-pane → a cut, authored as a stretch of the RAW ruler (issue #560). + // + // The pane used to hand over the asset and clip the words belonged TO, which is how a + // cut made on the voiceover lane came to be anchored on an audio fragment: it removed + // nothing from playback or the export while the word turned red. A cut is a moment of + // the programme, so the clips that carry it are resolved HERE, from the clips actually + // under the span — `ventilateTimelineSpanToTrims`, the same primitive a zoom straddling + // a boundary uses, so one gesture can become several rows and stay one pill. + // + // On `enqueueTimelineWrite`, not `applyTimelineOp`'s convenience or `tl.setTrimEntries`: + // the latter reads `useProjectStore.getState().document` unqueued, so correcting a word + // and immediately cutting the next one would let the word edit overwrite the cut. That + // is exactly the failure this chain exists to prevent. + const handleTrimTimelineSpan = useCallback( + (startSec: number, endSec: number, reason: string) => { + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc) return; + const ranges = ventilateTimelineSpanToTrims(startSec, endSec, doc.timeline.clips); + if (ranges.length === 0) { + // No nearest-clip fallback. A span over a gap, or past the last clip, names + // no film — cutting the closest thing instead would remove something the + // user never pointed at. + toast.error(te("errors.trimNoFilm")); + return; + } + const rows = ranges.map((range) => ({ + id: createId("trim"), + assetId: range.assetId, + clipId: range.clipId, + startSec: range.sourceStartSec, + endSec: range.sourceEndSec, reason, - }, - { history: true }, - ); + origin: "user" as const, + })); + await saveDocument( + { + ...doc, + timeline: { ...doc.timeline, trimRanges: [...doc.timeline.trimRanges, ...rows] }, + }, + { history: true }, + ); + }); }, - [applyTimelineOp], + [enqueueTimelineWrite, saveDocument, te], ); - const handleRemoveTrimRange = useCallback( - (trimId: string) => { - void applyTimelineOp( - { - type: "remove_trim_range", - trimId, - reason: "Restored from transcript pane.", - }, - { history: true }, - ); + // Every row of the pill at once: a cut ventilated across a clip boundary is several + // rows and one pill, and `dropTrimPillsByIds` resolves the rest of the group from any + // member. Dropping half would leave the word still cut with nothing on screen to say so. + const handleRemoveTrimRanges = useCallback( + (trimIds: string[]) => { + if (trimIds.length === 0) return; + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc) return; + const next = dropTrimPillsByIds( + doc.timeline.trimRanges, + doc.timeline.clips, + trimIds, + doc.timeline.insertRanges ?? [], + ); + if (next.length === doc.timeline.trimRanges.length) return; + await saveDocument( + { ...doc, timeline: { ...doc.timeline, trimRanges: next } }, + { history: true }, + ); + }); + }, + [enqueueTimelineWrite, saveDocument], + ); + + // transcript-pane → the word's own text. Unlike Backspace (which writes a trimRange and + // cuts the media), this writes only `transcript.words[].text`: the captions follow, the + // film is untouched. Queued on the SAME chain as the trims so correcting a word and + // cutting the next one cannot overwrite each other's save. + const handleSetWordText = useCallback( + (assetId: string, wordId: string, text: string) => { + void enqueueTimelineWrite(async () => { + // Read inside the chain: the previous save has resolved by now, so the store + // holds the document this edit has to be applied to. + const doc = useProjectStore.getState().document; + if (!doc) return; + try { + await saveDocument(setDocumentWordText(doc, assetId, wordId, text), { history: true }); + } catch (err) { + // The word or its transcript vanished under the edit (a regeneration landed + // mid-typing). Nothing to retry — say so rather than dropping it silently. + toast.error(te("errors.wordEditFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + + // transcript-pane → a word nobody said. It takes the silence it is dropped into and no + // audio at all, so unlike a cut it changes nothing about the film; today it reaches the + // captions and stops there. + const handleInsertWord = useCallback( + (assetId: string, anchorWordId: string, side: InsertSide, text: string) => { + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc) return; + try { + await saveDocument(insertDocumentWord(doc, assetId, anchorWordId, side, text), { + history: true, + }); + } catch (err) { + toast.error(te("errors.wordInsertFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); + }, + [enqueueTimelineWrite, saveDocument, te], + ); + + // Deleting inserted words. One save for the whole set, so a Backspace over several of + // them is one Ctrl+Z, and the document layer refuses anything that was actually spoken. + const handleRemoveWords = useCallback( + (assetId: string, wordIds: string[]) => { + void enqueueTimelineWrite(async () => { + const doc = useProjectStore.getState().document; + if (!doc || wordIds.length === 0) return; + try { + await saveDocument(removeDocumentWords(doc, assetId, wordIds), { history: true }); + } catch (err) { + toast.error(te("errors.wordRemoveFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } + }); }, - [applyTimelineOp], + [enqueueTimelineWrite, saveDocument, te], ); const handleSelectProject = useCallback( @@ -770,6 +907,56 @@ export function NewEditorShell() { }); }, []); + // Voiceover recording (the one audio gesture that is not a file import). The + // dialog owns the mic; the shell owns the transport and the placement. + const [voiceoverFlow, setVoiceoverFlow] = useState<{ maxDurationSec: number } | null>(null); + // The playhead as it was when RECORDING STARTED. Recording plays the video so + // the user can narrate what they see, which means the live playhead has moved + // on by the take's own length by the time the take ends — reading it then + // placed every voiceover one full take-length to the right of where it was + // spoken. Captured on the way in, used on the way out. + const voiceoverStartSecRef = useRef(0); + + const openVoiceoverFlow = useCallback(() => { + const doc = useProjectStore.getState().document; + if (!doc) return; + const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0); + const playhead = useProjectStore.getState().currentTimeSec; + voiceoverStartSecRef.current = playhead; + // Recording stops itself at the end of the timeline: a take can never + // outlive the video it was recorded over. + setVoiceoverFlow({ maxDurationSec: Math.max(0.5, total - playhead) }); + }, []); + + // Silences the timeline's own audio tracks for the duration of a take — see + // where it is passed to the preview. + const [voiceoverRecording, setVoiceoverRecording] = useState(false); + + const handleVoiceoverRecordingStart = useCallback(() => { + // Re-capture: the user may have scrubbed between opening the dialog and + // hitting Record, and playback starts from wherever the playhead is now. + voiceoverStartSecRef.current = useProjectStore.getState().currentTimeSec; + setVoiceoverRecording(true); + if (videoElement?.paused) void videoElement.play().catch(() => undefined); + }, [videoElement]); + + const handleVoiceoverRecordingStop = useCallback(() => { + setVoiceoverRecording(false); + videoElement?.pause(); + }, [videoElement]); + + const handleVoiceoverReady = useCallback( + async (assetId: string, durationSec: number) => { + setVoiceoverFlow(null); + await tl.addAudioTrack(assetId, voiceoverStartSecRef.current, { + kind: "voiceover", + durationSec, + spanSec: durationSec, + }); + }, + [tl], + ); + const pasteRegion = useCallback(async () => { const doc = useProjectStore.getState().document; if (!doc) return; @@ -786,6 +973,21 @@ export function NewEditorShell() { return; } + // Validate before building anything. The clipboard outlives the project, so + // a track copied in one project and pasted in another would reference an + // asset that only exists back where it came from — a pill that plays + // nothing and exports nothing. Audio is the only kind carrying a reference + // out of the document today; the next one belongs here too, rather than in + // its own branch below. + const referencedAssetId = (snapshot.region as { assetId?: unknown }).assetId; + if ( + typeof referencedAssetId === "string" && + !doc.assets.some((a) => a.id === referencedAssetId) + ) { + toast.error(te("regionClipboard.pasteAssetMissing")); + return; + } + const { anchorRegionsWithDerivedMs } = await import("@/lib/ai-edition/timeline/timelineMap"); const { createId } = await import("@/lib/ai-edition/document/ids"); @@ -793,6 +995,27 @@ export function NewEditorShell() { const timeMs = Math.round(useProjectStore.getState().currentTimeSec * 1000); const src = snapshot.region as { startMs: number; endMs: number }; const prefix = snapshot.kind === "annotation" ? "ann" : snapshot.kind; + + // Audio re-ventilates through its own anchorer, which advances each + // fragment's source offset — the generic one would copy the offset into + // every fragment and restart the file at each cut. + if (snapshot.kind === "audio") { + const { placeAudioTrackInDocument } = await import("@/lib/ai-edition/document/audioTracks"); + const track = { + ...(snapshot.region as unknown as AxcutAudioTrack), + id: createId("audio"), + trackId: undefined, + startMs: timeMs, + endMs: timeMs + (Number(src.endMs) - Number(src.startMs)), + }; + // Pasting onto an occupied lane queues behind what is there rather than doubling + // the row — the same rule every other placement obeys (issue #560). + const next = placeAudioTrackInDocument(doc, track, () => createId("audio"), "create"); + if (next === doc) return; + await saveDocument(next, { history: true }); + toast.success("Region pasted"); + return; + } const pasted = { ...snapshot.region, id: createId(prefix), @@ -842,7 +1065,7 @@ export function NewEditorShell() { // `tl` belongs here now that the trim branch calls tl.addTrim: useTimeline // returns a fresh object each render, so memoizing on saveDocument alone // would paste through a callback holding a stale document. - }, [saveDocument, tl]); + }, [saveDocument, tl, te]); // Copy the SELECTED pill. Reads the same arrays the lanes render, so what gets // copied is what the user is looking at — the old version dug into the raw @@ -860,7 +1083,7 @@ export function NewEditorShell() { // (properties kept, position taken from the playhead). if (sel.kind === "trim") { const { coalescedTrimGroups } = await import("@/lib/ai-edition/timeline/trim-mapping"); - const group = coalescedTrimGroups(tl.trimRanges, tl.clips).find((g) => + const group = coalescedTrimGroups(tl.trimRanges, tl.clips, tl.insertRanges ?? []).find((g) => g.ids.includes(sel.id), ); if (!group) return; @@ -870,6 +1093,22 @@ export function NewEditorShell() { return; } + // An audio track is stored as one fragment per clip it covers; the user + // copied the PILL, so collapse it back before it goes on the clipboard. + if (sel.kind === "audio") { + const { collapseTracksToPills, trackGroupId } = await import( + "@/lib/ai-edition/document/audioTracks" + ); + const [pill] = collapseTracksToPills( + tl.audioTracks.filter((t) => trackGroupId(t) === sel.id), + ); + if (!pill) return; + copyRegion({ kind: "audio", region: pill as unknown as Record }); + setCopiedClipId(null); + toast.success("Region copied"); + return; + } + const source = sel.kind === "zoom" ? tl.zoomRegions @@ -948,6 +1187,14 @@ export function NewEditorShell() { } if (tl.selection) { void tl.removeRegion(tl.selection.kind, tl.selection.id); + return; + } + // An audio track is selected through its OWN channel, not `selection` + // (the two are mutually exclusive — see addAudioTrack), so it needs its + // own branch here or Delete does nothing on the one lane that looks + // exactly like every other. + if (tl.selectedAudioTrackId) { + void tl.removeAudioTrack(tl.selectedAudioTrackId); } }; @@ -1032,6 +1279,18 @@ export function NewEditorShell() { void tl.addAnnotation(newRegionDurationSec()); return; } + // Unlike its neighbours this opens a file picker rather than dropping a region at + // the playhead — there is nothing to size, so it takes no duration (issue #350). + if (matchesShortcut(e, shortcuts.addAudio, isMac)) { + e.preventDefault(); + void tl.addAudio(); + return; + } + if (matchesShortcut(e, shortcuts.addVoiceover, isMac)) { + e.preventDefault(); + openVoiceoverFlow(); + return; + } if (matchesShortcut(e, shortcuts.addSpeed, isMac)) { e.preventDefault(); void tl.addSpeed(newRegionDurationSec()); @@ -1086,6 +1345,7 @@ export function NewEditorShell() { isMac, togglePlay, handleSeek, + openVoiceoverFlow, ]); const showTimeline = mode !== "rec"; @@ -1144,13 +1404,17 @@ export function NewEditorShell() { const transcriptProps = { clips, + audioTracks: document?.audioTracks ?? [], transcripts: document?.transcripts ?? [], assets: document?.assets ?? [], trimRanges: document?.timeline?.trimRanges ?? [], busyAssetIds, onSeek: handleSeek, - onAddTrimRange: handleAddTrimRange, - onRemoveTrimRange: handleRemoveTrimRange, + onTrimTimelineSpan: handleTrimTimelineSpan, + onRemoveTrimRanges: handleRemoveTrimRanges, + onSetWordText: handleSetWordText, + onInsertWord: handleInsertWord, + onRemoveWords: handleRemoveWords, onTranscribe: handleTranscribe, canTranscribe: hasAsset, isTranscribing: transcriptGate.state === "pending", @@ -1165,7 +1429,11 @@ export function NewEditorShell() { className={v4.app} style={{ gridTemplateRows: `58px 1fr ${showTimeline ? timelineRow : "0px"}` }} > - + void tl.commitZoomFocus()} @@ -1324,6 +1603,7 @@ export function NewEditorShell() { onTogglePlay={togglePlay} onPrevClip={handlePrevClip} onNextClip={handleNextClip} + onAddVoiceover={openVoiceoverFlow} onEditClip={setEditClipTarget} /> @@ -1383,6 +1663,21 @@ export function NewEditorShell() { onChoose={handleConfirmUnsaved} /> setExportOpen(false)} document={document} /> + { + // Belt and braces: the recorder's own stop handler clears this, but a + // flow that ends any other way must not leave the timeline muted. + setVoiceoverRecording(false); + setVoiceoverFlow(null); + }} + onComplete={(assetId, durationSec) => { + void handleVoiceoverReady(assetId, durationSec); + }} + onRecordingStart={handleVoiceoverRecordingStart} + onRecordingStop={handleVoiceoverRecordingStop} + /> ); } diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index aaf04f2db..2753f9635 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -3,7 +3,9 @@ import type { CameraFullscreenRegion, ZoomFocus } from "@/components/video-edito import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutAnnotationRegion, + AxcutAudioTrack, AxcutClip, + AxcutInsertRange, AxcutTrimRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; @@ -21,11 +23,18 @@ interface PreviewProps { hasProject: boolean; hasAsset: boolean; videoSources: VideoSource[]; + /** Imported audio tracks and the (unfiltered) asset URLs they resolve to + * (issue #350). Passed straight through to VirtualPreview — unlike the video + * `previewSources` below, these are NOT narrowed to clip-referenced assets, + * since an audio track has no clip. */ + audioTracks?: AxcutAudioTrack[]; + audioSources?: VideoSource[]; clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; + insertRanges?: AxcutInsertRange[]; selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; @@ -52,11 +61,14 @@ export function Preview({ hasProject, hasAsset, videoSources, + audioTracks = [], + audioSources = [], clips, zoomRegions, speedRegions, cameraFullscreenRegions, trimRanges, + insertRanges, selectedZoomRegionId, onZoomFocusChange, onZoomFocusCommit, @@ -178,11 +190,14 @@ export function Preview({ <> ; interface PreviewCanvasProps { videoSources: VideoSource[]; + /** Imported audio tracks + their asset URLs (issue #350), forwarded to + * VirtualPreview. */ + audioTracks?: AxcutAudioTrack[]; + audioSources?: VideoSource[]; clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; + insertRanges?: AxcutInsertRange[]; selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; @@ -214,17 +224,18 @@ export function PreviewCanvas(props: PreviewCanvasProps) { // clip the playhead is currently inside, the same lookup VirtualPreview // itself uses to map playback position back to a clip. `undefined` (no // crop stored) normalises to the identity region. + const previewInserts = props.insertRanges ?? EMPTY_INSERT_RANGES; const activeClip = useMemo( - () => locateVirtualPosition(props.clips, props.currentTimeSec)?.clip ?? null, - [props.clips, props.currentTimeSec], + () => locateVirtualPosition(props.clips, props.currentTimeSec, previewInserts)?.clip ?? null, + [props.clips, props.currentTimeSec, previewInserts], ); const cropRegion: CropRegion = activeClip?.cropRegion ?? DEFAULT_CROP_REGION; // P4 — the layout preset is global (one panel for the whole timeline) but the camera // is per clip, so the layout has to be resolved against the clip under the playhead. const activeCameraTrack = useMemo( - () => resolveActiveCameraTrack(assets, props.clips, props.currentTimeSec), - [assets, props.clips, props.currentTimeSec], + () => resolveActiveCameraTrack(assets, props.clips, props.currentTimeSec, previewInserts), + [assets, props.clips, props.currentTimeSec, previewInserts], ); const activeClipHasCamera = Boolean(activeCameraTrack?.visible && activeCameraTrack.sourcePath); diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 2e15f7b12..10a478186 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -7,20 +7,25 @@ import { AudioLines, + Captions as CaptionsIcon, ChevronDown, FileText, HelpCircle, Layout as LayoutIcon, Loader2, + Mic, MousePointerClick, + Music, Sliders, Trash2, + Undo2, + Video, } from "lucide-react"; import { type ChangeEvent, type CSSProperties, - type FormEvent, + Fragment, memo, type ClipboardEvent as ReactClipboardEvent, type KeyboardEvent as ReactKeyboardEvent, @@ -38,10 +43,15 @@ import defaultCursorPreviewUrl from "@/assets/cursors/Cursor=Default.svg"; import GradientEditor, { type GradientEditorState } from "@/components/ui/gradient-editor"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; +import { resolveCaptionLane } from "@/lib/ai-edition/captions/settings"; +import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks"; import { collectNativeFormats } from "@/lib/ai-edition/document/outputFormat"; +import type { InsertSide } from "@/lib/ai-edition/document/transcript"; import type { AxcutAsset, + AxcutAudioTrack, AxcutClip, + AxcutInsertRange, AxcutTranscript, AxcutTrimRange, AxcutWord, @@ -51,18 +61,26 @@ import { type EditorSettingsPatch, } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { useCaptions } from "@/lib/ai-edition/store/useCaptions"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; +import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { buildAggregatedSections, type ClipSection, type ClipWord, findCueWordId, + isInsertedWord, isSilenceWord, + placementRawExtent, + placementRawSec, + type TranscriptLane, type TrimRun, + voiceoverPlacements, } from "@/lib/ai-edition/timeline/aggregated-transcript"; import { hasAnyClipWithCamera } from "@/lib/ai-edition/timeline/camera"; import { formatMs } from "@/lib/ai-edition/timeline/format"; -import { locateVirtualPosition } from "@/lib/ai-edition/timeline/virtual-preview"; +import { takeInserts } from "@/lib/ai-edition/timeline/insert-mapping"; +import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time"; import type { TranscriptGateReason } from "@/lib/ai-edition/transcription/status"; import { getAssetPath } from "@/lib/assetPath"; import { resolveWebcamLayoutPreset, supportsWebcamReactiveZoom } from "@/lib/compositeLayout"; @@ -82,17 +100,24 @@ import { getAspectRatioLabel, } from "@/utils/aspectRatioUtils"; import { useCanSegmentCamera } from "../../native/hooks/useSegmentationSupport"; +import { CaptionsPane } from "./CaptionsPane"; import styles from "./NewEditorShell.module.css"; +/** Stable identity, so the memos below are not invalidated every render by a fresh `[]`. */ +const EMPTY_INSERT_RANGES: readonly AxcutInsertRange[] = []; + interface PaneProps { title: string; icon: ReactNode; // P3.3 — contextual help shown in a popover when the ? button is clicked. helpText: string; + // A control that belongs to the pane as a whole rather than to any one of its + // rows, sitting left of the Help button. + actions?: ReactNode; children: ReactNode; } -function Pane({ title, icon, helpText, children }: PaneProps) { +function Pane({ title, icon, helpText, actions, children }: PaneProps) { const ts = useScopedT("settings"); const helpLabel = ts("panes.help"); const [helpOpen, setHelpOpen] = useState(false); @@ -100,7 +125,8 @@ function Pane({ title, icon, helpText, children }: PaneProps) {

{title}

- + + {actions} + +
+ {/* Said out loud, because the choice reaches further than this tab: it decides the + text burnt into the exported file. A user must never be surprised by which + lane their captions came from. */} +

{ts("transcript.laneFeedsCaptions")}

+ + ); +} + +/** + * Caption settings, reached from the transcript tab (issue #560). + * + * The pane is reused VERBATIM rather than rebuilt into a popover body: it is ~600 + * lines of settings that already work, and "make it a popover" is a question about + * where it is mounted, not about what it contains. Rebuilding it would have been the + * one reliable way to arrive at a popover that is not at parity with the tab it + * replaces. + * + * Safe inside a Popover specifically because nothing in it takes focus away — no file + * input, no OS dialog. That is the trap `useWallpaperFileInput` documents above, and + * it is worth re-checking if a picker is ever added to captions. + */ +function CaptionSettingsButton() { + const ts = useScopedT("settings"); + const [open, setOpen] = useState(false); + return ( + + + + + + + + + ); +} + export function TranscriptPane({ clips, + audioTracks, transcripts, assets, trimRanges, busyAssetIds, onSeek, - onAddTrimRange, - onRemoveTrimRange, + onTrimTimelineSpan, + onRemoveTrimRanges, + onSetWordText, + onInsertWord, + onRemoveWords, onTranscribe, canTranscribe, isTranscribing, blocked, }: { clips: AxcutClip[]; + /** Every audio track on the timeline. Only the voiceover ones can be read from; + * music is not transcribed at all, so it never becomes a lane to choose. */ + audioTracks: AxcutAudioTrack[]; transcripts: AxcutTranscript[]; assets: AxcutAsset[]; trimRanges: AxcutTrimRange[]; @@ -720,8 +830,17 @@ export function TranscriptPane({ * background pass, with nothing on screen to say why. */ busyAssetIds: readonly string[]; onSeek: (sec: number) => void; - onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; - onRemoveTrimRange: (trimId: string) => void; + onTrimTimelineSpan: (startSec: number, endSec: number, reason: string) => void; + onRemoveTrimRanges: (trimIds: string[]) => void; + /** Rewrite ONE word's text. Takes the bare `AxcutWord.id`, never the clip-scoped + * `ClipWord.id`: the transcript belongs to the asset, so a correction lands on the + * media and shows on every clip that plays it — which is the point. */ + onSetWordText: (assetId: string, wordId: string, text: string) => void; + /** Add a word nobody said, beside the word the caret was resting on. Bare id, as above. */ + onInsertWord: (assetId: string, anchorWordId: string, side: InsertSide, text: string) => void; + /** Delete inserted words. Only ever called with `source: "synth"` ids — a transcribed + * word is cut with a trim, never deleted. */ + onRemoveWords: (assetId: string, wordIds: string[]) => void; onTranscribe: () => void; canTranscribe: boolean; isTranscribing: boolean; @@ -738,42 +857,86 @@ export function TranscriptPane({ // on `cueWordId`, so a frame that doesn't cross a word boundary re-renders nothing // but this component's own (cheap) lookup. const currentTimeSec = useProjectStore((s) => s.currentTimeSec); + + // Stored in the document, through the caption settings (issue #560). It was local + // state until the captions had to follow it — and the captions are burnt into the + // exported file by a path that never runs React, so a lane living here would caption + // the preview from one lane and the export from the other. + // + // `resolveCaptionLane` carries the fallback, in the pure layer for the same reason: + // deleting the last voiceover pill while reading it must not leave the pane, the + // preview and the exporter disagreeing about which lane that project has. + const { settings: captionSettings, set: setCaptionSettings } = useCaptions(); + const document = useProjectStore((s) => s.document); + // From the RECORDING clips and the whole trim set, never from `placements`: the + // programme is one thing, and the voiceover lane is asking whether the film still + // contains a moment — not whether some trim happens to name an audio fragment. + const insertRanges = document?.timeline.insertRanges ?? EMPTY_INSERT_RANGES; + const removed = useMemo( + () => removedRawSpans(clips, trimRanges, insertRanges), + [clips, trimRanges, insertRanges], + ); + // The take's placements are fed the cuts AND its own insertions, so a word after a pause + // is struck through — and highlighted — at the moment it is actually heard (issue #560). + const insertsFor = useCallback( + (groupId: string) => (document ? takeInserts(document, groupId) : []), + [document], + ); + const voiceover = useMemo( + () => voiceoverPlacements(audioTracks, removed, insertsFor), + [audioTracks, removed, insertsFor], + ); + const activeLane = resolveCaptionLane(document, captionSettings); + const setLane = useCallback( + (captionLane: TranscriptLane) => { + void setCaptionSettings({ captionLane }); + }, + [setCaptionSettings], + ); + const placements = activeLane === "voiceover" ? voiceover : clips; + const sections = useMemo( - () => buildAggregatedSections(clips, transcripts, assets, trimRanges), - [clips, transcripts, assets, trimRanges], + () => buildAggregatedSections(placements, transcripts, assets, removed, insertRanges), + [placements, transcripts, assets, removed, insertRanges], ); - // the cue position is the playback head's location in the current clip's source time. // `currentTimeSec` is the RAW/document timeline (same referential as the ruler, see - // NewEditorShell) — looked up against the raw `clips`, matching that referential. - // `clipId` is what `findCueWordId` keys on — do NOT drop it as unused: source time is - // per asset, so without it the resolver falls back to the first section of the asset - // and the cue tracks clip 1 forever on a timeline that plays one media twice. - const cue = useMemo(() => { - if (clips.length === 0) return null; - const position = locateVirtualPosition(clips, currentTimeSec); - if (!position) return null; - return { - assetId: position.clip.assetId, - clipId: position.clip.id, - sourceTimeSec: position.sourceTimeSec, - }; - }, [clips, currentTimeSec]); - - const cueWordId = useMemo(() => findCueWordId(sections, cue), [sections, cue]); + // NewEditorShell), which is exactly what `findCueWordId` now takes. It used to be + // resolved through `locateVirtualPosition` into a clip id + source second, and a clip + // id is something only the recording lane has — so the voiceover lane never + // highlighted. Raw seconds are the coordinate both lanes share. + const cueWordId = useMemo( + () => findCueWordId(sections, currentTimeSec, insertRanges), + [sections, currentTimeSec, insertRanges], + ); - const hasAnyTranscript = transcripts.length > 0; + const laneSwitch = + voiceover.length > 0 ? : null; + // Asked of the LANE, not the document: a project with a recording transcript and a + // freshly imported voiceover has transcripts, and the voiceover lane still has + // nothing to show — the empty state is what says so. + const hasAnyTranscript = sections.some((section) => section.transcript !== null); // Only silence is a dead end: every other reason (a retryable failure, no // engine, nothing attempted) leaves the button worth pressing. const silentMedia = blocked?.reason === "no-audio"; - if (clips.length === 0 || !hasAnyTranscript) { + // The insert gesture is dev-only until TTS (see openInsertion), so the copy follows + // the same gate: release builds must not advertise a dead gesture. + const helpText = + ts("transcript.help") + (import.meta.env.DEV ? ` ${ts("transcript.helpInsert")}` : ""); + const editingHint = ts( + import.meta.env.DEV ? "transcript.editingHintDev" : "transcript.editingHint", + ); + + if (placements.length === 0 || !hasAnyTranscript) { return ( } - helpText={ts("transcript.help")} + helpText={helpText} + actions={} > + {laneSwitch}

- {clips.length === 0 + {placements.length === 0 ? ts("transcript.noClips") : isTranscribing ? ts("transcript.transcribing") @@ -817,25 +980,44 @@ export function TranscriptPane({ } return ( -

-
-

{ts("transcript.title")}

-
-
- {sections.map((section, idx) => ( - - ))} -
-
+ } + helpText={helpText} + actions={} + > + {laneSwitch} + {/* The gestures are invisible until tried: nothing on a plain word stream says + * that double-click corrects and Backspace cuts. One muted line names them; the + * ? popover above carries the long version (amber inserts, hover-bin restore). */} +

+ {editingHint} +

+ {sections.map((section, idx) => ( + + ))} + ); } @@ -843,7 +1025,7 @@ export function TranscriptPane({ // range) and a flowing word stream. The stream contains every transcript // word inside the clip's source range, color-coded by whether the word // is inside any trimRange. Backspace/Delete adds a new trimRange via -// onAddTrimRange; hover-bin on a skip run removes it via onRemoveTrimRange. +// onTrimTimelineSpan; hover-bin on a skip run removes it via onRemoveTrimRanges. // // `memo` matters here: this renders one DOM node per transcript word, and its // parent now re-renders on every playhead tick (~60×/s during playback). The only @@ -855,26 +1037,54 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ index, section, busy, + lane, cueWordId, + insertRanges, onSeek, - onAddTrimRange, - onRemoveTrimRange, + onTrimTimelineSpan, + onRemoveTrimRanges, + onSetWordText, + onInsertWord, + onRemoveWords, }: { index: number; section: ClipSection; busy: boolean; + /** Which lane the block belongs to. Only the insert gesture cares: a pause is a held + * CLIP frame, and a voiceover placement has no clip to hold. */ + lane: TranscriptLane; cueWordId: string | null; + /** The insertions this placement carries: the block maps its words' SOURCE spans onto + * the ruler to author a cut, and a clip carrying insertions is longer than its source + * window (issue #560). */ + insertRanges: readonly AxcutInsertRange[]; onSeek: (sec: number) => void; - onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; - onRemoveTrimRange: (trimId: string) => void; + onTrimTimelineSpan: (startSec: number, endSec: number, reason: string) => void; + onRemoveTrimRanges: (trimIds: string[]) => void; + onSetWordText: (assetId: string, wordId: string, text: string) => void; + onInsertWord: (assetId: string, anchorWordId: string, side: InsertSide, text: string) => void; + onRemoveWords: (assetId: string, wordIds: string[]) => void; }) { const ts = useScopedT("settings"); const { clip, asset, words } = section; // Memoised: `TranscriptWord` renders once per word, so a fresh object literal here // would break referential equality for the whole stream on every parent render. - const trimTarget = useMemo( - () => ({ assetId: clip.assetId, clipId: clip.id }), - [clip.assetId, clip.id], + // A cut is authored in RAW seconds, CLAMPED to this placement's own extent. + // `wordsInRange` admits a word by OVERLAP and consecutive fragments have touching + // source windows, so a word straddling an edge would otherwise produce a span reaching + // past this placement — and `ventilateTimelineSpanToTrims` walks every clip a span + // touches, so the overspill would cut the head of a neighbouring clip that has nothing + // to do with the word the user deleted. + const toRawSpan = useCallback( + (startSec: number, endSec: number): [number, number] => { + const extent = placementRawExtent(clip, insertRanges); + const lo = extent?.startSec ?? clip.timelineStartSec; + const hi = extent?.endSec ?? Number.POSITIVE_INFINITY; + const clamp = (sec: number) => + Math.min(Math.max(placementRawSec(clip, sec, insertRanges), lo), hi); + return [clamp(startSec), clamp(endSec)]; + }, + [clip, insertRanges], ); const filename = asset?.label ?? clip.assetId; const sourceRangeLabel = @@ -939,25 +1149,38 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // Only skip words that are currently kept (don't double-skip). const keptRange = rangeWords.filter((w) => w.kept); if (keptRange.length === 0) return; + // An inserted word has no audio to cut, so Backspace deletes it outright. Only a + // range made entirely of inserts takes this path: mixed with spoken words the trim + // covers them anyway — they sit inside its span and read as cut, which is what the + // keystroke asked for. + if (keptRange.every((w) => isInsertedWord(w.word))) { + onRemoveWords( + clip.assetId, + keptRange.map((w) => w.word.id), + ); + return; + } pendingCaretWordIdRef.current = keptRange[0].id; const startSec = Math.min(...keptRange.map((w) => w.word.startSec)); const endSec = Math.max(...keptRange.map((w) => w.word.endSec)); - onAddTrimRange( - trimTarget, - startSec, - endSec, + onTrimTimelineSpan( + ...toRawSpan(startSec, endSec), `Skip ${formatMs(startSec * 1000)}-${formatMs(endSec * 1000)} from ${clip.assetId}.`, ); }, - [busy, clip.assetId, trimTarget, onAddTrimRange], + [busy, clip.assetId, toRawSpan, onTrimTimelineSpan, onRemoveWords], ); const removeTrimRun = useCallback( (run: TrimRun) => { - if (busy || !run.trimId) return; - onRemoveTrimRange(run.trimId); + // An empty set is a gap between clips: removed from the film, but by nothing + // there is a pill for. Otherwise every row goes at once — a cut ventilated across + // a clip boundary is several rows and ONE pill, and dropping half of it would + // leave the word still cut with nothing left on screen to say so. + if (busy || run.trimIds.length === 0) return; + onRemoveTrimRanges(run.trimIds); }, - [busy, onRemoveTrimRange], + [busy, onRemoveTrimRanges], ); const cutNativeSelection = useCallback( @@ -1012,38 +1235,98 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ [cutNativeSelection], ); - const handleBeforeInput = useCallback( - (event: FormEvent) => { - const inputEvent = event.nativeEvent as InputEvent; - if (inputEvent.inputType.startsWith("delete")) { + // The word an insert will sit beside, and what has been typed into it so far. Held on + // the block rather than the word, because the field belongs BETWEEN two words: the id is + // only how it finds its place in the stream. + const [insertion, setInsertion] = useState<{ + clipWordId: string; + side: InsertSide; + draft: string; + } | null>(null); + const insertionAbandonedRef = useRef(false); + + const openInsertion = useCallback( + (seed: string) => { + // ponytail: word insertion ships dev-only until a voice can be synthesized for + // the word — without one it only borrows free silence, and once it creates + // timeline time (the pause gesture) it is a silent freeze frame. Drop this gate + // when TTS lands. + if (!import.meta.env.DEV) return; + if (busy || !seed.trim()) return; + const editor = editorRef.current; + const selection = globalThis.getSelection(); + if (!editor || !selection) return; + if (!editor.contains(selection.anchorNode)) return; + const caret = findInsertionAnchor(editor, selection.anchorNode, selection.anchorOffset); + if (!caret) return; + const anchor = resolveInsertionAnchor(words, caret.clipWordId, caret.side); + if (!anchor) return; + setInsertion({ ...anchor, draft: seed }); + }, + [busy, lane, ts, words], + ); + + const commitInsertion = useCallback(() => { + const pending = insertion; + setInsertion(null); + if (!pending) return; + const text = pending.draft.trim(); + if (!text) return; + const anchor = words.find((w) => w.id === pending.clipWordId); + if (!anchor) return; + onInsertWord(clip.assetId, anchor.word.id, pending.side, text); + }, [insertion, words, onInsertWord, clip.assetId]); + + // Attached to the DOM, not through React's `onBeforeInput`. + // + // React 18 does not build that synthetic event from the native `beforeinput`: it + // derives it from the legacy `textInput`, whose event object is a `TextEvent` and + // carries no `inputType` at all. So the guard that was supposed to keep typed text out + // of the projection threw `Cannot read properties of undefined (reading 'startsWith')` + // on every character, never reached its own `preventDefault`, and let the character + // land in the contentEditable — the exact desynchronisation between the DOM and `words` + // it was written to prevent. Verified in the browser before this was moved. + // + // The native event is a real `InputEvent`, its `inputType` is the thing both branches + // switch on, and preventing it actually stops the browser. + useEffect(() => { + const editor = editorRef.current; + if (!editor) return; + const onBeforeInput = (event: InputEvent) => { + // The word editor and the insertion field are ``s INSIDE this element, so + // their own typing bubbles here natively — React's `stopPropagation` only ever + // stopped the synthetic tree. Their text is theirs. + if (event.target instanceof HTMLInputElement) return; + if (event.inputType.startsWith("delete")) { event.preventDefault(); - cutNativeSelection( - inputEvent.inputType === "deleteContentForward" ? "forward" : "backward", - ); + cutNativeSelection(event.inputType === "deleteContentForward" ? "forward" : "backward"); return; } - // Inserts are blocked to keep the projection stable: every run of text - // here maps back to a `transcript.words` entry by id, and free text has - // no id to land on. Deletion is fine because it goes through - // `cutNativeSelection`, which resolves the selection to word ids first. - // - // This used to defer to `SourceTranscriptModal`, deleted with the v3 - // media pane — it never got past read-only, so it was never the answer - // it was cited as. Editing a word's TEXT therefore has no in-app path - // today. Adding one means a word-level mutation alongside - // `skipWordRange`, reached from here; lifting this guard on its own - // would only desynchronise the DOM from `words`. - if (inputEvent.inputType === "insertText" || inputEvent.inputType === "insertFromPaste") { + // Free text never lands in the block itself: every run of text here maps back to a + // `transcript.words` entry by id, and typed characters have no id. What they open + // instead is a field beside the word the caret was on, whose commit creates a real + // word to hold them. So the gesture is the document one — put the caret somewhere + // and type — without the DOM ever getting ahead of `words`. + if (event.inputType.startsWith("insert")) { event.preventDefault(); + openInsertion(event.data ?? ""); } + }; + editor.addEventListener("beforeinput", onBeforeInput); + return () => editor.removeEventListener("beforeinput", onBeforeInput); + }, [cutNativeSelection, openInsertion]); + + const handlePaste = useCallback( + (event: ReactClipboardEvent) => { + // Handled here rather than through `insertFromPaste`: preventing the paste stops + // that beforeinput from ever firing, and this is the only place the clipboard text + // is still readable. + event.preventDefault(); + openInsertion(event.clipboardData.getData("text/plain")); }, - [cutNativeSelection], + [openInsertion], ); - const handlePaste = useCallback((event: ReactClipboardEvent) => { - event.preventDefault(); - }, []); - const handlePointerUp = useCallback( (event: ReactPointerEvent) => { if (event.button !== 0) return; @@ -1183,11 +1466,13 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ spellCheck={false} aria-label={ts("transcript.editorAria", { filename })} aria-multiline="true" - onBeforeInput={handleBeforeInput} onKeyDown={handleKeyDown} onPaste={handlePaste} onPointerUp={handlePointerUp} style={{ + // Inline so a split clip reads as one sentence rather than one line per + // piece. The block that fronts a run still owns the header above it. + display: "inline", padding: "4px 4px", font: "400 13px/1.65 var(--font-body)", color: "var(--fg)", @@ -1205,16 +1490,39 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // scrollbar that breaks the cue auto-scroll UX. }} > - {words.map((cw) => ( - - ))} + {words.map((cw) => { + const field = + insertion?.clipWordId === cw.id ? ( + setInsertion({ ...insertion, draft })} + onCommit={commitInsertion} + onCancel={() => { + insertionAbandonedRef.current = true; + setInsertion(null); + }} + abandonedRef={insertionAbandonedRef} + /> + ) : null; + return ( + + {insertion?.side === "before" ? field : null} + + {insertion?.side === "after" ? field : null} + + ); + })}
)} @@ -1238,24 +1546,70 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // the two words whose `isCue` actually flipped re-render. // // This holds because every other prop is referentially stable across a -// playhead tick: `cw` comes from the memoised `sections`, `target` from a +// playhead tick: `cw` comes from the memoised `sections`, `assetId` from a // `useMemo`, and both callbacks from `useCallback`s that do not depend on time. const TranscriptWord = memo(function TranscriptWord({ cw, isCue, - target, + editable, + assetId, + toRawSpan, onRestore, - onAddTrimRange, + onTrimTimelineSpan, + onSetWordText, + onRemoveWords, }: { cw: ClipWord; isCue: boolean; - target: TrimTarget; + /** False while this clip's transcript is being regenerated — the words on screen are + * about to be replaced, so an edit typed into them would be thrown away. */ + editable: boolean; + assetId: string; + /** Clamped source→raw for this word's placement — see `toRawSpan` above. */ + toRawSpan: (startSec: number, endSec: number) => [number, number]; onRestore: (run: TrimRun) => void; - onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; + onTrimTimelineSpan: (startSec: number, endSec: number, reason: string) => void; + onSetWordText: (assetId: string, wordId: string, text: string) => void; + onRemoveWords: (assetId: string, wordIds: string[]) => void; }) { const ts = useScopedT("settings"); const [hover, setHover] = useState(false); + // The text being typed, or null when the word is not under edit. + const [draft, setDraft] = useState(null); + // Escape unmounts the field, and an abandoned field's blur must not commit what the + // user just walked away from. + const abandonedRef = useRef(false); const removed = !cw.kept; + // `originalText` is only ever written by a user edit (see `document/transcript.ts`), so + // it is what tells a corrected word from a transcribed one. + const original = cw.word.originalText; + const corrected = original !== undefined; + const blanked = corrected && cw.word.text.trim().length === 0; + + const startEditing = useCallback(() => { + if (!editable) return; + setDraft(cw.word.text); + }, [editable, cw.word.text]); + + const commitDraft = useCallback(() => { + const next = (draft ?? "").trim(); + setDraft(null); + if (next === cw.word.text) return; + onSetWordText(assetId, cw.word.id, next); + }, [draft, cw.word.text, cw.word.id, onSetWordText, assetId]); + + const inserted = isInsertedWord(cw.word); + + const removeInserted = useCallback(() => { + onRemoveWords(assetId, [cw.word.id]); + }, [onRemoveWords, assetId, cw.word.id]); + + const revert = useCallback(() => { + if (original === undefined) return; + // Writing the original back through the same path is what clears the provenance + // pair — there is no separate "unedit" operation that could fall out of step. + onSetWordText(assetId, cw.word.id, original); + }, [original, cw.word.id, onSetWordText, assetId]); if (isSilenceWord(cw.word)) { const durationSec = cw.word.endSec - cw.word.startSec; @@ -1273,7 +1627,7 @@ const TranscriptWord = memo(function TranscriptWord({ onClick={(e) => { e.stopPropagation(); onRestore({ - trimId: cw.trimId ?? "", + trimIds: cw.trimIds, assetId: "", startWordIndex: 0, endWordIndex: 0, @@ -1308,10 +1662,8 @@ const TranscriptWord = memo(function TranscriptWord({ aria-label={ts("transcript.trimSilence", { duration })} onClick={(e) => { e.stopPropagation(); - onAddTrimRange( - target, - cw.word.startSec, - cw.word.endSec, + onTrimTimelineSpan( + ...toRawSpan(cw.word.startSec, cw.word.endSec), `Skip silence ${formatMs(cw.word.startSec * 1000)}-${formatMs(cw.word.endSec * 1000)}.`, ); }} @@ -1333,31 +1685,197 @@ const TranscriptWord = memo(function TranscriptWord({ ); } + // The inline editor. `contentEditable={false}` keeps the browser from treating it as + // part of the enclosing editable block, and every event it raises is stopped here rather + // than in the block handlers: Backspace inside the field has to type, not cut, and a + // click in it must not seek. + if (draft !== null) { + return ( + setDraft(event.target.value)} + onFocus={(event) => event.currentTarget.select()} + onBlur={() => { + if (abandonedRef.current) { + abandonedRef.current = false; + return; + } + commitDraft(); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") { + event.preventDefault(); + commitDraft(); + } else if (event.key === "Escape") { + event.preventDefault(); + abandonedRef.current = true; + setDraft(null); + } + }} + onPaste={(event) => event.stopPropagation()} + onPointerUp={(event) => event.stopPropagation()} + style={{ + display: "inline", + // `ch` is the digit width, not the real glyph width, so this only + // approximates the word it replaces — the slack keeps it from clipping. + width: `${Math.max(draft.length, 3) + 2}ch`, + margin: 0, + padding: "0 2px", + border: 0, + borderBottom: "2px solid var(--accent)", + borderRadius: 0, + background: "var(--accent-soft)", + color: "var(--fg)", + font: "inherit", + outline: "none", + }} + /> + ); + } + + // A word nobody said. Amber rather than the accent: this one is not a fix to what was + // heard, it is text with no sound underneath — the caveat is the point. Double-click + // rewrites it like any other word; the cross deletes it, because there is no audio for a + // trim to remove. + if (inserted) { + return ( + setHover(true)} + onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} + > + + {cw.word.text} + + {hover ? ( + + + ) : null}{" "} + + ); + } + + // A word the user emptied. It still owns a span of the media, so it keeps a place in + // the stream: rendered as its own (empty) text it would be a bare space — invisible, + // impossible to click, and therefore impossible to undo. + if (blanked) { + return ( + setHover(true)} + onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} + > + + {ts("transcript.blankedWord")} + + {hover ? ( + + ) : null}{" "} + + ); + } + return ( setHover(true)} onMouseLeave={() => setHover(false)} + onDoubleClick={(e) => { + // Without this the browser selects the word inside the enclosing + // contentEditable; the field about to replace it does its own selecting. + e.preventDefault(); + e.stopPropagation(); + startEditing(); + }} > {/* no filler chip. axcut renders every word the same way; the LLM is the only place that names a word a filler (via the filler_or_hesitation reason when generating suggestions). */} {cw.word.text}{" "} - {removed && hover && cw.trimId ? ( + {removed && hover && cw.trimIds.length > 0 ? ( ) : null} + {/* A cut word's bin already restores it — showing the revert beside it would put + two undos for two different things one pixel apart. */} + {!removed && corrected && hover ? ( + + ) : null} ); }); +/** Hover affordance on a corrected word: put the transcriber's own text back. Mirrors the + * bin on a cut word — same size, same place, the accent rather than the danger colour, + * since reverting a correction restores something instead of removing it. */ +function RevertWordButton({ label, onRevert }: { label: string; onRevert: () => void }) { + return ( + + + ); +} + +/** The one hover control shape the word stream uses, in whichever colour says what it does. + * `contentEditable={false}` keeps it out of the enclosing editable block, and the click is + * stopped so it never reaches the seek handler underneath. */ +function WordChipButton({ + label, + tone, + onPress, + children, +}: { + label: string; + tone: string; + onPress: () => void; + children: ReactNode; +}) { + return ( + + ); +} + +/** + * The field a typed character opens between two words. It is not a word yet — nothing is + * written until it commits — so it carries no `data-word-id` and no place in `words`. + * + * Every event it raises is stopped at the field, for the same reason the word editor stops + * its own: the block around it reads Backspace as a cut and a click as a seek. + */ +function InsertionField({ + value, + label, + onChange, + onCommit, + onCancel, + abandonedRef, +}: { + value: string; + label: string; + onChange: (value: string) => void; + onCommit: () => void; + onCancel: () => void; + abandonedRef: { current: boolean }; +}) { + return ( + onChange(event.target.value)} + onBlur={() => { + if (abandonedRef.current) { + abandonedRef.current = false; + return; + } + onCommit(); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") { + event.preventDefault(); + onCommit(); + } else if (event.key === "Escape") { + event.preventDefault(); + onCancel(); + } + }} + onBeforeInput={(event) => event.stopPropagation()} + onPaste={(event) => event.stopPropagation()} + onPointerUp={(event) => event.stopPropagation()} + style={{ + display: "inline", + width: `${Math.max(value.length, 3) + 2}ch`, + margin: "0 3px 2px 0", + padding: "0 5px", + border: "1px solid var(--warn)", + borderRadius: 999, + background: "var(--warn-soft)", + color: "var(--fg)", + font: "inherit", + outline: "none", + }} + /> + ); +} + // ─── Caret / selection helpers ──────────────────────────────────── // Ponytail port of axcut's findCollapsedDeletionWordId. The non-collapsed // path uses findWordId directly (a range selection's endpoints already @@ -1422,7 +2068,7 @@ function findCollapsedDeletionWordId( ): string | null { // read the kept/skip state from the words array, not the // DOM's data-skip-id. The DOM may be lagging a render behind (its - // trimId is only set on the next React commit), so a DOM check would + // skip id is only set on the next React commit), so a DOM check would // re-trim an already-trimmed word. The words array is the React state // captured at the call site — always current. const skippedIds = new Set(words.filter((w) => !w.kept).map((w) => w.id)); @@ -1492,6 +2138,79 @@ function findCollapsedDeletionWordId( return pool.find((wordNode) => isKept(wordNode.dataset.wordId ?? null))?.dataset.wordId ?? null; } +/** + * Where a typed character goes: beside the word the caret was resting on, never inside it. + * + * A caret in the middle of a word anchors AFTER that word rather than splitting it in two — + * a split would need two words where the transcript has one, and neither half would own the + * audio any more. At the very start of the block there is nothing to sit after, so the + * anchor is the first word and the new one lands before it. + */ +function findInsertionAnchor( + editor: HTMLElement, + node: Node | null, + offset: number, +): { clipWordId: string; side: InsertSide } | null { + const wordNodes = Array.from(editor.querySelectorAll("[data-word-id]")); + if (wordNodes.length === 0 || !node) return null; + + const direct = closestWordElement(node); + if (direct?.dataset.wordId) { + const atStart = node.nodeType === Node.TEXT_NODE && offset <= 0; + return { clipWordId: direct.dataset.wordId, side: atStart ? "before" : "after" }; + } + + // The caret is between the block's own children, and `offset` is a child index — the + // same shape `findCollapsedDeletionWordId` reads when it resolves a cut. Walk back for + // the word to sit after; if there is none, the caret is at the head of the stream and + // the new word goes before the first word ahead of it. + const childNodes = Array.from(node.childNodes); + for (const candidate of childNodes.slice(0, clampRangeOffset(node, offset)).reverse()) { + const wordId = findWordId(candidate) ?? findDescendantWordId(candidate); + if (wordId) return { clipWordId: wordId, side: "after" }; + } + for (const candidate of childNodes.slice(clampRangeOffset(node, offset))) { + const wordId = findWordId(candidate) ?? findDescendantWordId(candidate); + if (wordId) return { clipWordId: wordId, side: "before" }; + } + const first = wordNodes[0]; + return first?.dataset.wordId ? { clipWordId: first.dataset.wordId, side: "before" } : null; +} + +/** + * Pull the DOM's answer back onto a word the TRANSCRIPT has. + * + * `[silence]` pills carry a `data-word-id` like everything else in the stream, but they are + * pseudo-words `withSilenceGaps` invents per clip — there is nothing in `transcript.words` + * for a new word to be inserted next to. So the anchor walks off a silence to the nearest + * real word in the direction the caret was already facing, and only crosses to the other + * side when that direction runs out of stream. + */ +function resolveInsertionAnchor( + words: ClipWord[], + clipWordId: string, + side: InsertSide, +): { clipWordId: string; side: InsertSide } | null { + const from = words.findIndex((w) => w.id === clipWordId); + if (from < 0) return null; + const real = (index: number) => + index >= 0 && index < words.length && !isSilenceWord(words[index].word); + if (side === "after") { + for (let i = from; i >= 0; i--) if (real(i)) return { clipWordId: words[i].id, side: "after" }; + for (let i = 0; i < words.length; i++) { + if (real(i)) return { clipWordId: words[i].id, side: "before" }; + } + return null; + } + for (let i = from; i < words.length; i++) { + if (real(i)) return { clipWordId: words[i].id, side: "before" }; + } + for (let i = words.length - 1; i >= 0; i--) { + if (real(i)) return { clipWordId: words[i].id, side: "after" }; + } + return null; +} + function findDescendantWordId(node: Node): string | null { if (node instanceof HTMLElement && node.dataset.wordId) { return node.dataset.wordId; @@ -2303,6 +3022,166 @@ export function AudioPane() { ); } +type TimelineApi = ReturnType; + +// Per-track controls for the selected imported audio track (issue #350). Shown by +// the inspector in place of the facet when an audio track is selected (see +// FloatingInspector). The header is the generic "Audio track"; the body leads +// with the file name, then the volume (a local live value during the drag, +// committed as one undo step on release), then a delete button styled like the +// region panes' (position and mute are edited on the lane itself). +// Longest fade the inspector offers. Past a few seconds a fade stops reading as +// a fade and starts reading as a level change, and the track's own span caps it +// anyway (`resolveFadeSecs` reduces one that does not fit). +const FADE_MAX_MS = 5000; + +export function AudioTrackPane({ tl }: { tl: TimelineApi }) { + const ts = useScopedT("settings"); + const trackId = tl.selectedAudioTrackId; + // The document stores one clip-anchored fragment per clip the track covers; + // the inspector edits the user-visible TRACK, so collapse first. Editing a + // single fragment would let the halves of a split take disagree. + const track = trackId + ? collapseTracksToPills(tl.audioTracks.filter((t) => trackGroupId(t) === trackId))[0] + : undefined; + const asset = track ? tl.assets.find((a) => a.id === track.assetId) : undefined; + // Live-drag values; null means "show the committed value". + const [liveGain, setLiveGain] = useState(null); + const [liveFadeIn, setLiveFadeIn] = useState(null); + const [liveFadeOut, setLiveFadeOut] = useState(null); + // Drop the live value when the selected track changes: a drag released outside + // the input never fires onCommit, so without this an uncommitted -10 dB from + // track A would show as track B's gain the moment B is selected. + // biome-ignore lint/correctness/useExhaustiveDependencies: trackId is the trigger, not a read — the body only resets the live value. + useEffect(() => { + setLiveGain(null); + setLiveFadeIn(null); + setLiveFadeOut(null); + }, [trackId]); + if (!track) return null; + const fileName = track.label || asset?.label || asset?.originalPath?.split(/[\\/]/).pop() || ""; + + // Match the region panes' danger-outlined delete button (see SelectionPane). + const deleteBtnStyle: CSSProperties = { + display: "flex", + width: "100%", + alignItems: "center", + justifyContent: "center", + gap: 7, + padding: "9px 14px", + borderRadius: 10, + border: "1px solid var(--danger)", + background: "var(--danger-soft)", + color: "var(--danger)", + font: "600 13px var(--font-display)", + cursor: "pointer", + }; + + return ( + } + helpText={ts("audioTrack.help")} + > +
+ {fileName} +
+
+ setLiveGain(value)} + onCommit={() => { + if (liveGain !== null) void tl.setAudioTrackGain(track.id, liveGain); + setLiveGain(null); + }} + /> + { + if (liveFadeIn !== null) void tl.updateAudioTrack(track.id, { fadeInMs: liveFadeIn }); + setLiveFadeIn(null); + }} + /> + { + if (liveFadeOut !== null) + void tl.updateAudioTrack(track.id, { fadeOutMs: liveFadeOut }); + setLiveFadeOut(null); + }} + /> +
+
+ {ts("audioTrack.mute")} + void tl.updateAudioTrack(track.id, { muted: v })} + /> +
+
+ {ts("audioTrack.loop")} + void tl.setAudioTrackLoop(track.id, v)} + /> +
+ + + + ); +} + // ─── Cursor ─────────────────────────────────────────────────────── function safeAssetUrl(relativePath: string): string { diff --git a/src/components/ai-edition/TranscriptPane.captions.test.tsx b/src/components/ai-edition/TranscriptPane.captions.test.tsx new file mode 100644 index 000000000..96aae2c17 --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.captions.test.tsx @@ -0,0 +1,107 @@ +// @vitest-environment jsdom +// Issue #560: captions used to be their own inspector tab, next to the transcript +// they are a view OF. Two tabs meant two entry points to the same background pass, +// and the caption one was the only one many people ever found. +// +// So the tab is gone and its pane hangs off the transcript tab instead. What that +// costs is reachability, and that is exactly what these assertions pin: the control +// is present in BOTH of the transcript pane's states — including the empty one, +// where a user with no transcript would otherwise have no way back to caption +// settings at all — and it really does open the pane, not a rebuilt stub of it. + +import "@testing-library/jest-dom"; +import { cleanup, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutClip, AxcutTranscript } from "@/lib/ai-edition/schema"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 12, + cameraTrack: null, +}; + +const CLIPS: AxcutClip[] = [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 12, + timelineStartSec: 0, + timelineEndSec: 12, + wordRefs: [], + origin: "user", + reason: "", + }, +]; + +const TRANSCRIPT: AxcutTranscript = { + assetId: "asset_1", + language: "en", + words: [ + { id: "w_1", text: "hello", startSec: 0.2, endSec: 0.6 }, + { id: "w_2", text: "there", startSec: 0.6, endSec: 1.1 }, + ], + segments: [], +} as unknown as AxcutTranscript; + +function mount(transcripts: AxcutTranscript[]) { + return render( + + + , + ); +} + +afterEach(() => { + cleanup(); +}); + +describe("caption settings on the transcript tab", () => { + it("is reachable before any transcript exists", () => { + mount([]); + expect(screen.getByRole("button", { name: "Captions" })).toBeInTheDocument(); + // The single transcription gate stays where it was: on this pane, not + // hidden one popover deep. + expect(screen.getByRole("button", { name: "Transcribe now" })).toBeInTheDocument(); + }); + + it("is reachable once there is a transcript to caption", () => { + mount([TRANSCRIPT]); + expect(screen.getByRole("button", { name: "Captions" })).toBeInTheDocument(); + }); + + it("opens the real caption settings rather than a stub", async () => { + const user = userEvent.setup(); + mount([TRANSCRIPT]); + await user.click(screen.getByRole("button", { name: "Captions" })); + // A control that only the actual CaptionsPane renders — proof the pane was + // mounted whole rather than reimplemented into the popover. + expect(await screen.findByText("Show captions")).toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/TranscriptPane.gating.test.tsx b/src/components/ai-edition/TranscriptPane.gating.test.tsx index 008c6161f..b3ed5f4b9 100644 --- a/src/components/ai-edition/TranscriptPane.gating.test.tsx +++ b/src/components/ai-edition/TranscriptPane.gating.test.tsx @@ -49,13 +49,17 @@ function renderPane( ('[role="textbox"]'); if (!editor) throw new Error("transcript editor not rendered"); - return { ...view, editor, onAddTrimRange }; + return { ...view, editor, onTrimTimelineSpan }; } /** @@ -109,10 +113,11 @@ function caretBeforeWordAt(editor: HTMLElement, index: number) { selection?.addRange(range); } -/** The words the pane cut, as `[startSec, endSec]` — what `onAddTrimRange` was asked for. */ -function cutRange(onAddTrimRange: ReturnType): [number, number] | null { - const call = onAddTrimRange.mock.calls.at(-1); - return call ? [call[1] as number, call[2] as number] : null; +/** The RAW span the pane cut — what `onTrimTimelineSpan` was asked for. There is no + * leading target any more: a cut names a moment of the programme, not an owner. */ +function cutRange(onTrimTimelineSpan: ReturnType): [number, number] | null { + const call = onTrimTimelineSpan.mock.calls.at(-1); + return call ? [call[0] as number, call[1] as number] : null; } beforeEach(() => { @@ -127,29 +132,29 @@ afterEach(() => { describe("keyboard cut with the caret between words", () => { it("Backspace cuts the word before the caret", () => { // The ordinary case, and the one that already worked: nothing trimmed yet. - const { editor, onAddTrimRange } = renderPane([]); + const { editor, onTrimTimelineSpan } = renderPane([]); caretBeforeWordAt(editor, 3); // before "quatre" fireEvent.keyDown(editor, { key: "Backspace" }); - expect(cutRange(onAddTrimRange)).toEqual([2, 3]); // "trois" + expect(cutRange(onTrimTimelineSpan)).toEqual([2, 3]); // "trois" }); it("keeps cutting while ANOTHER asset is being transcribed", () => { // The background pass runs on its own now, so a run on some other media must // not quietly turn this block into an editor that ignores Backspace — the // read-only state is scoped to the asset whose transcript is being rewritten. - const { editor, onAddTrimRange } = renderPane([], vi.fn(), ["asset_other"]); + const { editor, onTrimTimelineSpan } = renderPane([], vi.fn(), ["asset_other"]); caretBeforeWordAt(editor, 3); fireEvent.keyDown(editor, { key: "Backspace" }); - expect(cutRange(onAddTrimRange)).toEqual([2, 3]); + expect(cutRange(onTrimTimelineSpan)).toEqual([2, 3]); }); it("stops cutting, visibly, while THIS asset is being transcribed", () => { // Its transcript is about to be replaced, so the block is read-only — and it // says so, instead of swallowing the keystroke in silence. - const { editor, onAddTrimRange, getByText } = renderPane([], vi.fn(), ["asset_1"]); + const { editor, onTrimTimelineSpan, getByText } = renderPane([], vi.fn(), ["asset_1"]); caretBeforeWordAt(editor, 3); fireEvent.keyDown(editor, { key: "Backspace" }); - expect(cutRange(onAddTrimRange)).toBeNull(); + expect(cutRange(onTrimTimelineSpan)).toBeNull(); expect(editor).toHaveAttribute("aria-busy", "true"); expect(getByText("Transcribing…")).toBeInTheDocument(); }); @@ -159,36 +164,36 @@ describe("keyboard cut with the caret between words", () => { // immediately before the caret has nothing left to cut. The keystroke used to // resolve to it anyway, `skipWordRange` dropped it as not-kept, and the user got // silence — they had to click elsewhere to carry on. - const { editor, onAddTrimRange } = renderPane([W2_TRIMMED]); + const { editor, onTrimTimelineSpan } = renderPane([W2_TRIMMED]); caretBeforeWordAt(editor, 2); // before "trois", i.e. right after the trimmed "deux" fireEvent.keyDown(editor, { key: "Backspace" }); - expect(cutRange(onAddTrimRange)).toEqual([0, 1]); // "un" — the nearest word still there + expect(cutRange(onTrimTimelineSpan)).toEqual([0, 1]); // "un" — the nearest word still there }); it("Delete skips over an already-trimmed word instead of doing nothing", () => { // The mirror case going forward. It had no guard at all: the candidate walk simply // returned the first word it met, trimmed or not. - const { editor, onAddTrimRange } = renderPane([W2_TRIMMED]); + const { editor, onTrimTimelineSpan } = renderPane([W2_TRIMMED]); caretBeforeWordAt(editor, 1); // before the trimmed "deux" fireEvent.keyDown(editor, { key: "Delete" }); - expect(cutRange(onAddTrimRange)).toEqual([2, 3]); // "trois" + expect(cutRange(onTrimTimelineSpan)).toEqual([2, 3]); // "trois" }); it("does nothing when every word in that direction is already trimmed", () => { // Not a regression — there is genuinely nothing left to cut, so no document write. - const { editor, onAddTrimRange } = renderPane([ + const { editor, onTrimTimelineSpan } = renderPane([ { ...W2_TRIMMED, id: "t_head", startSec: 0, endSec: 2 }, ]); caretBeforeWordAt(editor, 2); // before "trois"; "un" and "deux" are both gone fireEvent.keyDown(editor, { key: "Backspace" }); - expect(onAddTrimRange).not.toHaveBeenCalled(); + expect(onTrimTimelineSpan).not.toHaveBeenCalled(); }); it("cuts nothing when the caret is at the very start and Backspace is pressed", () => { - const { editor, onAddTrimRange } = renderPane([]); + const { editor, onTrimTimelineSpan } = renderPane([]); caretBeforeWordAt(editor, 0); fireEvent.keyDown(editor, { key: "Backspace" }); - expect(onAddTrimRange).not.toHaveBeenCalled(); + expect(onTrimTimelineSpan).not.toHaveBeenCalled(); }); // The story the whole thing exists for, in the shape the user meets it: the tests above @@ -207,12 +212,13 @@ describe("keyboard cut with the caret between words", () => { + onTrimTimelineSpan={(startSec: number, endSec: number) => setTrims((prev) => [ ...prev, { @@ -226,7 +232,10 @@ describe("keyboard cut with the caret between words", () => { }, ]) } - onRemoveTrimRange={vi.fn()} + onRemoveTrimRanges={vi.fn()} + onSetWordText={vi.fn()} + onInsertWord={vi.fn()} + onRemoveWords={vi.fn()} onTranscribe={vi.fn()} canTranscribe isTranscribing={false} diff --git a/src/components/ai-edition/TranscriptPane.lanes.test.tsx b/src/components/ai-edition/TranscriptPane.lanes.test.tsx new file mode 100644 index 000000000..e84868f3e --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.lanes.test.tsx @@ -0,0 +1,187 @@ +// @vitest-environment jsdom +// Issue #560: the transcript tab reads ONE lane, and which one is the user's +// choice. These pin the two halves of that: the switch only exists when there is +// somewhere to switch to, and choosing actually changes what the tab is reading — +// not what it is showing of the same thing. + +import "@testing-library/jest-dom"; +import { cleanup, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { + AxcutAsset, + AxcutAudioTrack, + AxcutClip, + AxcutTranscript, +} from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); + +const ASSETS: AxcutAsset[] = [ + { + id: "asset_rec", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 12, + cameraTrack: null, + }, + { + id: "asset_vo", + kind: "audio", + label: "voiceover.mp3", + originalPath: "/vo.mp3", + durationSec: 30, + cameraTrack: null, + }, +]; + +const CLIPS: AxcutClip[] = [ + { + id: "clip_1", + assetId: "asset_rec", + sourceStartSec: 0, + sourceEndSec: 12, + timelineStartSec: 0, + timelineEndSec: 12, + wordRefs: [], + origin: "user", + reason: "", + }, +]; + +function words(...texts: string[]) { + return texts.map((text, i) => ({ + id: `w${i}`, + segmentId: "s", + text, + startSec: i * 0.5, + endSec: i * 0.5 + 0.4, + })); +} + +const TRANSCRIPTS = [ + { assetId: "asset_rec", language: "en", words: words("filmed", "words"), segments: [] }, + { assetId: "asset_vo", language: "en", words: words("narrated", "words"), segments: [] }, +] as unknown as AxcutTranscript[]; + +const VOICEOVER: AxcutAudioTrack = { + id: "track_1", + startMs: 0, + endMs: 4000, + clipId: "clip_1", + sourceStartSec: 0, + sourceEndSec: 4, + assetId: "asset_vo", + kind: "voiceover", + durationSec: 30, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "", + origin: "user", +} as unknown as AxcutAudioTrack; + +function mount(audioTracks: AxcutAudioTrack[]) { + // The lane lives in the DOCUMENT now (issue #560), so the switch needs one to write + // to — it is no longer a piece of component state that answers on its own. + useProjectStore.setState({ + projectId: "proj_1", + document: { + schemaVersion: 7, + project: { + id: "proj_1", + title: "T", + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z", + primaryAssetId: "asset_rec", + }, + assets: ASSETS, + transcript: null, + transcripts: TRANSCRIPTS, + timeline: { + clips: CLIPS, + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + audioTracks, + legacyEditor: null, + // biome-ignore lint/suspicious/noExplicitAny: fixture, not a schema exercise + } as any, + status: "ready", + error: null, + dirty: false, + }); + render( + + + , + ); +} + +afterEach(() => { + cleanup(); + useProjectStore.getState().clear(); +}); + +describe("transcript lane switch", () => { + it("stays out of the way when there is no voiceover to switch to", () => { + mount([]); + expect(screen.queryByRole("group", { name: "Read the transcript from" })).toBeNull(); + expect(screen.getByText("filmed", { exact: false })).toBeInTheDocument(); + }); + + it("appears once a voiceover is on the timeline, reading the recording first", () => { + mount([VOICEOVER]); + expect(screen.getByRole("group", { name: "Read the transcript from" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Recording" })).toHaveAttribute( + "aria-pressed", + "true", + ); + expect(screen.getByText("filmed", { exact: false })).toBeInTheDocument(); + }); + + it("reads the voiceover's own words once chosen", async () => { + const user = userEvent.setup(); + mount([VOICEOVER]); + await user.click(screen.getByRole("button", { name: "Voice-over" })); + expect(await screen.findByText("narrated", { exact: false })).toBeInTheDocument(); + // The recording is not filtered out of a shared view — it is not what the tab + // is reading any more. + expect(screen.queryByText("filmed", { exact: false })).toBeNull(); + }); + + it("ignores music, which is never transcribed", () => { + mount([{ ...VOICEOVER, kind: "music" } as AxcutAudioTrack]); + expect(screen.queryByRole("group", { name: "Read the transcript from" })).toBeNull(); + }); +}); diff --git a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx index f11151122..4f1aa8c79 100644 --- a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx +++ b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx @@ -71,13 +71,17 @@ function renderPane(onSeek: (sec: number) => void = vi.fn()) { ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 3, + cameraTrack: null, +}; + +const CLIP: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 3, + timelineStartSec: 0, + timelineEndSec: 3, + wordRefs: [], + origin: "user", + reason: "", +}; + +// Contiguous: a gap would insert a `[silence]` pill between the words and move the +// indices these tests address words by. +const WORDS: AxcutWord[] = [ + { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "Bonjour" }, + { id: "w2", segmentId: "s", startSec: 1, endSec: 2, text: "Kubernetes" }, + { id: "w3", segmentId: "s", startSec: 2, endSec: 3, text: "tout" }, +]; + +function transcript(words: AxcutWord[] = WORDS): AxcutTranscript { + return { assetId: "asset_1", language: "fr", segments: [], words }; +} + +function renderPane(words?: AxcutWord[], busyAssetIds: string[] = []) { + const onSetWordText = vi.fn(); + const onAddTrimRange = vi.fn(); + const view = render( + + + , + ); + const wordEl = (id: string) => { + const el = view.container.querySelector(`[data-word-id="clip_1:${id}"]`); + if (!el) throw new Error(`word ${id} not rendered`); + return el; + }; + const field = () => view.container.querySelector("input[data-word-editor]"); + return { ...view, wordEl, field, onSetWordText, onAddTrimRange }; +} + +afterEach(cleanup); + +describe("telling the user the gestures exist", () => { + it("shows the editing hint line and the ? help when a transcript is on screen", () => { + // The gestures are invisible until tried — the pane must name them itself. + const view = renderPane(); + expect(view.getByText(/Double-click a word to correct it/)).toBeInTheDocument(); + expect(view.getByRole("button", { name: "Help" })).toBeInTheDocument(); + }); +}); + +describe("correcting a word", () => { + it("opens an editing field on the word a double-click lands on", () => { + const view = renderPane(); + expect(view.field()).toBeNull(); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toHaveValue("Kubernetes"); + }); + + it("commits on Enter, addressing the word by its BARE id and the clip's asset", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Kubernetes 1.31" } }); + fireEvent.keyDown(field, { key: "Enter" }); + // `clip_1:w2` is what the DOM node carries; the transcript knows only `w2`. + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Kubernetes 1.31"); + }); + + it("commits on blur, so clicking away does not throw the correction out", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w1")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Bonsoir" } }); + fireEvent.blur(field); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w1", "Bonsoir"); + }); + + it("abandons on Escape, and a blur afterwards does not resurrect the draft", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w1")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.change(field, { target: { value: "Bonsoir" } }); + fireEvent.keyDown(field, { key: "Escape" }); + fireEvent.blur(field); + expect(view.onSetWordText).not.toHaveBeenCalled(); + expect(view.field()).toBeNull(); + }); + + it("writes nothing when the text comes back unchanged", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onSetWordText).not.toHaveBeenCalled(); + }); + + // The field lives inside the block's contentEditable, whose Backspace handler cuts the + // media. Without the stopPropagation on the field, deleting a letter would trim the clip. + it("does not cut the media when Backspace is pressed inside the field", () => { + const view = renderPane(); + fireEvent.doubleClick(view.wordEl("w2")); + const field = view.field(); + if (!field) throw new Error("no editing field"); + fireEvent.keyDown(field, { key: "Backspace" }); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); + + it("stays read-only while the transcript is being regenerated", () => { + const view = renderPane(undefined, ["asset_1"]); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toBeNull(); + }); +}); + +describe("a word already corrected", () => { + const CORRECTED: AxcutWord[] = [ + WORDS[0], + { ...WORDS[1], text: "Kubernetes", originalText: "Cuber Nettes", source: "user" }, + WORDS[2], + ]; + + it("is marked as corrected and names what the transcriber heard", () => { + const view = renderPane(CORRECTED); + const el = view.wordEl("w2"); + expect(el).toHaveAttribute("data-corrected", "true"); + expect(el.title).toContain("Cuber Nettes"); + }); + + it("offers a revert that writes the transcriber's own text back", () => { + const view = renderPane(CORRECTED); + fireEvent.mouseEnter(view.wordEl("w2")); + const revert = view.wordEl("w2").querySelector("button"); + if (!revert) throw new Error("no revert control"); + fireEvent.click(revert); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Cuber Nettes"); + }); + + it("leaves an untouched word unmarked and without a revert", () => { + const view = renderPane(CORRECTED); + fireEvent.mouseEnter(view.wordEl("w1")); + expect(view.wordEl("w1")).not.toHaveAttribute("data-corrected"); + expect(view.wordEl("w1").querySelector("button")).toBeNull(); + }); +}); + +// Emptying a word is how a junk token gets out of the captions without cutting the audio. +// Rendered as its own (empty) text it would be a bare space: invisible, un-clickable, and +// therefore impossible to undo. +describe("a word the user emptied", () => { + const BLANKED: AxcutWord[] = [ + WORDS[0], + { ...WORDS[1], text: "", originalText: "Kubernetes", source: "user" }, + WORDS[2], + ]; + + it("keeps a visible, clickable place in the stream", () => { + const view = renderPane(BLANKED); + const el = view.wordEl("w2"); + expect(el).toHaveAttribute("data-blanked", "true"); + expect(el.textContent?.trim()).not.toBe(""); + }); + + it("can be reopened for editing and reverted", () => { + const view = renderPane(BLANKED); + fireEvent.doubleClick(view.wordEl("w2")); + expect(view.field()).toHaveValue(""); + + fireEvent.keyDown(view.field() as HTMLInputElement, { key: "Escape" }); + fireEvent.mouseEnter(view.wordEl("w2")); + const revert = view.wordEl("w2").querySelector("button"); + if (!revert) throw new Error("no revert control"); + fireEvent.click(revert); + expect(view.onSetWordText).toHaveBeenCalledWith("asset_1", "w2", "Kubernetes"); + }); +}); diff --git a/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx new file mode 100644 index 000000000..d582cbdbb --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.wordInsert.test.tsx @@ -0,0 +1,260 @@ +// @vitest-environment jsdom +// Typing a word into the transcript that nobody said. +// +// This is the third gesture on the one word stream, and the one that had to get past a +// guard: the block used to swallow every keystroke outright, because free text has no +// `transcript.words` entry to land on. It still never lands in the block — what a typed +// character opens is a field beside the word the caret was on, and only its commit makes a +// word. These tests hold that: the DOM never gets ahead of `words`, and Backspace inside +// the field types instead of cutting the clip out from under it. + +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutClip, AxcutTranscript, AxcutWord } from "@/lib/ai-edition/schema"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 3, + cameraTrack: null, +}; + +const CLIP: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 3, + timelineStartSec: 0, + timelineEndSec: 3, + wordRefs: [], + origin: "user", + reason: "", +}; + +// Contiguous, so no `[silence]` pill sits between them to shift the caret indices. +const WORDS: AxcutWord[] = [ + { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "Bonjour" }, + { id: "w2", segmentId: "s", startSec: 1, endSec: 2, text: "à" }, + { id: "w3", segmentId: "s", startSec: 2, endSec: 3, text: "tous" }, +]; + +function renderPane(words: AxcutWord[] = WORDS, busyAssetIds: string[] = []) { + const onInsertWord = vi.fn(); + const onRemoveWords = vi.fn(); + const onAddTrimRange = vi.fn(); + const transcript: AxcutTranscript = { + assetId: "asset_1", + language: "fr", + segments: [], + words, + }; + const view = render( + + + , + ); + const editor = view.container.querySelector('[role="textbox"]'); + if (!editor) throw new Error("transcript editor not rendered"); + const field = () => view.container.querySelector("input[data-word-inserter]"); + const wordEl = (id: string) => { + const el = view.container.querySelector(`[data-word-id="clip_1:${id}"]`); + if (!el) throw new Error(`word ${id} not rendered`); + return el; + }; + return { ...view, editor, field, wordEl, onInsertWord, onRemoveWords, onAddTrimRange }; +} + +/** Park the caret between words at editor level, the way `restoreCaretBeforeWord` does. */ +function caretBeforeWordAt(editor: HTMLElement, index: number) { + const range = document.createRange(); + range.setStart(editor, index); + range.collapse(true); + const selection = window.getSelection(); + selection?.removeAllRanges(); + selection?.addRange(range); +} + +/** + * A real native `beforeinput`, because that is what the block listens to. + * + * Not `fireEvent.beforeInput`: React 18 builds its `onBeforeInput` from the legacy + * `textInput` event, whose `TextEvent` has no `inputType` — which is exactly why the guard + * moved off React and onto the DOM. Driving the synthetic one here would test a path the + * browser never takes. + */ +function type(editor: HTMLElement, data: string) { + // Through `fireEvent` so the state the listener sets is flushed, but with an event + // built by hand — `fireEvent.beforeInput` does not exist here, and the point is to + // dispatch the real thing. + fireEvent( + editor, + new InputEvent("beforeinput", { + data, + inputType: "insertText", + bubbles: true, + cancelable: true, + }), + ); +} + +afterEach(() => { + cleanup(); + window.getSelection()?.removeAllRanges(); +}); + +describe("typing between two words", () => { + it("opens a field there instead of dropping the keystroke", () => { + const view = renderPane(); + expect(view.field()).toBeNull(); + caretBeforeWordAt(view.editor, 2); // between "à" and "tous" + type(view.editor, "v"); + expect(view.field()).toHaveValue("v"); + }); + + it("stays inert outside dev builds — the gesture waits for TTS", () => { + // An inserted word with no voice only borrows free silence, so the gesture ships + // dev-only (see openInsertion). Release builds must drop the keystroke silently, + // the same way they did before the feature existed. + vi.stubEnv("DEV", false); + try { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.field()).toBeNull(); + expect(view.onInsertWord).not.toHaveBeenCalled(); + expect(view.editor.textContent).not.toContain("v "); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("never writes the typed text into the block itself", () => { + // The whole reason inserts were blocked: a run of text with no word id behind it + // desynchronises the DOM from `words`. + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.editor.textContent).not.toContain("v "); + expect(view.onInsertWord).not.toHaveBeenCalled(); + }); + + it("commits on Enter, against the word the caret was after", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.change(field, { target: { value: "vraiment" } }); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w2", "after", "vraiment"); + }); + + it("anchors before the first word when the caret is at the very start", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 0); + type(view.editor, "E"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Enter" }); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w1", "before", "E"); + }); + + it("abandons on Escape without writing anything", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Escape" }); + fireEvent.blur(field); + expect(view.onInsertWord).not.toHaveBeenCalled(); + expect(view.field()).toBeNull(); + }); + + it("commits on blur", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 1); + type(view.editor, "x"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.change(field, { target: { value: "donc" } }); + fireEvent.blur(field); + expect(view.onInsertWord).toHaveBeenCalledWith("asset_1", "w1", "after", "donc"); + }); + + it("does not cut the media when Backspace is pressed inside the field", () => { + const view = renderPane(); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + const field = view.field(); + if (!field) throw new Error("no insertion field"); + fireEvent.keyDown(field, { key: "Backspace" }); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); + + it("stays shut while this clip's transcript is being regenerated", () => { + const view = renderPane(WORDS, ["asset_1"]); + caretBeforeWordAt(view.editor, 2); + type(view.editor, "v"); + expect(view.field()).toBeNull(); + }); +}); + +describe("a word that was inserted", () => { + const INSERTED: AxcutWord[] = [ + WORDS[0], + { id: "synth_1", segmentId: "s", startSec: 1, endSec: 1, text: "vraiment", source: "synth" }, + WORDS[1], + WORDS[2], + ]; + + it("reads as its own thing, not as a transcribed word", () => { + const view = renderPane(INSERTED); + const el = view.wordEl("synth_1"); + expect(el).toHaveAttribute("data-inserted", "true"); + expect(el.textContent).toContain("vraiment"); + }); + + // There is no audio for a trim to remove, so the gesture that makes a spoken word go + // away cannot be the one that makes this go away. + it("is deleted outright by its own control", () => { + const view = renderPane(INSERTED); + fireEvent.mouseEnter(view.wordEl("synth_1")); + const remove = view.wordEl("synth_1").querySelector("button"); + if (!remove) throw new Error("no delete control"); + fireEvent.click(remove); + expect(view.onRemoveWords).toHaveBeenCalledWith("asset_1", ["synth_1"]); + }); + + it("is deleted, not trimmed, when Backspace lands on it alone", () => { + const view = renderPane(INSERTED); + caretBeforeWordAt(view.editor, 2); // right after the insert + fireEvent.keyDown(view.editor, { key: "Backspace" }); + expect(view.onRemoveWords).toHaveBeenCalledWith("asset_1", ["synth_1"]); + expect(view.onAddTrimRange).not.toHaveBeenCalled(); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.audio.test.ts b/src/components/ai-edition/VirtualPreview.audio.test.ts index 208bf05a2..624272848 100644 --- a/src/components/ai-edition/VirtualPreview.audio.test.ts +++ b/src/components/ai-edition/VirtualPreview.audio.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from "vitest"; +import { projectRawTimelineSecToPlayback } from "@/lib/ai-edition/document/timeline"; +import type { AxcutAudioTrack, AxcutClip, AxcutTrimRange } from "@/lib/ai-edition/schema"; import { applyPreviewAudioSettings, type PreviewAudioGraph, resolveAudioTrackPlayback, + resolveTimelineAudioPlayback, + timelineAudioFadeAt, } from "./VirtualPreview"; /** Minimal stand-in: the function only ever touches `gain.gain.value`. */ @@ -80,3 +84,227 @@ describe("applyPreviewAudioSettings", () => { expect(graph.gain.gain.value).toBeCloseTo(0.5, 4); }); }); + +describe("resolveTimelineAudioPlayback", () => { + // A 6s track placed 10s into the RAW timeline, playing the source from 2s in. + const track: AxcutAudioTrack = { + id: "t1", + assetId: "a1", + kind: "music", + startMs: 10_000, + endMs: 16_000, + durationSec: 20, + offsetMs: 2000, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "", + origin: "user", + }; + const spanSec = (t: AxcutAudioTrack) => (t.endMs - t.startMs) / 1000; + + // Second arg is the track head projected to output seconds; with no trims it is + // just the raw head (10), so these read the same as before the output-space + // change — the projection is exercised separately below. + it("maps the playhead to a source position offset by the track offset", () => { + // 3s into the track's span → 2 (offset) + 3 = 5s of source. + expect(resolveTimelineAudioPlayback(13, 10, track, spanSec(track))).toEqual({ + targetTimeSec: 5, + shouldPlay: true, + }); + }); + + it("does not play before the track starts, parked at the in-point", () => { + expect(resolveTimelineAudioPlayback(9, 10, track, spanSec(track))).toEqual({ + targetTimeSec: 2, + shouldPlay: false, + }); + }); + + it("does not play past the end of its span, parked at the out-point", () => { + expect(resolveTimelineAudioPlayback(16, 10, track, spanSec(track))).toEqual({ + targetTimeSec: 8, + shouldPlay: false, + }); + }); + + it("goes silent when the file runs out before the span does", () => { + // A 5s file under a 10s span: at 6s in there is no source left, and the + // element holds at the end rather than restarting. + const short = { ...track, endMs: 20_000, offsetMs: 0, durationSec: 5 }; + expect(resolveTimelineAudioPlayback(14, 10, short, spanSec(short))).toEqual({ + targetTimeSec: 4, + shouldPlay: true, + }); + expect(resolveTimelineAudioPlayback(16, 10, short, spanSec(short)).shouldPlay).toBe(false); + }); + + it("folds a looping track back into its window, in phase with the export", () => { + // 4s of source (offset 0, 4s file) under a 10s span. + const looped = { ...track, endMs: 20_000, offsetMs: 0, durationSec: 4, loop: true }; + const span = spanSec(looped); + expect(resolveTimelineAudioPlayback(13, 10, looped, span).targetTimeSec).toBeCloseTo(3, 6); + // 5s in is 1s into the second repeat — the export's second mix entry agrees. + expect(resolveTimelineAudioPlayback(15, 10, looped, span).targetTimeSec).toBeCloseTo(1, 6); + expect(resolveTimelineAudioPlayback(15, 10, looped, span).shouldPlay).toBe(true); + // Past the span it stops, however much file is left. + expect(resolveTimelineAudioPlayback(21, 10, looped, span).shouldPlay).toBe(false); + }); + + // The regression: an interior trim must NOT skip the track's own content, so the + // preview stays byte-for-byte with `audio::mix_external_tracks`, which overlays the + // decoded window contiguously. Same scenario as Etienne's review and the projection's + // own test: a 10s clip with raw 2..4 cut, background track at raw head 0 spanning 0..10. + it("plays contiguously across an interior trim, matching the export", () => { + const clip: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }; + const trim: AxcutTrimRange = { + id: "trim_1", + assetId: "asset_1", + startSec: 2, + endSec: 4, + origin: "user", + reason: "", + }; + const bgm: AxcutAudioTrack = { + ...track, + id: "bgm", + assetId: "a2", + startMs: 0, + endMs: 10_000, + durationSec: 10, + offsetMs: 0, + }; + const project = (rawSec: number) => projectRawTimelineSecToPlayback([clip], [trim], rawSec, []); + const outputStart = project(bgm.startMs / 1000); // 0 + + // Raw playhead 5 sits 1s past the 2s cut → output 3. The track is a contiguous + // block, so it must be at source 3 — NOT source 5, which the old raw-space + // `local` produced (the 2s desync). + expect(resolveTimelineAudioPlayback(project(5), outputStart, bgm, spanSec(bgm))).toEqual({ + targetTimeSec: 3, + shouldPlay: true, + }); + // Just before the cut is unaffected: raw 1 → output 1 → source 1. + expect( + resolveTimelineAudioPlayback(project(1), outputStart, bgm, spanSec(bgm)).targetTimeSec, + ).toBeCloseTo(1, 6); + }); +}); + +describe("resolveTimelineAudioPlayback under a trim", () => { + const clip: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 20, + timelineStartSec: 0, + timelineEndSec: 20, + wordRefs: [], + origin: "user", + reason: "", + }; + const trim: AxcutTrimRange = { + id: "trim_1", + assetId: "asset_1", + startSec: 4, + endSec: 8, + origin: "user", + reason: "", + }; + const project = (rawSec: number) => projectRawTimelineSecToPlayback([clip], [trim], rawSec, []); + + const buried: AxcutAudioTrack = { + id: "buried", + assetId: "a1", + kind: "music", + startMs: 5000, + endMs: 7000, + durationSec: 30, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "", + origin: "user", + }; + + it("never plays a track buried inside the trim", () => { + // Both ends project onto the cut, so the track's OUTPUT span is zero. Read + // off the raw span instead it stayed 2s long and played at the boundary, + // with nothing on screen to explain the sound. + const outStart = project(buried.startMs / 1000); + const outSpan = project(buried.endMs / 1000) - outStart; + expect(outSpan).toBeCloseTo(0, 6); + for (const raw of [3, 5, 6, 9, 12]) { + expect(resolveTimelineAudioPlayback(project(raw), outStart, buried, outSpan).shouldPlay).toBe( + false, + ); + } + }); + + it("plays a track that merely crosses the trim, for the length that survives", () => { + const crossing = { ...buried, id: "crossing", startMs: 2000, endMs: 12_000 }; + const outStart = project(crossing.startMs / 1000); + const outSpan = project(crossing.endMs / 1000) - outStart; + // Raw 2..12 with raw 4..8 cut leaves 6s of programme. + expect(outSpan).toBeCloseTo(6, 6); + expect(resolveTimelineAudioPlayback(project(3), outStart, crossing, outSpan).shouldPlay).toBe( + true, + ); + // Just past the end of what survives. + expect( + resolveTimelineAudioPlayback(outStart + 6.1, outStart, crossing, outSpan).shouldPlay, + ).toBe(false); + }); +}); + +describe("timelineAudioFadeAt", () => { + const track: AxcutAudioTrack = { + id: "t1", + assetId: "a1", + kind: "music", + startMs: 0, + endMs: 10_000, + durationSec: 20, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 1000, + fadeOutMs: 2000, + muted: false, + label: "", + origin: "user", + }; + + it("ramps in and out over the track's own edges", () => { + expect(timelineAudioFadeAt(track, 0, 10)).toBe(0); + expect(timelineAudioFadeAt(track, 0.5, 10)).toBeCloseTo(0.5, 6); + expect(timelineAudioFadeAt(track, 5, 10)).toBe(1); + expect(timelineAudioFadeAt(track, 9, 10)).toBeCloseTo(0.5, 6); + expect(timelineAudioFadeAt(track, 10, 10)).toBe(0); + }); + + it("is silent when muted", () => { + expect(timelineAudioFadeAt({ ...track, muted: true }, 5, 10)).toBe(0); + }); + + it("still reaches full volume when a fade is longer than the span", () => { + // Unreduced, the ramp never completes and the track plays near-silent. + const long = { ...track, fadeInMs: 20_000, fadeOutMs: 0 }; + expect(timelineAudioFadeAt(long, 2, 2)).toBe(1); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.playback.test.tsx b/src/components/ai-edition/VirtualPreview.playback.test.tsx index f0389fe2a..f4ffe88a0 100644 --- a/src/components/ai-edition/VirtualPreview.playback.test.tsx +++ b/src/components/ai-edition/VirtualPreview.playback.test.tsx @@ -207,3 +207,176 @@ describe("VirtualPreview playback across a clip boundary", () => { expect(video.pauseCalls).toHaveLength(0); }); }); + +// Issue #350 — imported audio tracks follow the RAW virtual playhead. The +// decision math is unit-tested in VirtualPreview.audio.test.ts; here we prove the +// rAF loop applies it to the mounted