From 9f9d6698312de91261fed15ac1dc8bb9d0dc6d3c Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Mon, 24 Aug 2026 22:56:04 +0200 Subject: [PATCH 01/84] feat(editor): add audio-track data model for external audio import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of issue #350 (import voiceover / BGM / SFX). Adds the document model for timeline audio tracks without any UI, IPC, or export wiring yet. - Widen assetSchema.kind to enum(["video","audio"]) so an imported audio file (no video stream) has its own kind. Additive — every existing doc holds "video", which still validates, so no schemaVersion bump. - Add audioTrackSchema: a timeline-global track addressed in OUTPUT (post-trim/post-speed) timeline seconds, the same domain the compositor's concatenated programme PCM lives in. That invariant is what will keep the live preview and the export in sync in later phases. - Add document.audioTracks[] (defaulted, so pre-#350 docs load unchanged), the AxcutAudioTrack type, and a createAudioTrack factory. - Tests cover defaults, trim/position validation, factory round-trip, the kind widening, and that a document omitting audioTracks defaults to []. - Fixture fallout: 15 test files + browserShim build full AxcutDocument literals and now carry audioTracks: [] alongside their zoomRanges: []. Co-Authored-By: Claude Opus 4.8 --- .../ai-edition/EditorEmptyState.test.tsx | 1 + .../ExportDialog.showInFolder.test.tsx | 1 + .../ai-edition/ExportDialog.test.ts | 1 + .../ai-edition/WebcamOverlay.test.tsx | 1 + .../ai-edition/document/outputFormat.test.ts | 1 + src/lib/ai-edition/document/timeline.test.ts | 1 + .../ai-edition/document/transcribe.test.ts | 1 + src/lib/ai-edition/schema/index.test.ts | 102 ++++++++++++++++-- src/lib/ai-edition/schema/index.ts | 65 ++++++++++- .../ai-edition/store/editorSettings.test.ts | 1 + src/lib/ai-edition/store/projectStore.test.ts | 1 + .../ai-edition/store/undo.modalGuard.test.tsx | 1 + src/lib/ai-edition/store/useCaptions.test.ts | 1 + .../store/useEditorSettings.test.ts | 1 + src/lib/ai-edition/store/useTimeline.test.ts | 1 + .../ai-edition/transcription/status.test.ts | 1 + src/native/browserShim.ts | 1 + src/native/sceneDescription.test.ts | 1 + 18 files changed, 175 insertions(+), 8 deletions(-) diff --git a/src/components/ai-edition/EditorEmptyState.test.tsx b/src/components/ai-edition/EditorEmptyState.test.tsx index acb513e4b..be7f130b7 100644 --- a/src/components/ai-edition/EditorEmptyState.test.tsx +++ b/src/components/ai-edition/EditorEmptyState.test.tsx @@ -53,6 +53,7 @@ const sampleDoc = vi.hoisted( }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }), ); diff --git a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx index 835d9c8fd..db4207c4e 100644 --- a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx +++ b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx @@ -75,6 +75,7 @@ const DOC: AxcutDocument = { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/components/ai-edition/ExportDialog.test.ts b/src/components/ai-edition/ExportDialog.test.ts index 3aa8d85b5..ed2f1ab1a 100644 --- a/src/components/ai-edition/ExportDialog.test.ts +++ b/src/components/ai-edition/ExportDialog.test.ts @@ -57,6 +57,7 @@ function doc(assets: AxcutAsset[], clips: AxcutClip[]): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; } diff --git a/src/components/ai-edition/WebcamOverlay.test.tsx b/src/components/ai-edition/WebcamOverlay.test.tsx index bdd36a558..9c4b2762d 100644 --- a/src/components/ai-edition/WebcamOverlay.test.tsx +++ b/src/components/ai-edition/WebcamOverlay.test.tsx @@ -74,6 +74,7 @@ function makeDocument(): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; } diff --git a/src/lib/ai-edition/document/outputFormat.test.ts b/src/lib/ai-edition/document/outputFormat.test.ts index 4ceac442f..c7e75ab40 100644 --- a/src/lib/ai-edition/document/outputFormat.test.ts +++ b/src/lib/ai-edition/document/outputFormat.test.ts @@ -67,6 +67,7 @@ function doc(assets: AxcutAsset[], clips: AxcutClip[]): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; } diff --git a/src/lib/ai-edition/document/timeline.test.ts b/src/lib/ai-edition/document/timeline.test.ts index a561dd2f9..bb5172178 100644 --- a/src/lib/ai-edition/document/timeline.test.ts +++ b/src/lib/ai-edition/document/timeline.test.ts @@ -57,6 +57,7 @@ function makeDoc(overrides: Partial = {}): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, ...overrides, }; diff --git a/src/lib/ai-edition/document/transcribe.test.ts b/src/lib/ai-edition/document/transcribe.test.ts index 72dc8d00b..9e8cbe4bd 100644 --- a/src/lib/ai-edition/document/transcribe.test.ts +++ b/src/lib/ai-edition/document/transcribe.test.ts @@ -49,6 +49,7 @@ function makeDoc(): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; } diff --git a/src/lib/ai-edition/schema/index.test.ts b/src/lib/ai-edition/schema/index.test.ts index bbba94fd8..2c6b25be4 100644 --- a/src/lib/ai-edition/schema/index.test.ts +++ b/src/lib/ai-edition/schema/index.test.ts @@ -3,8 +3,10 @@ import { migrateRawDocumentToCurrent } from "../document/migrate"; import { annotationRegionSchema, assetSchema, + audioTrackSchema, axcutSchemaVersion, clipSchema, + createAudioTrack, createEmptyDocument, documentSchema, ensureDocument, @@ -40,6 +42,7 @@ describe("axcut-schema v7", () => { expect(doc.timeline.captionRanges).toEqual([]); expect(doc.annotations).toEqual([]); expect(doc.zoomRanges).toEqual([]); + expect(doc.audioTracks).toEqual([]); expect(doc.transcripts).toEqual([]); expect(doc.legacyEditor).toBeNull(); }); @@ -73,14 +76,22 @@ describe("axcut-schema v7", () => { ).toThrow(); }); - it("assetSchema requires kind = 'video'", () => { + it("assetSchema accepts kind 'video' and 'audio', defaulting to 'video'", () => { + // Widened from a literal when external-audio import landed (issue #350). + const video = assetSchema.parse({ id: "a1", label: "x", originalPath: "/x.mp4" }); + expect(video.kind).toBe("video"); + const audio = assetSchema.parse({ + id: "a2", + kind: "audio", + label: "bgm", + originalPath: "/bgm.mp3", + }); + expect(audio.kind).toBe("audio"); + }); + + it("assetSchema rejects an unknown kind", () => { expect(() => - assetSchema.parse({ - id: "asset_1", - kind: "audio", - label: "x", - originalPath: "/x.mp4", - }), + assetSchema.parse({ id: "a1", kind: "image", label: "x", originalPath: "/x.png" }), ).toThrow(); }); @@ -962,3 +973,80 @@ describe("v6 -> v7 trim clip-anchor migration", () => { ]); }); }); + +describe("audio tracks (issue #350)", () => { + it("applies defaults for gain, mute, trim, position, and label", () => { + const track = audioTrackSchema.parse({ + id: "audio_1", + assetId: "asset_1", + durationSec: 42, + }); + expect(track.timelineStartSec).toBe(0); + expect(track.trimStartSec).toBe(0); + expect(track.trimEndSec).toBeUndefined(); + expect(track.gainDb).toBe(0); + expect(track.mute).toBe(false); + expect(track.label).toBe(""); + }); + + it("rejects a trim window whose end precedes its start", () => { + expect(() => + audioTrackSchema.parse({ + id: "audio_1", + assetId: "asset_1", + durationSec: 10, + trimStartSec: 5, + trimEndSec: 2, + }), + ).toThrow(); + }); + + it("rejects a negative timelineStartSec", () => { + expect(() => + audioTrackSchema.parse({ + id: "audio_1", + assetId: "asset_1", + durationSec: 10, + timelineStartSec: -1, + }), + ).toThrow(); + }); + + it("createAudioTrack builds a schema-valid track with a prefixed id", () => { + const track = createAudioTrack({ + assetId: "asset_1", + durationSec: 12.5, + timelineStartSec: 3, + label: "voiceover.mp3", + }); + expect(track.id).toMatch(/^audio_/); + expect(track.assetId).toBe("asset_1"); + expect(track.durationSec).toBe(12.5); + expect(track.timelineStartSec).toBe(3); + expect(track.label).toBe("voiceover.mp3"); + // The factory output must itself round-trip through the schema. + expect(() => audioTrackSchema.parse(track)).not.toThrow(); + }); + + it("defaults audioTracks to [] when a stored document omits the key", () => { + // A document written before issue #350 has no `audioTracks`; the defaulted + // array must fill in so older files load unchanged (no schemaVersion bump). + const { audioTracks: _drop, ...withoutAudio } = createEmptyDocument({ + projectId: "p", + title: "t", + }); + expect("audioTracks" in withoutAudio).toBe(false); + const parsed = documentSchema.parse(withoutAudio); + expect(parsed.audioTracks).toEqual([]); + }); + + it("round-trips a document carrying an audio track", () => { + const track = createAudioTrack({ assetId: "asset_1", durationSec: 8 }); + const doc = { + ...createEmptyDocument({ projectId: "p", title: "t" }), + audioTracks: [track], + }; + const parsed = documentSchema.parse(doc); + expect(parsed.audioTracks).toEqual([track]); + }); +}); diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index dd6936fde..a6efe746b 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -150,7 +150,12 @@ export const assetTranscriptionFailureSchema = z.object({ export const assetSchema = z.object({ id: z.string().min(1), - kind: z.literal("video"), + // Widened from a `"video"` literal when external-audio import landed (issue + // #350). An imported voiceover / BGM / SFX file carries no video stream, so it + // needs its own kind; every document written before this only ever held + // `"video"`, which still validates, so the widening is additive (no + // schemaVersion bump — same rule as `transcriptionFailure` below). + kind: z.enum(["video", "audio"]).default("video"), label: z.string().min(1), originalPath: z.string().min(1), proxyPath: z.string().optional(), @@ -471,6 +476,38 @@ export const zoomRegionSchema = endGteStart( "startMs", ); +// External audio import (issue #350) — voiceover / BGM / SFX layered over the +// assembled programme. Unlike zoom/speed/annotation/trim, an audio track is NOT +// clip-anchored: it floats over the whole timeline, so it is addressed in OUTPUT +// (post-trim, post-speed) timeline seconds — the same domain the compositor's +// concatenated programme PCM lives in (see `SceneAudio` in +// src/native/sceneDescription.ts and `audio.rs`). That single invariant is what +// keeps the live preview and the export in sync without a per-track sync offset. +// +// `assetId` points at an asset with `kind: "audio"`. `timelineStartSec` places +// the track's head on the programme; `trimStartSec`/`trimEndSec` window the +// source file (both in source seconds); `gainDb` + `mute` set its level. +export const audioTrackSchema = z + .object({ + id: z.string().min(1), + assetId: z.string().min(1), + timelineStartSec: z.number().nonnegative().default(0), + // Full source duration of the underlying file, cached here so the timeline + // can lay out the pill before the asset is re-probed on load. + durationSec: z.number().nonnegative().default(0), + trimStartSec: z.number().nonnegative().default(0), + // Absent means "play to the end of the file". Explicit when the user trims + // the tail so the pill and the export agree on where the track stops. + trimEndSec: z.number().nonnegative().optional(), + gainDb: z.number().default(0), + mute: z.boolean().default(false), + label: z.string().default(""), + }) + .refine((data) => data.trimEndSec === undefined || data.trimEndSec >= data.trimStartSec, { + message: "trimEndSec must be greater than or equal to trimStartSec", + path: ["trimEndSec"], + }); + // Legacy OpenScreen appearance / export settings that the v3 schema doesn't // normalize into the timeline / assets model. They are applied at export time // by the existing pipeline (see technical-documentation/architecture/document-model.md). @@ -503,6 +540,9 @@ const documentSchemaShape = z.object({ }), annotations: z.array(annotationRegionSchema).default([]), zoomRanges: z.array(zoomRegionSchema).default([]), + // Imported audio tracks (issue #350). Defaulted so every document written + // before this loads unchanged; an older build simply strips the key on save. + audioTracks: z.array(audioTrackSchema).default([]), legacyEditor: legacyEditorSchema.nullable().default(null), }); @@ -944,6 +984,7 @@ export type AxcutTimelineOperation = z.infer; export type AxcutAnnotationRegion = z.infer; export type AxcutZoomRegion = z.infer; export type AxcutCameraTrack = z.infer; +export type AxcutAudioTrack = z.infer; export type AxcutLegacyEditor = z.infer; export type AxcutDocument = z.infer; export type AxcutDocumentInput = z.input; @@ -979,6 +1020,7 @@ export function createEmptyDocument( }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }); } @@ -986,3 +1028,24 @@ export function createEmptyDocument( export function ensureDocument(value: unknown): AxcutDocument { return documentSchema.parse(value); } + +/** + * Build a timeline audio track for an imported audio asset (issue #350). The + * head is placed at `timelineStartSec` (output-timeline seconds) and the track + * spans the whole source file until the user trims it. Parsed through the schema + * so every default (gain, mute, trim) is applied in one place. + */ +export function createAudioTrack(input: { + assetId: string; + durationSec: number; + timelineStartSec?: number; + label?: string; +}): AxcutAudioTrack { + return audioTrackSchema.parse({ + id: createId("audio"), + assetId: input.assetId, + durationSec: input.durationSec, + timelineStartSec: input.timelineStartSec ?? 0, + label: input.label ?? "", + }); +} diff --git a/src/lib/ai-edition/store/editorSettings.test.ts b/src/lib/ai-edition/store/editorSettings.test.ts index 07982f47a..0848d3404 100644 --- a/src/lib/ai-edition/store/editorSettings.test.ts +++ b/src/lib/ai-edition/store/editorSettings.test.ts @@ -29,6 +29,7 @@ const baseDoc: AxcutDocument = { }, annotations: [], zoomRanges: [], + audioTracks: [], transcripts: [], transcript: null, legacyEditor: null, diff --git a/src/lib/ai-edition/store/projectStore.test.ts b/src/lib/ai-edition/store/projectStore.test.ts index a46a6b535..84074aad2 100644 --- a/src/lib/ai-edition/store/projectStore.test.ts +++ b/src/lib/ai-edition/store/projectStore.test.ts @@ -62,6 +62,7 @@ const sampleDoc = { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/lib/ai-edition/store/undo.modalGuard.test.tsx b/src/lib/ai-edition/store/undo.modalGuard.test.tsx index f54b12389..46e083b0b 100644 --- a/src/lib/ai-edition/store/undo.modalGuard.test.tsx +++ b/src/lib/ai-edition/store/undo.modalGuard.test.tsx @@ -35,6 +35,7 @@ function doc(title: string): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; } diff --git a/src/lib/ai-edition/store/useCaptions.test.ts b/src/lib/ai-edition/store/useCaptions.test.ts index b4063d68c..0cd641328 100644 --- a/src/lib/ai-edition/store/useCaptions.test.ts +++ b/src/lib/ai-edition/store/useCaptions.test.ts @@ -71,6 +71,7 @@ const docA: AxcutDocument = { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/lib/ai-edition/store/useEditorSettings.test.ts b/src/lib/ai-edition/store/useEditorSettings.test.ts index bd47cb452..4122c05bf 100644 --- a/src/lib/ai-edition/store/useEditorSettings.test.ts +++ b/src/lib/ai-edition/store/useEditorSettings.test.ts @@ -74,6 +74,7 @@ const docA: AxcutDocument = { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/lib/ai-edition/store/useTimeline.test.ts b/src/lib/ai-edition/store/useTimeline.test.ts index 78854e817..ef28cd571 100644 --- a/src/lib/ai-edition/store/useTimeline.test.ts +++ b/src/lib/ai-edition/store/useTimeline.test.ts @@ -110,6 +110,7 @@ const sampleDoc: AxcutDocument = { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/lib/ai-edition/transcription/status.test.ts b/src/lib/ai-edition/transcription/status.test.ts index 00261712e..82b282703 100644 --- a/src/lib/ai-edition/transcription/status.test.ts +++ b/src/lib/ai-edition/transcription/status.test.ts @@ -243,6 +243,7 @@ describe("transcriptRelevantAssetIds", () => { transcripts: [], annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/native/browserShim.ts b/src/native/browserShim.ts index 8ef5989a8..c00f53d51 100644 --- a/src/native/browserShim.ts +++ b/src/native/browserShim.ts @@ -386,6 +386,7 @@ function createShimBridgeClient() { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; documentsByProject[doc.project.id] = doc; diff --git a/src/native/sceneDescription.test.ts b/src/native/sceneDescription.test.ts index 570ae71ee..2c02137e0 100644 --- a/src/native/sceneDescription.test.ts +++ b/src/native/sceneDescription.test.ts @@ -91,6 +91,7 @@ function makeDoc( }, annotations: overrides.annotations ?? [], zoomRanges: overrides.zoomRanges ?? [], + audioTracks: overrides.audioTracks ?? [], legacyEditor: overrides.legacyEditor ?? null, }; } From 95e80d673ec7d54ca53908a00e35b713381334ca Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Mon, 24 Aug 2026 23:27:03 +0200 Subject: [PATCH 02/84] feat(editor): import external audio files as audio-kind assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of issue #350. Wires up picking an external audio file (voiceover / BGM / SFX) and adding it to a project as a kind:"audio" asset. No timeline placement, preview, or export yet. - IPC: open-audio-file-picker mirrors the video picker but approves against a dedicated audio extension set (mp3/wav/m4a/aac/flac/ogg/opus). Factor the path approver into a shared approveReadableMediaPath so the audio and video approvers differ only by their extension gate — an audio picker must not approve a video path or vice versa. - document-service.addAsset takes a kind; an audio import validates against audio extensions and never claims the empty primaryAssetId slot, so a BGM file dropped into a fresh project can't become its primary (video) asset. Threaded kind through the bridge chain (contracts, client, nativeBridge, aiEditionService) and the browser shim. - projectStore.addAudioAsset imports the file, skips the camera-sidecar lookup addAsset does, and probes the real duration up front (new probeAudioDuration, the