diff --git a/packages/base/audio-file-def.gts b/packages/base/audio-file-def.gts index c7eeb54285e..90199e98218 100644 --- a/packages/base/audio-file-def.gts +++ b/packages/base/audio-file-def.gts @@ -188,7 +188,11 @@ export async function waveformFor( } try { let bytes = await byteStreamToUint8Array(await getStream()); - return await extractAudioWaveform(bytes); + // The container's stated rate sizes the decoding context, so the decode + // isn't resampled before analysis. + return await extractAudioWaveform(bytes, { + sampleRateHz: budget.sampleRateHz, + }); } catch (error) { // A stream that won't re-read is not a reason to fail the whole extract; the // header-derived facts are already gathered. diff --git a/packages/base/audio-waveform.ts b/packages/base/audio-waveform.ts index 889acbff643..59544fce102 100644 --- a/packages/base/audio-waveform.ts +++ b/packages/base/audio-waveform.ts @@ -230,23 +230,62 @@ interface AudioDecoderLike { close?: () => Promise | void; } -type AudioDecoderConstructor = new (options?: { - sampleRate?: number; -}) => AudioDecoderLike; +type OfflineAudioContextConstructor = new ( + numberOfChannels: number, + length: number, + sampleRate: number, +) => AudioDecoderLike; -function audioDecoderConstructor(): AudioDecoderConstructor | undefined { +type AudioContextConstructor = new () => AudioDecoderLike; + +// The band every Web Audio implementation must accept for a context's sample +// rate. A container may state a rate outside it (FLAC admits up to 655350 Hz); +// the context gets the nearest legal rate and `decodeAudioData` resamples the +// decoded audio to it, which an amplitude envelope doesn't notice. +const MIN_CONTEXT_SAMPLE_RATE_HZ = 8000; +const MAX_CONTEXT_SAMPLE_RATE_HZ = 96000; +const DEFAULT_CONTEXT_SAMPLE_RATE_HZ = 44100; + +function contextSampleRate(sampleRateHz: number | undefined): number { + if ( + sampleRateHz === undefined || + !Number.isFinite(sampleRateHz) || + sampleRateHz <= 0 + ) { + return DEFAULT_CONTEXT_SAMPLE_RATE_HZ; + } + return Math.min( + Math.max(sampleRateHz, MIN_CONTEXT_SAMPLE_RATE_HZ), + MAX_CONTEXT_SAMPLE_RATE_HZ, + ); +} + +// Build a decoding context, or undefined where Web Audio is missing entirely. +// +// An OfflineAudioContext decodes without touching an output device, which is +// what a headless indexing pass wants — a live AudioContext would try to open +// hardware. Fall back to the real thing where offline isn't available. +// +// The offline constructor requires its render parameters up front. The render +// graph never runs — only `decodeAudioData` is used — so channel count and +// length are minimal; the sample rate is the one parameter that matters, +// because decoded audio is resampled to the context's rate. Callers pass the +// rate the container header stated so the analysis sees native resolution. +function makeAudioDecoder( + sampleRateHz: number | undefined, +): AudioDecoderLike | undefined { let scope = globalThis as unknown as { OfflineAudioContext?: unknown; AudioContext?: unknown; webkitAudioContext?: unknown; }; - // An OfflineAudioContext decodes without touching an output device, which is - // what a headless indexing pass wants — a live AudioContext would try to open - // hardware. Fall back to the real thing where offline isn't available. - let candidate = - scope.OfflineAudioContext ?? scope.AudioContext ?? scope.webkitAudioContext; - return typeof candidate === 'function' - ? (candidate as AudioDecoderConstructor) + if (typeof scope.OfflineAudioContext === 'function') { + let Offline = scope.OfflineAudioContext as OfflineAudioContextConstructor; + return new Offline(1, 1, contextSampleRate(sampleRateHz)); + } + let Live = scope.AudioContext ?? scope.webkitAudioContext; + return typeof Live === 'function' + ? new (Live as AudioContextConstructor)() : undefined; } @@ -255,8 +294,9 @@ function audioDecoderConstructor(): AudioDecoderConstructor | undefined { // still index with its header-derived metadata intact. export async function extractAudioWaveform( bytes: Uint8Array, - barCount = WAVEFORM_BAR_COUNT, + opts: { barCount?: number; sampleRateHz?: number } = {}, ): Promise { + let { barCount = WAVEFORM_BAR_COUNT, sampleRateHz } = opts; if (bytes.byteLength === 0) { return { decodeStatus: 'skipped', decodeError: 'File is empty' }; } @@ -271,17 +311,15 @@ export async function extractAudioWaveform( )} MB unbudgeted ceiling`, }; } - let Decoder = audioDecoderConstructor(); - if (!Decoder) { - return { - decodeStatus: 'unsupported', - decodeError: 'Web Audio is not available in this environment', - }; - } - let context: AudioDecoderLike | undefined; try { - context = new Decoder(); + context = makeAudioDecoder(sampleRateHz); + if (!context) { + return { + decodeStatus: 'unsupported', + decodeError: 'Web Audio is not available in this environment', + }; + } // `decodeAudioData` detaches the buffer it is given, so hand it a copy — // otherwise the caller's bytes, which other extractors still need to read, // come back as a zero-length view. diff --git a/packages/host/tests/unit/audio-metadata-extractor-test.ts b/packages/host/tests/unit/audio-metadata-extractor-test.ts index e38dbc8ff87..d04a40d61e7 100644 --- a/packages/host/tests/unit/audio-metadata-extractor-test.ts +++ b/packages/host/tests/unit/audio-metadata-extractor-test.ts @@ -361,6 +361,7 @@ module('Unit | audio metadata extractors', function (hooks) { let analyzeDecodedAudio: typeof AudioWaveformModule.analyzeDecodedAudio; let decodeSkipReason: typeof AudioWaveformModule.decodeSkipReason; let predictedDecodedBytes: typeof AudioWaveformModule.predictedDecodedBytes; + let extractAudioWaveform: typeof AudioWaveformModule.extractAudioWaveform; let audioAttributes: typeof AudioFileDefModule.audioAttributes; let extractMidiMetadata: typeof MidiModule.extractMidiMetadata; @@ -390,10 +391,14 @@ module('Unit | audio metadata extractors', function (hooks) { ({ parseVorbisComments } = await loader.import( '@cardstack/base/vorbis-comment-parser', )); - ({ analyzeDecodedAudio, decodeSkipReason, predictedDecodedBytes } = - await loader.import( - '@cardstack/base/audio-waveform', - )); + ({ + analyzeDecodedAudio, + decodeSkipReason, + predictedDecodedBytes, + extractAudioWaveform, + } = await loader.import( + '@cardstack/base/audio-waveform', + )); ({ audioAttributes } = await loader.import( '@cardstack/base/audio-file-def', )); @@ -1738,4 +1743,76 @@ module('Unit | audio metadata extractors', function (hooks) { ); }); }); + + module('waveform decode (real Web Audio)', function () { + // Real 16-bit PCM the browser's own decoder accepts, so these pin the + // decoding-context construction contract — OfflineAudioContext requires + // its render parameters up front — which a structural fake never + // exercises. + function pcmWav(seconds: number, sampleRate: number): Uint8Array { + let frameCount = Math.round(seconds * sampleRate); + let pcm: number[] = []; + for (let i = 0; i < frameCount; i++) { + // Silent first half, half-scale second half: a shape the envelope + // assertions can recognize. + let value = i < frameCount / 2 ? 0 : Math.round(0.5 * 32767); + pcm.push(...uint16le(value)); + } + let fmtBody = [ + ...uint16le(1), + ...uint16le(1), + ...uint32le(sampleRate), + ...uint32le(sampleRate * 2), + ...uint16le(2), + ...uint16le(16), + ]; + let payload = [ + ...ascii('WAVE'), + ...riffChunk('fmt ', fmtBody), + ...riffChunk('data', pcm), + ]; + return new Uint8Array([ + ...ascii('RIFF'), + ...uint32le(payload.length), + ...payload, + ]); + } + + test('decodes real bytes through the environment Web Audio implementation', async function (assert) { + let result = await extractAudioWaveform(pcmWav(0.5, 8000), { + barCount: 8, + sampleRateHz: 8000, + }); + assert.strictEqual( + result.decodeStatus, + 'ok', + `decode succeeds (${result.decodeError ?? 'no error'})`, + ); + assert.strictEqual(result.barCount, 8); + assert.true( + Math.abs((result.durationSeconds ?? 0) - 0.5) < 0.05, + `duration ~0.5s (got ${result.durationSeconds})`, + ); + let bars = JSON.parse(result.barsJson ?? '[]') as number[]; + assert.strictEqual(bars.length, 8); + assert.true( + bars[7]! > bars[0]!, + 'the loud half reads louder than the silent half', + ); + }); + + test('a stated rate outside the Web Audio band still decodes', async function (assert) { + // FLAC admits rates up to 655350 Hz; the context clamps to a supported + // rate and the decode resamples rather than failing. + let result = await extractAudioWaveform(pcmWav(0.25, 8000), { + barCount: 4, + sampleRateHz: 655350, + }); + assert.strictEqual( + result.decodeStatus, + 'ok', + `decode succeeds (${result.decodeError ?? 'no error'})`, + ); + }); + }); });