Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion packages/base/audio-file-def.gts
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,11 @@ export async function waveformFor(
}
try {
let bytes = await byteStreamToUint8Array(await getStream());
return await extractAudioWaveform(bytes);
// The container's stated rate sizes the decoding context, so the decode
// isn't resampled before analysis.
return await extractAudioWaveform(bytes, {
sampleRateHz: budget.sampleRateHz,
});
} catch (error) {
// A stream that won't re-read is not a reason to fail the whole extract; the
// header-derived facts are already gathered.
Expand Down
80 changes: 59 additions & 21 deletions packages/base/audio-waveform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,23 +230,62 @@ interface AudioDecoderLike {
close?: () => Promise<void> | void;
}

type AudioDecoderConstructor = new (options?: {
sampleRate?: number;
}) => AudioDecoderLike;
type OfflineAudioContextConstructor = new (
numberOfChannels: number,
length: number,
sampleRate: number,
) => AudioDecoderLike;

function audioDecoderConstructor(): AudioDecoderConstructor | undefined {
type AudioContextConstructor = new () => AudioDecoderLike;

// The band every Web Audio implementation must accept for a context's sample
// rate. A container may state a rate outside it (FLAC admits up to 655350 Hz);
// the context gets the nearest legal rate and `decodeAudioData` resamples the
// decoded audio to it, which an amplitude envelope doesn't notice.
const MIN_CONTEXT_SAMPLE_RATE_HZ = 8000;
const MAX_CONTEXT_SAMPLE_RATE_HZ = 96000;
const DEFAULT_CONTEXT_SAMPLE_RATE_HZ = 44100;

function contextSampleRate(sampleRateHz: number | undefined): number {
if (
sampleRateHz === undefined ||
!Number.isFinite(sampleRateHz) ||
sampleRateHz <= 0
) {
return DEFAULT_CONTEXT_SAMPLE_RATE_HZ;
}
return Math.min(
Math.max(sampleRateHz, MIN_CONTEXT_SAMPLE_RATE_HZ),
MAX_CONTEXT_SAMPLE_RATE_HZ,
);
}

// Build a decoding context, or undefined where Web Audio is missing entirely.
//
// An OfflineAudioContext decodes without touching an output device, which is
// what a headless indexing pass wants — a live AudioContext would try to open
// hardware. Fall back to the real thing where offline isn't available.
//
// The offline constructor requires its render parameters up front. The render
// graph never runs — only `decodeAudioData` is used — so channel count and
// length are minimal; the sample rate is the one parameter that matters,
// because decoded audio is resampled to the context's rate. Callers pass the
// rate the container header stated so the analysis sees native resolution.
function makeAudioDecoder(
sampleRateHz: number | undefined,
): AudioDecoderLike | undefined {
let scope = globalThis as unknown as {
OfflineAudioContext?: unknown;
AudioContext?: unknown;
webkitAudioContext?: unknown;
};
// An OfflineAudioContext decodes without touching an output device, which is
// what a headless indexing pass wants — a live AudioContext would try to open
// hardware. Fall back to the real thing where offline isn't available.
let candidate =
scope.OfflineAudioContext ?? scope.AudioContext ?? scope.webkitAudioContext;
return typeof candidate === 'function'
? (candidate as AudioDecoderConstructor)
if (typeof scope.OfflineAudioContext === 'function') {
let Offline = scope.OfflineAudioContext as OfflineAudioContextConstructor;
return new Offline(1, 1, contextSampleRate(sampleRateHz));
}
let Live = scope.AudioContext ?? scope.webkitAudioContext;
return typeof Live === 'function'
? new (Live as AudioContextConstructor)()
: undefined;
}

Expand All @@ -255,8 +294,9 @@ function audioDecoderConstructor(): AudioDecoderConstructor | undefined {
// still index with its header-derived metadata intact.
export async function extractAudioWaveform(
bytes: Uint8Array,
barCount = WAVEFORM_BAR_COUNT,
opts: { barCount?: number; sampleRateHz?: number } = {},
): Promise<WaveformMetadata> {
let { barCount = WAVEFORM_BAR_COUNT, sampleRateHz } = opts;
if (bytes.byteLength === 0) {
return { decodeStatus: 'skipped', decodeError: 'File is empty' };
}
Expand All @@ -271,17 +311,15 @@ export async function extractAudioWaveform(
)} MB unbudgeted ceiling`,
};
}
let Decoder = audioDecoderConstructor();
if (!Decoder) {
return {
decodeStatus: 'unsupported',
decodeError: 'Web Audio is not available in this environment',
};
}

let context: AudioDecoderLike | undefined;
try {
context = new Decoder();
context = makeAudioDecoder(sampleRateHz);
if (!context) {
return {
decodeStatus: 'unsupported',
decodeError: 'Web Audio is not available in this environment',
};
}
// `decodeAudioData` detaches the buffer it is given, so hand it a copy —
// otherwise the caller's bytes, which other extractors still need to read,
// come back as a zero-length view.
Expand Down
85 changes: 81 additions & 4 deletions packages/host/tests/unit/audio-metadata-extractor-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,7 @@ module('Unit | audio metadata extractors', function (hooks) {
let analyzeDecodedAudio: typeof AudioWaveformModule.analyzeDecodedAudio;
let decodeSkipReason: typeof AudioWaveformModule.decodeSkipReason;
let predictedDecodedBytes: typeof AudioWaveformModule.predictedDecodedBytes;
let extractAudioWaveform: typeof AudioWaveformModule.extractAudioWaveform;
let audioAttributes: typeof AudioFileDefModule.audioAttributes;
let extractMidiMetadata: typeof MidiModule.extractMidiMetadata;

Expand Down Expand Up @@ -390,10 +391,14 @@ module('Unit | audio metadata extractors', function (hooks) {
({ parseVorbisComments } = await loader.import<typeof VorbisModule>(
'@cardstack/base/vorbis-comment-parser',
));
({ analyzeDecodedAudio, decodeSkipReason, predictedDecodedBytes } =
await loader.import<typeof AudioWaveformModule>(
'@cardstack/base/audio-waveform',
));
({
analyzeDecodedAudio,
decodeSkipReason,
predictedDecodedBytes,
extractAudioWaveform,
} = await loader.import<typeof AudioWaveformModule>(
'@cardstack/base/audio-waveform',
));
({ audioAttributes } = await loader.import<typeof AudioFileDefModule>(
'@cardstack/base/audio-file-def',
));
Expand Down Expand Up @@ -1738,4 +1743,76 @@ module('Unit | audio metadata extractors', function (hooks) {
);
});
});

module('waveform decode (real Web Audio)', function () {
// Real 16-bit PCM the browser's own decoder accepts, so these pin the
// decoding-context construction contract — OfflineAudioContext requires
// its render parameters up front — which a structural fake never
// exercises.
function pcmWav(seconds: number, sampleRate: number): Uint8Array {
let frameCount = Math.round(seconds * sampleRate);
let pcm: number[] = [];
for (let i = 0; i < frameCount; i++) {
// Silent first half, half-scale second half: a shape the envelope
// assertions can recognize.
let value = i < frameCount / 2 ? 0 : Math.round(0.5 * 32767);
pcm.push(...uint16le(value));
}
let fmtBody = [
...uint16le(1),
...uint16le(1),
...uint32le(sampleRate),
...uint32le(sampleRate * 2),
...uint16le(2),
...uint16le(16),
];
let payload = [
...ascii('WAVE'),
...riffChunk('fmt ', fmtBody),
...riffChunk('data', pcm),
];
return new Uint8Array([
...ascii('RIFF'),
...uint32le(payload.length),
...payload,
]);
}

test('decodes real bytes through the environment Web Audio implementation', async function (assert) {
let result = await extractAudioWaveform(pcmWav(0.5, 8000), {
barCount: 8,
sampleRateHz: 8000,
});
assert.strictEqual(
result.decodeStatus,
'ok',
`decode succeeds (${result.decodeError ?? 'no error'})`,
);
assert.strictEqual(result.barCount, 8);
assert.true(
Math.abs((result.durationSeconds ?? 0) - 0.5) < 0.05,
`duration ~0.5s (got ${result.durationSeconds})`,
);
let bars = JSON.parse(result.barsJson ?? '[]') as number[];
assert.strictEqual(bars.length, 8);
assert.true(
bars[7]! > bars[0]!,
'the loud half reads louder than the silent half',
);
});

test('a stated rate outside the Web Audio band still decodes', async function (assert) {
// FLAC admits rates up to 655350 Hz; the context clamps to a supported
// rate and the decode resamples rather than failing.
let result = await extractAudioWaveform(pcmWav(0.25, 8000), {
barCount: 4,
sampleRateHz: 655350,
});
assert.strictEqual(
result.decodeStatus,
'ok',
`decode succeeds (${result.decodeError ?? 'no error'})`,
);
});
});
});
Loading