From 58c2477eadec51a1956c6b2f4a958d790eea6373 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 21 Jul 2026 17:27:48 +0100 Subject: [PATCH 01/15] Add transparent background rendering --- README.md | 22 + src/AnamClient.ts | 22 +- src/lib/ClientMetrics.ts | 1 + src/modules/StreamingClient.ts | 50 +- src/modules/TransparentBackgroundRenderer.ts | 541 ++++++++++++++++++ src/types/AnamPublicClientOptions.ts | 14 + src/types/TransparentBackgroundOptions.ts | 24 + src/types/coreApi/StartSessionOptions.ts | 6 + src/types/index.ts | 1 + src/types/streaming/StreamingClientOptions.ts | 5 + 10 files changed, 680 insertions(+), 6 deletions(-) create mode 100644 src/modules/TransparentBackgroundRenderer.ts create mode 100644 src/types/TransparentBackgroundOptions.ts diff --git a/README.md b/README.md index b5f69fd..7c7cb80 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,28 @@ await anamClient.streamToVideoElement('video-element-id'); This will start a new session using the pre-configured persona id and start streaming video element in the DOM with the matching element id. +### Transparent avatar background + +Set `transparentBackground` when creating the client. The SDK requests the +avatar's green-screen rendition and renders it through a source-resolution +WebGL canvas over your video element. + +```typescript +const anamClient = createClient('your-session-token', { + transparentBackground: true, +}); + +await anamClient.streamToVideoElement('video-element-id'); +``` + +The supplied video must be attached to the DOM and should use `autoplay` and +`playsinline` as usual. The underlying WebRTC `MediaStream` is still an opaque +green-screen video; transparency exists in the SDK-managed canvas only. As a +result, `stream()` returns the raw green source, and video-only browser features +such as native controls and picture-in-picture do not automatically capture the +transparent canvas. Use `getTransparentBackgroundCanvas()` if you need the +managed canvas element for layout or capture behavior. + To stop a session use the `stopStreaming` method. ```typescript diff --git a/src/AnamClient.ts b/src/AnamClient.ts index 8a3c693..6b2b451 100644 --- a/src/AnamClient.ts +++ b/src/AnamClient.ts @@ -203,6 +203,10 @@ export default class AnamClient { if (this.clientOptions?.voiceDetection) { sessionOptions.voiceDetection = this.clientOptions.voiceDetection; } + if (this.clientOptions?.transparentBackground !== undefined) { + sessionOptions.transparentBackground = + this.clientOptions.transparentBackground; + } // return undefined if no options are set if (Object.keys(sessionOptions).length === 0) { return undefined; @@ -321,6 +325,10 @@ export default class AnamClient { disableInputAudio: this.clientOptions?.disableInputAudio, }, apiGateway: this.clientOptions?.api?.apiGateway, + transparentBackground: { + enabled: this.clientOptions?.transparentBackground === true, + keyOptions: this.clientOptions?.transparentBackgroundOptions, + }, metrics: { showPeerConnectionStatsReport: this.clientOptions?.metrics?.showPeerConnectionStatsReport ?? @@ -492,7 +500,6 @@ export default class AnamClient { }); throw new Error('Already streaming'); } - this._isStreaming = true; if (!this.streamingClient) { connectionMilestones.publishFailure({ failureStage: 'streaming_client_missing', @@ -502,16 +509,29 @@ export default class AnamClient { try { this.streamingClient.setMediaStreamTargetById(videoElementId); + this._isStreaming = true; this.streamingClient.startConnection(); } catch (error) { connectionMilestones.publishFailure({ failureStage: 'start_connection', ...getErrorMilestoneTags(error), }); + // Renderer initialization (notably WebGL capability detection) happens + // after the server has allocated a session. Do not leak that session when + // the local render target cannot be created. + await this.stopStreaming(); throw error; } } + /** + * Returns the SDK-managed alpha canvas when transparent background rendering + * is enabled and `streamToVideoElement` has installed its render target. + */ + public getTransparentBackgroundCanvas(): HTMLCanvasElement | null { + return this.streamingClient?.getTransparentBackgroundCanvas() ?? null; + } + /** * Send a talk command to make the persona speak the provided content. * @param content - The text content for the persona to speak diff --git a/src/lib/ClientMetrics.ts b/src/lib/ClientMetrics.ts index 1f5ebd8..50b9465 100644 --- a/src/lib/ClientMetrics.ts +++ b/src/lib/ClientMetrics.ts @@ -13,6 +13,7 @@ export enum ClientMetricMeasurement { CLIENT_METRIC_MEASUREMENT_SESSION_ATTEMPT = 'client_session_attempt', CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS = 'client_session_success', CLIENT_METRIC_MEASUREMENT_ICE_RESTART = 'client_ice_restart', + CLIENT_METRIC_MEASUREMENT_TRANSPARENT_RENDERER = 'client_transparent_renderer', } const CLIENT_METRICS_MAX_BATCH_SIZE = 50; diff --git a/src/modules/StreamingClient.ts b/src/modules/StreamingClient.ts index 967d77d..59c9d28 100644 --- a/src/modules/StreamingClient.ts +++ b/src/modules/StreamingClient.ts @@ -27,6 +27,7 @@ import { WebRtcPersonaConfigUpdateAppliedEvent, WebRtcTextMessageEvent, WebRtcReasoningTextMessageEvent, + TransparentBackgroundOptions, } from '../types'; import { AgentAudioInputStream } from '../types/AgentAudioInputStream'; import { ToolCallResultReceivedPayload } from '../types/toolCalling/ToolCallPayload'; @@ -38,6 +39,7 @@ import { WebRtcToolCallStartedEvent, } from '../types/streaming/WebRtcToolCallEvent'; import { ToolCallManager } from './ToolCallManager'; +import { TransparentBackgroundRenderer } from './TransparentBackgroundRenderer'; const SUCCESS_METRIC_POLLING_TIMEOUT_MS = 15000; // After this time we will stop polling for the first frame and consider the session a failure. const STATS_COLLECTION_INTERVAL_MS = 5000; @@ -82,6 +84,12 @@ export class StreamingClient { private inputAudioStream: MediaStream | null = null; private dataChannel: RTCDataChannel | null = null; private videoElement: HTMLVideoElement | null = null; + private transparentBackgroundRenderer: TransparentBackgroundRenderer | null = + null; + private readonly transparentBackgroundEnabled: boolean; + private readonly transparentBackgroundKeyOptions: + | TransparentBackgroundOptions + | undefined; private videoStream: MediaStream | null = null; private audioStream: MediaStream | null = null; private inputAudioState: InputAudioState = { @@ -116,6 +124,10 @@ export class StreamingClient { this.toolCallManager = toolCallManager; this.connectionMilestones = connectionMilestones; this.apiGatewayConfig = options.apiGateway; + this.transparentBackgroundEnabled = + options.transparentBackground?.enabled === true; + this.transparentBackgroundKeyOptions = + options.transparentBackground?.keyOptions; // initialize input audio state const { inputAudio } = options; this.inputAudioState = inputAudio.inputAudioState; @@ -479,10 +491,26 @@ export class StreamingClient { `StreamingClient: video element with id ${videoElementId} not found`, ); } - this.videoElement = videoElement as HTMLVideoElement; + if (!(videoElement instanceof HTMLVideoElement)) { + throw new Error( + `StreamingClient: element ${videoElementId} must be a video element`, + ); + } + this.videoElement = videoElement; + if (this.transparentBackgroundEnabled) { + this.transparentBackgroundRenderer?.destroy(); + this.transparentBackgroundRenderer = new TransparentBackgroundRenderer( + this.videoElement, + this.transparentBackgroundKeyOptions, + ); + } } } + public getTransparentBackgroundCanvas(): HTMLCanvasElement | null { + return this.transparentBackgroundRenderer?.getCanvas() ?? null; + } + public startConnection() { try { if (this.peerConnection) { @@ -1206,12 +1234,18 @@ export class StreamingClient { ); if (this.videoElement) { this.videoElement.srcObject = this.videoStream; - const handle = this.videoElement.requestVideoFrameCallback(() => { - // unregister the callback after the first frame - this.videoElement?.cancelVideoFrameCallback(handle); + this.transparentBackgroundRenderer?.start(); + const onFirstFrame = () => { this.publicEventEmitter.emit(AnamEvent.VIDEO_PLAY_STARTED); this.recordSessionSuccess('videoElement'); - }); + }; + if (this.videoElement.requestVideoFrameCallback) { + this.videoElement.requestVideoFrameCallback(onFirstFrame); + } else { + this.videoElement.addEventListener('loadeddata', onFirstFrame, { + once: true, + }); + } } } else if (event.track.kind === 'audio') { this.connectionMilestones?.record('audio_track_received'); @@ -1569,6 +1603,12 @@ export class StreamingClient { } this.successMetricFired = false; + this.transparentBackgroundRenderer?.destroy(); + this.transparentBackgroundRenderer = null; + if (this.videoElement) { + this.videoElement.srcObject = null; + } + // stop the input audio stream try { if (this.inputAudioStream) { diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts new file mode 100644 index 0000000..c78daae --- /dev/null +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -0,0 +1,541 @@ +import { + ClientMetricMeasurement, + sendClientMetric, +} from '../lib/ClientMetrics'; +import { TransparentBackgroundOptions } from '../types/TransparentBackgroundOptions'; + +// Calibrated on 96 person mattes after the engine's initial H.264 Main/I420 +// profile (520 kbps). A broad soft transition preserves fractional-alpha hair +// substantially better than a narrow conventional chroma-key threshold. +const DEFAULT_SIMILARITY = 0.02; +const DEFAULT_SMOOTHNESS = 0.36; +const DEFAULT_SPILL = 0.45; +const TELEMETRY_FRAME_INTERVAL = 250; + +const VERTEX_SHADER_SOURCE = ` +attribute vec2 a_position; +varying vec2 v_texCoord; + +void main() { + gl_Position = vec4(a_position, 0.0, 1.0); + v_texCoord = (a_position + 1.0) * 0.5; +} +`; + +// Key in chroma space so luminance variation introduced by H.264 does not +// turn a uniformly-green background into a noisy alpha plane. Output is +// premultiplied because that is the browser compositor's preferred WebGL +// canvas representation and avoids a bright fringe at translucent edges. +const FRAGMENT_SHADER_SOURCE = ` +precision mediump float; + +uniform sampler2D u_frame; +uniform float u_similarity; +uniform float u_smoothness; +uniform float u_spill; +varying vec2 v_texCoord; + +vec2 chroma(vec3 rgb) { + float cb = -0.168736 * rgb.r - 0.331264 * rgb.g + 0.5 * rgb.b; + float cr = 0.5 * rgb.r - 0.418688 * rgb.g - 0.081312 * rgb.b; + return vec2(cb, cr); +} + +void main() { + vec3 rgb = texture2D(u_frame, v_texCoord).rgb; + vec2 greenChroma = chroma(vec3(0.0, 1.0, 0.0)); + float chromaDistance = distance(chroma(rgb), greenChroma); + float alpha = smoothstep( + u_similarity, + u_similarity + max(u_smoothness, 0.0001), + chromaDistance + ); + + float greenExcess = max(rgb.g - max(rgb.r, rgb.b), 0.0); + rgb.g = max( + rgb.g - greenExcess * u_spill * (1.0 - alpha), + 0.0 + ); + + gl_FragColor = vec4(rgb * alpha, alpha); +} +`; + +type OptionalVideoFrameCallbacks = { + requestVideoFrameCallback?: ( + callback: (now: DOMHighResTimeStamp) => void, + ) => number; + cancelVideoFrameCallback?: (handle: number) => void; +}; + +interface ResolvedKeyOptions { + similarity: number; + smoothness: number; + spill: number; +} + +interface GlResources { + program: WebGLProgram; + positionBuffer: WebGLBuffer; + texture: WebGLTexture; + positionLocation: number; + similarityLocation: WebGLUniformLocation; + smoothnessLocation: WebGLUniformLocation; + spillLocation: WebGLUniformLocation; +} + +export interface TransparentRendererDiagnostics { + framesRendered: number; + averageSubmissionMs: number; + maxSubmissionMs: number; +} + +export class TransparentBackgroundRenderer { + private readonly video: HTMLVideoElement; + private readonly canvas: HTMLCanvasElement; + private readonly parent: HTMLElement; + private readonly keyOptions: ResolvedKeyOptions; + private readonly gl: WebGLRenderingContext; + private resources: GlResources; + private resizeObserver: ResizeObserver | null = null; + private videoFrameCallbackHandle: number | null = null; + private animationFrameHandle: number | null = null; + private destroyed = false; + private started = false; + private contextLost = false; + private renderErrorReported = false; + private frameCount = 0; + private submissionTimeTotalMs = 0; + private submissionTimeMaxMs = 0; + private readonly originalVideoOpacity: string; + private readonly originalParentPosition: string; + private changedParentPosition = false; + + constructor(video: HTMLVideoElement, options?: TransparentBackgroundOptions) { + if (!video.parentElement) { + this.report('initialization_failed', 1, { reason: 'missing_parent' }); + throw new Error( + 'Transparent background requires the target video element to be attached to the DOM.', + ); + } + + this.video = video; + this.parent = video.parentElement; + this.keyOptions = resolveKeyOptions(options); + this.originalVideoOpacity = video.style.opacity; + this.originalParentPosition = this.parent.style.position; + + this.canvas = document.createElement('canvas'); + this.canvas.setAttribute('aria-hidden', 'true'); + this.canvas.dataset.anamTransparentBackground = 'true'; + if (video.id) { + this.canvas.id = `${video.id}--anam-transparent`; + } + + const gl = this.canvas.getContext('webgl', { + alpha: true, + antialias: false, + depth: false, + stencil: false, + premultipliedAlpha: true, + preserveDrawingBuffer: false, + powerPreference: 'high-performance', + }); + if (!gl) { + this.report('initialization_failed', 1, { + reason: 'webgl_unavailable', + }); + throw new Error( + 'Transparent background is unavailable because WebGL could not be initialized on this device.', + ); + } + this.gl = gl; + this.resources = this.createGlResources(); + + this.onContextLost = this.onContextLost.bind(this); + this.onContextRestored = this.onContextRestored.bind(this); + this.syncOverlayGeometry = this.syncOverlayGeometry.bind(this); + this.canvas.addEventListener('webglcontextlost', this.onContextLost); + this.canvas.addEventListener( + 'webglcontextrestored', + this.onContextRestored, + ); + + this.installOverlay(); + this.report('initialized', 1, { + renderer: 'webgl1', + scheduling: getFrameCallbackApi(this.video).requestVideoFrameCallback + ? 'rvfc' + : 'raf', + }); + } + + public start(): void { + if (this.destroyed || this.started) return; + this.started = true; + this.scheduleNextFrame(); + } + + public getCanvas(): HTMLCanvasElement { + return this.canvas; + } + + public getDiagnostics(): TransparentRendererDiagnostics { + return { + framesRendered: this.frameCount, + averageSubmissionMs: + this.frameCount === 0 + ? 0 + : roundMetric(this.submissionTimeTotalMs / this.frameCount), + maxSubmissionMs: roundMetric(this.submissionTimeMaxMs), + }; + } + + public destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + + const frameCallbackApi = getFrameCallbackApi(this.video); + if ( + this.videoFrameCallbackHandle !== null && + frameCallbackApi.cancelVideoFrameCallback + ) { + frameCallbackApi.cancelVideoFrameCallback(this.videoFrameCallbackHandle); + } + if (this.animationFrameHandle !== null) { + cancelAnimationFrame(this.animationFrameHandle); + } + this.videoFrameCallbackHandle = null; + this.animationFrameHandle = null; + + this.resizeObserver?.disconnect(); + this.resizeObserver = null; + window.removeEventListener('resize', this.syncOverlayGeometry); + this.canvas.removeEventListener('webglcontextlost', this.onContextLost); + this.canvas.removeEventListener( + 'webglcontextrestored', + this.onContextRestored, + ); + this.canvas.remove(); + this.video.style.opacity = this.originalVideoOpacity; + if ( + this.changedParentPosition && + this.parent.style.position === 'relative' + ) { + this.parent.style.position = this.originalParentPosition; + } + + if (this.frameCount > 0) { + this.report( + 'renderer_summary', + this.submissionTimeTotalMs / this.frameCount, + { + frames: this.frameCount, + maxSubmissionMs: roundMetric(this.submissionTimeMaxMs), + }, + ); + } + } + + private installOverlay(): void { + const parentStyle = window.getComputedStyle(this.parent); + if (parentStyle.position === 'static') { + this.parent.style.position = 'relative'; + this.changedParentPosition = true; + } + + Object.assign(this.canvas.style, { + position: 'absolute', + pointerEvents: 'none', + background: 'transparent', + }); + // Keep the canvas immediately above the source video in DOM paint order. + // Appending it to the parent would put it above later sibling UI (for + // example call controls) unless every overlay supplied an explicit z-index. + this.video.insertAdjacentElement('afterend', this.canvas); + this.syncOverlayGeometry(); + + // Opacity keeps the source element active for media playback and iOS + // frame delivery while the transparent canvas supplies the visible pixels. + this.video.style.opacity = '0'; + + if (typeof ResizeObserver !== 'undefined') { + this.resizeObserver = new ResizeObserver(this.syncOverlayGeometry); + this.resizeObserver.observe(this.video); + this.resizeObserver.observe(this.parent); + } else { + window.addEventListener('resize', this.syncOverlayGeometry); + } + } + + private syncOverlayGeometry(): void { + if (this.destroyed) return; + const computedVideoStyle = window.getComputedStyle(this.video); + Object.assign(this.canvas.style, { + left: `${this.video.offsetLeft}px`, + top: `${this.video.offsetTop}px`, + width: `${this.video.offsetWidth}px`, + height: `${this.video.offsetHeight}px`, + borderRadius: computedVideoStyle.borderRadius, + clipPath: computedVideoStyle.clipPath, + transform: computedVideoStyle.transform, + transformOrigin: computedVideoStyle.transformOrigin, + zIndex: computedVideoStyle.zIndex, + }); + } + + private scheduleNextFrame(): void { + if (this.destroyed) return; + + const frameCallbackApi = getFrameCallbackApi(this.video); + if (frameCallbackApi.requestVideoFrameCallback) { + this.videoFrameCallbackHandle = + frameCallbackApi.requestVideoFrameCallback(() => { + this.videoFrameCallbackHandle = null; + this.drawFrame(); + this.scheduleNextFrame(); + }); + return; + } + + this.animationFrameHandle = requestAnimationFrame(() => { + this.animationFrameHandle = null; + this.drawFrame(); + this.scheduleNextFrame(); + }); + } + + private drawFrame(): void { + if ( + this.destroyed || + this.contextLost || + this.video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA || + this.video.videoWidth === 0 || + this.video.videoHeight === 0 + ) { + return; + } + + const startedAt = performance.now(); + const gl = this.gl; + try { + if ( + this.canvas.width !== this.video.videoWidth || + this.canvas.height !== this.video.videoHeight + ) { + // Deliberately use source pixels, not devicePixelRatio. A DPR-scaled + // backing canvas would add work without adding information. + this.canvas.width = this.video.videoWidth; + this.canvas.height = this.video.videoHeight; + } + + gl.viewport(0, 0, this.canvas.width, this.canvas.height); + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.useProgram(this.resources.program); + gl.bindBuffer(gl.ARRAY_BUFFER, this.resources.positionBuffer); + gl.enableVertexAttribArray(this.resources.positionLocation); + gl.vertexAttribPointer( + this.resources.positionLocation, + 2, + gl.FLOAT, + false, + 0, + 0, + ); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.resources.texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1); + gl.texImage2D( + gl.TEXTURE_2D, + 0, + gl.RGBA, + gl.RGBA, + gl.UNSIGNED_BYTE, + this.video, + ); + gl.uniform1f( + this.resources.similarityLocation, + this.keyOptions.similarity, + ); + gl.uniform1f( + this.resources.smoothnessLocation, + this.keyOptions.smoothness, + ); + gl.uniform1f(this.resources.spillLocation, this.keyOptions.spill); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + } catch (error) { + if (!this.renderErrorReported) { + this.renderErrorReported = true; + console.warn( + 'Transparent background renderer failed to process a frame.', + error, + ); + this.report('render_error', 1, { + reason: error instanceof Error ? error.name : 'unknown', + }); + } + return; + } + + const submissionMs = performance.now() - startedAt; + this.frameCount += 1; + this.submissionTimeTotalMs += submissionMs; + this.submissionTimeMaxMs = Math.max(this.submissionTimeMaxMs, submissionMs); + if (this.frameCount === 1) { + this.report('first_frame', submissionMs); + } else if (this.frameCount % TELEMETRY_FRAME_INTERVAL === 0) { + this.report( + 'submission_sample', + this.submissionTimeTotalMs / this.frameCount, + { + frames: this.frameCount, + maxSubmissionMs: roundMetric(this.submissionTimeMaxMs), + }, + ); + } + } + + private createGlResources(): GlResources { + const gl = this.gl; + const vertexShader = compileShader( + gl, + gl.VERTEX_SHADER, + VERTEX_SHADER_SOURCE, + ); + const fragmentShader = compileShader( + gl, + gl.FRAGMENT_SHADER, + FRAGMENT_SHADER_SOURCE, + ); + const program = gl.createProgram(); + if (!program) throw new Error('Unable to create WebGL program.'); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + gl.deleteShader(vertexShader); + gl.deleteShader(fragmentShader); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + const message = gl.getProgramInfoLog(program) ?? 'unknown link error'; + gl.deleteProgram(program); + throw new Error(`Unable to link transparent renderer: ${message}`); + } + + const positionBuffer = gl.createBuffer(); + const texture = gl.createTexture(); + if (!positionBuffer || !texture) { + throw new Error('Unable to allocate transparent renderer resources.'); + } + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), + gl.STATIC_DRAW, + ); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + + const positionLocation = gl.getAttribLocation(program, 'a_position'); + const similarityLocation = gl.getUniformLocation(program, 'u_similarity'); + const smoothnessLocation = gl.getUniformLocation(program, 'u_smoothness'); + const spillLocation = gl.getUniformLocation(program, 'u_spill'); + if ( + positionLocation < 0 || + !similarityLocation || + !smoothnessLocation || + !spillLocation + ) { + throw new Error('Unable to resolve transparent renderer shader inputs.'); + } + + return { + program, + positionBuffer, + texture, + positionLocation, + similarityLocation, + smoothnessLocation, + spillLocation, + }; + } + + private onContextLost(event: Event): void { + event.preventDefault(); + this.contextLost = true; + this.report('context_lost', 1); + } + + private onContextRestored(): void { + try { + this.resources = this.createGlResources(); + this.contextLost = false; + this.renderErrorReported = false; + this.report('context_restored', 1); + } catch (error) { + console.warn( + 'Transparent background WebGL context did not restore.', + error, + ); + this.report('context_restore_failed', 1); + } + } + + private report( + event: string, + value: number, + tags: Record = {}, + ): void { + void sendClientMetric( + ClientMetricMeasurement.CLIENT_METRIC_MEASUREMENT_TRANSPARENT_RENDERER, + roundMetric(value), + { event, ...tags }, + ); + } +} + +function compileShader( + gl: WebGLRenderingContext, + type: number, + source: string, +): WebGLShader { + const shader = gl.createShader(type); + if (!shader) throw new Error('Unable to create WebGL shader.'); + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + const message = gl.getShaderInfoLog(shader) ?? 'unknown compile error'; + gl.deleteShader(shader); + throw new Error(`Unable to compile transparent renderer: ${message}`); + } + return shader; +} + +function resolveKeyOptions( + options?: TransparentBackgroundOptions, +): ResolvedKeyOptions { + return { + similarity: clampUnit(options?.similarity ?? DEFAULT_SIMILARITY), + smoothness: clampUnit(options?.smoothness ?? DEFAULT_SMOOTHNESS), + spill: clampUnit(options?.spill ?? DEFAULT_SPILL), + }; +} + +function clampUnit(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.min(1, Math.max(0, value)); +} + +function roundMetric(value: number): number { + return Math.round(value * 1000) / 1000; +} + +function getFrameCallbackApi( + video: HTMLVideoElement, +): OptionalVideoFrameCallbacks { + // Older WebViews can lack rVFC even though it is present in the current DOM + // TypeScript definitions, so runtime detection is still required. + return video as unknown as OptionalVideoFrameCallbacks; +} diff --git a/src/types/AnamPublicClientOptions.ts b/src/types/AnamPublicClientOptions.ts index eb6c9cc..e1e4d8d 100644 --- a/src/types/AnamPublicClientOptions.ts +++ b/src/types/AnamPublicClientOptions.ts @@ -1,6 +1,7 @@ import { ApiOptions } from '../types'; import { ConnectionMilestoneMetricsOptions } from './ConnectionMilestoneMetricsOptions'; import { VoiceDetectionOptions } from './VoiceDetectionOptions'; +import { TransparentBackgroundOptions } from './TransparentBackgroundOptions'; export interface AnamPublicClientOptions { api?: ApiOptions; @@ -25,4 +26,17 @@ export interface AnamPublicClientOptions { * precedence over `rtcConfiguration.iceServers`. */ rtcConfiguration?: RTCConfiguration; + /** + * Request the avatar's generated green-screen rendition and render it as a + * transparent WebGL canvas over the video element supplied to + * `streamToVideoElement`. + * + * The underlying MediaStream remains an ordinary opaque WebRTC video. Calls + * to `stream()` therefore return the green-screen source; transparent pixels + * exist only in the SDK-managed canvas renderer. + * @default false + */ + transparentBackground?: boolean; + /** Optional client-side key tuning for `transparentBackground`. */ + transparentBackgroundOptions?: TransparentBackgroundOptions; } diff --git a/src/types/TransparentBackgroundOptions.ts b/src/types/TransparentBackgroundOptions.ts new file mode 100644 index 0000000..38ba2a4 --- /dev/null +++ b/src/types/TransparentBackgroundOptions.ts @@ -0,0 +1,24 @@ +/** + * Client-side chroma-key controls used when `transparentBackground` is on. + * + * The defaults are tuned for Anam's generated exact-green avatar rendition. + * Most applications should not need to change these values. + */ +export interface TransparentBackgroundOptions { + /** + * Chroma distance that is treated as fully transparent. Lower values keep + * more green-adjacent detail; higher values remove more of the backdrop. + * @default 0.02 + */ + similarity?: number; + /** + * Width of the soft transition around the key threshold. + * @default 0.36 + */ + smoothness?: number; + /** + * Strength of green-spill suppression at semi-transparent edges. + * @default 0.45 + */ + spill?: number; +} diff --git a/src/types/coreApi/StartSessionOptions.ts b/src/types/coreApi/StartSessionOptions.ts index 69e500d..7eda7d8 100644 --- a/src/types/coreApi/StartSessionOptions.ts +++ b/src/types/coreApi/StartSessionOptions.ts @@ -2,4 +2,10 @@ import { VoiceDetectionOptions } from '../VoiceDetectionOptions'; export interface StartSessionOptions { voiceDetection?: VoiceDetectionOptions; + /** + * Selects the avatar's private green-screen rendition for this session. + * The JavaScript SDK separately composites that rendition into a transparent + * canvas when streaming to a video element. + */ + transparentBackground?: boolean; } diff --git a/src/types/index.ts b/src/types/index.ts index e16de16..4167c98 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,6 @@ export type { AnamClientOptions } from './AnamClientOptions'; export type { ConnectionMilestoneMetricsOptions } from './ConnectionMilestoneMetricsOptions'; +export type { TransparentBackgroundOptions } from './TransparentBackgroundOptions'; export type * from './signalling'; export { SignalMessageAction } from './signalling'; // need to export this explicitly to avoid enum import issues export type * from './streaming'; diff --git a/src/types/streaming/StreamingClientOptions.ts b/src/types/streaming/StreamingClientOptions.ts index 0ae8326..4e9023c 100644 --- a/src/types/streaming/StreamingClientOptions.ts +++ b/src/types/streaming/StreamingClientOptions.ts @@ -2,6 +2,7 @@ import { SignallingClientOptions } from '../../types'; import { EngineApiRestClientOptions } from '../engineApi/EngineApiRestClientOptions'; import { InputAudioOptions } from './InputAudioOptions'; import { ApiGatewayConfig } from '../ApiGatewayConfig'; +import { TransparentBackgroundOptions } from '../TransparentBackgroundOptions'; export interface StreamingClientOptions { engine: EngineApiRestClientOptions; @@ -11,6 +12,10 @@ export interface StreamingClientOptions { rtcConfiguration?: RTCConfiguration; inputAudio: InputAudioOptions; apiGateway?: ApiGatewayConfig; + transparentBackground?: { + enabled: boolean; + keyOptions?: TransparentBackgroundOptions; + }; metrics?: { showPeerConnectionStatsReport?: boolean; peerConnectionStatsReportOutputFormat?: 'console' | 'json'; From 07c87c55d158e020951e4627e0084d0900c46163 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 21 Jul 2026 17:37:22 +0100 Subject: [PATCH 02/15] Match transparent canvas to video fit --- src/modules/TransparentBackgroundRenderer.ts | 109 ++++++++++++++++++- 1 file changed, 105 insertions(+), 4 deletions(-) diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index c78daae..c793899 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -271,11 +271,19 @@ export class TransparentBackgroundRenderer { private syncOverlayGeometry(): void { if (this.destroyed) return; const computedVideoStyle = window.getComputedStyle(this.video); + const fittedRect = resolveObjectFitRect( + this.video.offsetWidth, + this.video.offsetHeight, + this.video.videoWidth, + this.video.videoHeight, + computedVideoStyle.objectFit, + computedVideoStyle.objectPosition, + ); Object.assign(this.canvas.style, { - left: `${this.video.offsetLeft}px`, - top: `${this.video.offsetTop}px`, - width: `${this.video.offsetWidth}px`, - height: `${this.video.offsetHeight}px`, + left: `${this.video.offsetLeft + fittedRect.left}px`, + top: `${this.video.offsetTop + fittedRect.top}px`, + width: `${fittedRect.width}px`, + height: `${fittedRect.height}px`, borderRadius: computedVideoStyle.borderRadius, clipPath: computedVideoStyle.clipPath, transform: computedVideoStyle.transform, @@ -532,6 +540,99 @@ function roundMetric(value: number): number { return Math.round(value * 1000) / 1000; } +interface FittedContentRect { + left: number; + top: number; + width: number; + height: number; +} + +/** Mirror the replaced-content rectangle produced by CSS `object-fit`. */ +function resolveObjectFitRect( + elementWidth: number, + elementHeight: number, + sourceWidth: number, + sourceHeight: number, + objectFit: string, + objectPosition: string, +): FittedContentRect { + if ( + elementWidth <= 0 || + elementHeight <= 0 || + sourceWidth <= 0 || + sourceHeight <= 0 || + objectFit === 'fill' + ) { + return { left: 0, top: 0, width: elementWidth, height: elementHeight }; + } + + const containScale = Math.min( + elementWidth / sourceWidth, + elementHeight / sourceHeight, + ); + let scale: number; + switch (objectFit) { + case 'contain': + scale = containScale; + break; + case 'cover': + scale = Math.max( + elementWidth / sourceWidth, + elementHeight / sourceHeight, + ); + break; + case 'none': + scale = 1; + break; + case 'scale-down': + scale = Math.min(1, containScale); + break; + default: + return { left: 0, top: 0, width: elementWidth, height: elementHeight }; + } + + const width = sourceWidth * scale; + const height = sourceHeight * scale; + const [positionX, positionY] = parseObjectPosition(objectPosition); + return { + left: (elementWidth - width) * positionX, + top: (elementHeight - height) * positionY, + width, + height, + }; +} + +function parseObjectPosition(value: string): [number, number] { + const parts = value.trim().split(/\s+/); + const first = parts[0] ?? '50%'; + const second = parts[1] ?? '50%'; + return [ + parsePositionComponent(first, 'x'), + parsePositionComponent(second, 'y'), + ]; +} + +function parsePositionComponent(value: string, axis: 'x' | 'y'): number { + const keywordPositions: Record = { + left: 0, + top: 0, + center: 0.5, + right: 1, + bottom: 1, + }; + if (value in keywordPositions) return keywordPositions[value]; + if (value.endsWith('%')) { + const percentage = Number.parseFloat(value); + if (Number.isFinite(percentage)) return percentage / 100; + } + + // Computed styles normally normalize object-position to two percentages. + // For unsupported length/calc syntax, centering is the least surprising + // fallback and matches the browser default on either axis. + void axis; + return 0.5; +} + function getFrameCallbackApi( video: HTMLVideoElement, ): OptionalVideoFrameCallbacks { From 173d38ff717ea4fee844279a332e0ef075717fa9 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 21 Jul 2026 18:50:40 +0100 Subject: [PATCH 03/15] Fix transparent overlay object fitting --- package.json | 1 + src/modules/TransparentBackgroundRenderer.ts | 119 ++------------- test/transparentBackgroundGeometryHarness.js | 148 +++++++++++++++++++ 3 files changed, 163 insertions(+), 105 deletions(-) create mode 100644 test/transparentBackgroundGeometryHarness.js diff --git a/package.json b/package.json index 9e727b3..83a4ae6 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "lint": "eslint \"{src,test}/**/*.ts\" --fix", "test": "echo \"Error: no test specified\" && exit 1", "test:connection-milestones": "npm run build:main && node test/connectionMilestonesHarness.js", + "test:transparent-background": "npm run build:main && node test/transparentBackgroundGeometryHarness.js", "watch": "nodemon -e ts --watch src --exec \"npm run build\"", "prepare": "npm run build" }, diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index c793899..906167c 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -271,19 +271,21 @@ export class TransparentBackgroundRenderer { private syncOverlayGeometry(): void { if (this.destroyed) return; const computedVideoStyle = window.getComputedStyle(this.video); - const fittedRect = resolveObjectFitRect( - this.video.offsetWidth, - this.video.offsetHeight, - this.video.videoWidth, - this.video.videoHeight, - computedVideoStyle.objectFit, - computedVideoStyle.objectPosition, - ); + + // Keep the canvas's replaced-element box exactly on the video box. Its + // intrinsic bitmap remains videoWidth x videoHeight, so the browser can + // apply the same object-fit/object-position algorithm to both elements. + // Replaced content is clipped to its own content box, which prevents + // `cover` pixels escaping when the parent has visible overflow. Delegating + // positioning to CSS also preserves the full grammar (lengths, + // edge offsets, and calc()), rather than approximating it in JavaScript. Object.assign(this.canvas.style, { - left: `${this.video.offsetLeft + fittedRect.left}px`, - top: `${this.video.offsetTop + fittedRect.top}px`, - width: `${fittedRect.width}px`, - height: `${fittedRect.height}px`, + left: `${this.video.offsetLeft}px`, + top: `${this.video.offsetTop}px`, + width: `${this.video.offsetWidth}px`, + height: `${this.video.offsetHeight}px`, + objectFit: computedVideoStyle.objectFit, + objectPosition: computedVideoStyle.objectPosition, borderRadius: computedVideoStyle.borderRadius, clipPath: computedVideoStyle.clipPath, transform: computedVideoStyle.transform, @@ -540,99 +542,6 @@ function roundMetric(value: number): number { return Math.round(value * 1000) / 1000; } -interface FittedContentRect { - left: number; - top: number; - width: number; - height: number; -} - -/** Mirror the replaced-content rectangle produced by CSS `object-fit`. */ -function resolveObjectFitRect( - elementWidth: number, - elementHeight: number, - sourceWidth: number, - sourceHeight: number, - objectFit: string, - objectPosition: string, -): FittedContentRect { - if ( - elementWidth <= 0 || - elementHeight <= 0 || - sourceWidth <= 0 || - sourceHeight <= 0 || - objectFit === 'fill' - ) { - return { left: 0, top: 0, width: elementWidth, height: elementHeight }; - } - - const containScale = Math.min( - elementWidth / sourceWidth, - elementHeight / sourceHeight, - ); - let scale: number; - switch (objectFit) { - case 'contain': - scale = containScale; - break; - case 'cover': - scale = Math.max( - elementWidth / sourceWidth, - elementHeight / sourceHeight, - ); - break; - case 'none': - scale = 1; - break; - case 'scale-down': - scale = Math.min(1, containScale); - break; - default: - return { left: 0, top: 0, width: elementWidth, height: elementHeight }; - } - - const width = sourceWidth * scale; - const height = sourceHeight * scale; - const [positionX, positionY] = parseObjectPosition(objectPosition); - return { - left: (elementWidth - width) * positionX, - top: (elementHeight - height) * positionY, - width, - height, - }; -} - -function parseObjectPosition(value: string): [number, number] { - const parts = value.trim().split(/\s+/); - const first = parts[0] ?? '50%'; - const second = parts[1] ?? '50%'; - return [ - parsePositionComponent(first, 'x'), - parsePositionComponent(second, 'y'), - ]; -} - -function parsePositionComponent(value: string, axis: 'x' | 'y'): number { - const keywordPositions: Record = { - left: 0, - top: 0, - center: 0.5, - right: 1, - bottom: 1, - }; - if (value in keywordPositions) return keywordPositions[value]; - if (value.endsWith('%')) { - const percentage = Number.parseFloat(value); - if (Number.isFinite(percentage)) return percentage / 100; - } - - // Computed styles normally normalize object-position to two percentages. - // For unsupported length/calc syntax, centering is the least surprising - // fallback and matches the browser default on either axis. - void axis; - return 0.5; -} - function getFrameCallbackApi( video: HTMLVideoElement, ): OptionalVideoFrameCallbacks { diff --git a/test/transparentBackgroundGeometryHarness.js b/test/transparentBackgroundGeometryHarness.js new file mode 100644 index 0000000..2275f4e --- /dev/null +++ b/test/transparentBackgroundGeometryHarness.js @@ -0,0 +1,148 @@ +const assert = require('assert'); +const { setClientMetricsDisabled } = require('../dist/main/lib/ClientMetrics'); +const { + TransparentBackgroundRenderer, +} = require('../dist/main/modules/TransparentBackgroundRenderer'); + +setClientMetricsDisabled(true); + +const createWebGlStub = () => ({ + VERTEX_SHADER: 1, + FRAGMENT_SHADER: 2, + COMPILE_STATUS: 3, + LINK_STATUS: 4, + ARRAY_BUFFER: 5, + STATIC_DRAW: 6, + TEXTURE_2D: 7, + TEXTURE_MIN_FILTER: 8, + TEXTURE_MAG_FILTER: 9, + LINEAR: 10, + TEXTURE_WRAP_S: 11, + TEXTURE_WRAP_T: 12, + CLAMP_TO_EDGE: 13, + createShader: () => ({}), + shaderSource: () => {}, + compileShader: () => {}, + getShaderParameter: () => true, + getShaderInfoLog: () => '', + deleteShader: () => {}, + createProgram: () => ({}), + attachShader: () => {}, + linkProgram: () => {}, + getProgramParameter: () => true, + getProgramInfoLog: () => '', + deleteProgram: () => {}, + createBuffer: () => ({}), + createTexture: () => ({}), + bindBuffer: () => {}, + bufferData: () => {}, + bindTexture: () => {}, + texParameteri: () => {}, + getAttribLocation: () => 0, + getUniformLocation: () => ({}), +}); + +const createRendererHarness = ({ objectFit, objectPosition }) => { + const parent = { + style: { position: '' }, + }; + const canvas = { + style: {}, + dataset: {}, + id: '', + setAttribute: () => {}, + getContext: () => createWebGlStub(), + addEventListener: () => {}, + removeEventListener: () => {}, + remove: () => {}, + }; + const video = { + parentElement: parent, + id: 'persona-video', + style: { opacity: '' }, + offsetLeft: 13, + offsetTop: 17, + offsetWidth: 100, + offsetHeight: 100, + videoWidth: 200, + videoHeight: 100, + insertAdjacentElement: (position, element) => { + assert.equal(position, 'afterend'); + assert.equal(element, canvas); + }, + }; + + global.document = { + createElement: (tagName) => { + assert.equal(tagName, 'canvas'); + return canvas; + }, + }; + global.window = { + getComputedStyle: (element) => + element === parent + ? { position: 'relative', overflow: 'visible' } + : { + objectFit, + objectPosition, + borderRadius: '12px', + clipPath: 'none', + transform: 'none', + transformOrigin: '50% 50%', + zIndex: 'auto', + }, + addEventListener: () => {}, + removeEventListener: () => {}, + }; + global.ResizeObserver = class { + observe() {} + disconnect() {} + }; + + const renderer = new TransparentBackgroundRenderer(video); + return { canvas, renderer }; +}; + +const cover = createRendererHarness({ + objectFit: 'cover', + objectPosition: '50% 50%', +}); +assert.deepEqual( + { + left: cover.canvas.style.left, + top: cover.canvas.style.top, + width: cover.canvas.style.width, + height: cover.canvas.style.height, + objectFit: cover.canvas.style.objectFit, + objectPosition: cover.canvas.style.objectPosition, + }, + { + left: '13px', + top: '17px', + width: '100px', + height: '100px', + objectFit: 'cover', + objectPosition: '50% 50%', + }, + 'cover must stay inside the video element box even when the parent overflow is visible', +); +cover.renderer.destroy(); + +for (const objectPosition of [ + '10px 20px', + 'calc(100% - 12px) calc(50% + 4px)', + 'right 10px bottom 20px', +]) { + const harness = createRendererHarness({ + objectFit: 'contain', + objectPosition, + }); + assert.equal( + harness.canvas.style.objectPosition, + objectPosition, + `object-position must be delegated to CSS without rewriting ${objectPosition}`, + ); + harness.renderer.destroy(); +} + +console.log('transparent background geometry harness passed'); From 7d4142d0cf08c42b69d9761dcf9c0b70bbac22ef Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 21 Jul 2026 21:13:51 +0100 Subject: [PATCH 04/15] Fix transparent edge reconstruction --- package.json | 2 +- src/modules/TransparentBackgroundRenderer.ts | 85 +++++++++--- src/types/TransparentBackgroundOptions.ts | 10 +- ...nsparentBackgroundReconstructionHarness.js | 124 ++++++++++++++++++ 4 files changed, 201 insertions(+), 20 deletions(-) create mode 100644 test/transparentBackgroundReconstructionHarness.js diff --git a/package.json b/package.json index 83a4ae6..85e00ae 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "lint": "eslint \"{src,test}/**/*.ts\" --fix", "test": "echo \"Error: no test specified\" && exit 1", "test:connection-milestones": "npm run build:main && node test/connectionMilestonesHarness.js", - "test:transparent-background": "npm run build:main && node test/transparentBackgroundGeometryHarness.js", + "test:transparent-background": "npm run build:main && node test/transparentBackgroundGeometryHarness.js && node test/transparentBackgroundReconstructionHarness.js", "watch": "nodemon -e ts --watch src --exec \"npm run build\"", "prepare": "npm run build" }, diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index 906167c..c53e6a9 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -4,12 +4,14 @@ import { } from '../lib/ClientMetrics'; import { TransparentBackgroundOptions } from '../types/TransparentBackgroundOptions'; -// Calibrated on 96 person mattes after the engine's initial H.264 Main/I420 -// profile (520 kbps). A broad soft transition preserves fractional-alpha hair -// substantially better than a narrow conventional chroma-key threshold. -const DEFAULT_SIMILARITY = 0.02; -const DEFAULT_SMOOTHNESS = 0.36; -const DEFAULT_SPILL = 0.45; +// Calibrated on the held-out person-matting set after the engine's JPEG q90 +// and H.264 Main/I420 path. A broad transition retains fractional-alpha hair; +// the exact carrier inverse below removes the green contribution from it. +const DEFAULT_SIMILARITY = 0.005; +const DEFAULT_SMOOTHNESS = 0.56; +const DEFAULT_SPILL = 1.0; +const SPILL_GUARD_START_ALPHA = 0.9; +const SPILL_GUARD_END_ALPHA = 0.995; const TELEMETRY_FRAME_INTERVAL = 250; const VERTEX_SHADER_SOURCE = ` @@ -23,10 +25,13 @@ void main() { `; // Key in chroma space so luminance variation introduced by H.264 does not -// turn a uniformly-green background into a noisy alpha plane. Output is -// premultiplied because that is the browser compositor's preferred WebGL -// canvas representation and avoids a bright fringe at translucent edges. -const FRAGMENT_SHADER_SOURCE = ` +// turn a uniformly-green background into a noisy alpha plane. The source +// asset is composited over green directly in gamma-encoded sRGB, so subtracting +// the estimated green contribution recovers premultiplied foreground. Merely +// multiplying the carrier RGB by alpha would retain that green contribution +// and produce a bright fringe at translucent edges. +/** @internal */ +export const FRAGMENT_SHADER_SOURCE = ` precision mediump float; uniform sampler2D u_frame; @@ -51,13 +56,26 @@ void main() { chromaDistance ); - float greenExcess = max(rgb.g - max(rgb.r, rgb.b), 0.0); - rgb.g = max( - rgb.g - greenExcess * u_spill * (1.0 - alpha), + vec3 premultiplied = clamp( + rgb - (1.0 - alpha) * vec3(0.0, 1.0, 0.0), + vec3(0.0), + vec3(alpha) + ); + float spillGuard = 1.0 - smoothstep( + ${SPILL_GUARD_START_ALPHA.toFixed(3)}, + ${SPILL_GUARD_END_ALPHA.toFixed(3)}, + alpha + ); + float greenExcess = max( + premultiplied.g - max(premultiplied.r, premultiplied.b), + 0.0 + ); + premultiplied.g = max( + premultiplied.g - greenExcess * u_spill * spillGuard, 0.0 ); - gl_FragColor = vec4(rgb * alpha, alpha); + gl_FragColor = vec4(premultiplied, alpha); } `; @@ -523,7 +541,8 @@ function compileShader( return shader; } -function resolveKeyOptions( +/** @internal */ +export function resolveKeyOptions( options?: TransparentBackgroundOptions, ): ResolvedKeyOptions { return { @@ -533,11 +552,47 @@ function resolveKeyOptions( }; } +/** + * CPU reference for the fragment shader's carrier inversion. Kept alongside + * the shader so focused tests can verify edge cases without a GPU dependency. + * + * @internal + */ +export function reconstructPremultipliedForeground( + rgb: readonly [number, number, number], + alphaValue: number, + spillValue = DEFAULT_SPILL, +): [number, number, number] { + const alpha = clampUnit(alphaValue); + const spill = clampUnit(spillValue); + const premultiplied: [number, number, number] = [ + Math.min(alpha, Math.max(0, rgb[0])), + Math.min(alpha, Math.max(0, rgb[1] - (1 - alpha))), + Math.min(alpha, Math.max(0, rgb[2])), + ]; + const spillGuard = + 1 - smoothstep(SPILL_GUARD_START_ALPHA, SPILL_GUARD_END_ALPHA, alpha); + const greenExcess = Math.max( + premultiplied[1] - Math.max(premultiplied[0], premultiplied[2]), + 0, + ); + premultiplied[1] = Math.max( + premultiplied[1] - greenExcess * spill * spillGuard, + 0, + ); + return premultiplied; +} + function clampUnit(value: number): number { if (!Number.isFinite(value)) return 0; return Math.min(1, Math.max(0, value)); } +function smoothstep(edge0: number, edge1: number, value: number): number { + const t = clampUnit((value - edge0) / (edge1 - edge0)); + return t * t * (3 - 2 * t); +} + function roundMetric(value: number): number { return Math.round(value * 1000) / 1000; } diff --git a/src/types/TransparentBackgroundOptions.ts b/src/types/TransparentBackgroundOptions.ts index 38ba2a4..b5ca7d0 100644 --- a/src/types/TransparentBackgroundOptions.ts +++ b/src/types/TransparentBackgroundOptions.ts @@ -8,17 +8,19 @@ export interface TransparentBackgroundOptions { /** * Chroma distance that is treated as fully transparent. Lower values keep * more green-adjacent detail; higher values remove more of the backdrop. - * @default 0.02 + * @default 0.005 */ similarity?: number; /** * Width of the soft transition around the key threshold. - * @default 0.36 + * @default 0.56 */ smoothness?: number; /** - * Strength of green-spill suppression at semi-transparent edges. - * @default 0.45 + * Strength of guarded green-spill suppression at semi-transparent edges. + * Set to `0` to use only the exact green-carrier inverse, or `1` for the + * full clamp. Opaque foreground colours are preserved. + * @default 1 */ spill?: number; } diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js new file mode 100644 index 0000000..acc7c26 --- /dev/null +++ b/test/transparentBackgroundReconstructionHarness.js @@ -0,0 +1,124 @@ +const assert = require('assert'); +const { + FRAGMENT_SHADER_SOURCE, + reconstructPremultipliedForeground, + resolveKeyOptions, +} = require('../dist/main/modules/TransparentBackgroundRenderer'); + +const closeTo = (actual, expected, message, epsilon = 1e-7) => { + assert.equal(actual.length, expected.length, message); + actual.forEach((value, index) => { + assert.ok( + Math.abs(value - expected[index]) <= epsilon, + `${message}: channel ${index} expected ${expected[index]}, got ${value}`, + ); + }); +}; + +assert.deepEqual( + resolveKeyOptions(), + { similarity: 0.005, smoothness: 0.56, spill: 1 }, + 'calibrated defaults must remain explicit and covered by regression tests', +); +assert.deepEqual( + resolveKeyOptions({ similarity: 0.1, smoothness: 0.2, spill: 0 }), + { similarity: 0.1, smoothness: 0.2, spill: 0 }, + 'all public key controls must remain configurable, including inverse-only spill=0', +); + +closeTo( + reconstructPremultipliedForeground([0, 1, 0], 0), + [0, 0, 0], + 'fully transparent exact green must reconstruct to transparent black', +); + +const blonde = [0.8, 0.7, 0.55]; +const blondeAlpha = 0.5; +const blondeCarrier = [ + blondeAlpha * blonde[0], + blondeAlpha * blonde[1] + (1 - blondeAlpha), + blondeAlpha * blonde[2], +]; +closeTo( + reconstructPremultipliedForeground(blondeCarrier, blondeAlpha), + blonde.map((channel) => channel * blondeAlpha), + 'fractional blonde-like sRGB carrier must recover its premultiplied foreground', +); + +closeTo( + reconstructPremultipliedForeground([0.2, 0.3, 0.4], 1), + [0.2, 0.3, 0.4], + 'opaque ordinary colour must stay unchanged', +); +closeTo( + reconstructPremultipliedForeground([0.05, 0.9, 0.08], 1), + [0.05, 0.9, 0.08], + 'opaque genuine green must stay unchanged', +); + +const guardEndAlpha = 0.995; +const nearOpaqueGreen = [0.05, 0.9, 0.08]; +const nearOpaqueGreenCarrier = [ + guardEndAlpha * nearOpaqueGreen[0], + guardEndAlpha * nearOpaqueGreen[1] + (1 - guardEndAlpha), + guardEndAlpha * nearOpaqueGreen[2], +]; +closeTo( + reconstructPremultipliedForeground( + nearOpaqueGreenCarrier, + guardEndAlpha, + ), + nearOpaqueGreen.map((channel) => channel * guardEndAlpha), + 'green-spill clamp must be fully faded out by alpha 0.995', +); + +const greenForeground = [0.1, 0.9, 0.1]; +const greenAlpha = 0.5; +const greenCarrier = [ + greenAlpha * greenForeground[0], + greenAlpha * greenForeground[1] + (1 - greenAlpha), + greenAlpha * greenForeground[2], +]; +closeTo( + reconstructPremultipliedForeground(greenCarrier, greenAlpha, 0), + greenForeground.map((channel) => channel * greenAlpha), + 'spill=0 must perform only the exact carrier inverse', +); +closeTo( + reconstructPremultipliedForeground(greenCarrier, greenAlpha, 1), + [0.05, 0.05, 0.05], + 'spill=1 must apply the full guarded green-excess clamp at low alpha', +); + +for (const { rgb, alpha, spill } of [ + { rgb: [-0.2, 1.4, 0.7], alpha: 0.3, spill: 0 }, + { rgb: [0.8, 0.1, 1.2], alpha: 0.6, spill: 1 }, + { rgb: [0.5, 0.9, 0.2], alpha: 0.95, spill: 0.5 }, +]) { + const reconstructed = reconstructPremultipliedForeground(rgb, alpha, spill); + reconstructed.forEach((channel) => { + assert.ok(channel >= 0, 'premultiplied channels must not be negative'); + assert.ok( + channel <= alpha + 1e-7, + 'premultiplied channels must not exceed alpha', + ); + }); +} + +assert.match( + FRAGMENT_SHADER_SOURCE, + /rgb - \(1\.0 - alpha\) \* vec3\(0\.0, 1\.0, 0\.0\)/, + 'shader must invert the gamma-encoded green carrier', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /gl_FragColor = vec4\(premultiplied, alpha\)/, + 'shader must submit already-premultiplied RGB', +); +assert.doesNotMatch( + FRAGMENT_SHADER_SOURCE, + /gl_FragColor = vec4\(rgb \* alpha, alpha\)/, + 'shader must not multiply the green carrier by alpha', +); + +console.log('transparent background reconstruction harness passed'); From ce7e857ba92408536c512022398b22c6b8104daa Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 21 Jul 2026 23:08:20 +0100 Subject: [PATCH 05/15] Add packed alpha transparent video transport --- README.md | 22 +- package.json | 2 +- src/AnamClient.ts | 18 +- src/modules/PackedAlphaTransport.ts | 120 +++++ src/modules/StreamingClient.ts | 18 +- src/modules/TransparentBackgroundRenderer.ts | 418 ++++++++++++++---- src/types/AnamPublicClientOptions.ts | 12 +- src/types/TransparentBackgroundTransport.ts | 3 + src/types/coreApi/StartSessionOptions.ts | 6 + src/types/streaming/StreamingClientOptions.ts | 2 + test/packedAlphaTransportHarness.js | 160 +++++++ test/transparentBackgroundGeometryHarness.js | 66 ++- ...nsparentBackgroundReconstructionHarness.js | 43 +- 13 files changed, 771 insertions(+), 119 deletions(-) create mode 100644 src/modules/PackedAlphaTransport.ts create mode 100644 src/types/TransparentBackgroundTransport.ts create mode 100644 test/packedAlphaTransportHarness.js diff --git a/README.md b/README.md index 7c7cb80..324dccd 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,11 @@ This will start a new session using the pre-configured persona id and start stre ### Transparent avatar background -Set `transparentBackground` when creating the client. The SDK requests the -avatar's green-screen rendition and renders it through a source-resolution -WebGL canvas over your video element. +Set `transparentBackground` when creating the client. The SDK requests a +packed colour/matte rendition and reconstructs it through a source-resolution +WebGL canvas over your video element. Browsers that explicitly report no +support for the packed H.264 stream, and servers that return the original +1152x768 frame, automatically use the legacy green-screen keyer. ```typescript const anamClient = createClient('your-session-token', { @@ -79,12 +81,14 @@ await anamClient.streamToVideoElement('video-element-id'); ``` The supplied video must be attached to the DOM and should use `autoplay` and -`playsinline` as usual. The underlying WebRTC `MediaStream` is still an opaque -green-screen video; transparency exists in the SDK-managed canvas only. As a -result, `stream()` returns the raw green source, and video-only browser features -such as native controls and picture-in-picture do not automatically capture the -transparent canvas. Use `getTransparentBackgroundCanvas()` if you need the -managed canvas element for layout or capture behavior. +`playsinline` as usual. The underlying WebRTC `MediaStream` is still ordinary +opaque video: it contains vertically packed colour and alpha planes, or a +green-screen frame on the compatibility path. Transparency exists in the +SDK-managed canvas only. As a result, `stream()` returns that raw transport, +and video-only browser features such as native controls and picture-in-picture +do not automatically capture the transparent canvas. Use +`getTransparentBackgroundCanvas()` if you need the managed canvas element for +layout or capture behavior. To stop a session use the `stopStreaming` method. diff --git a/package.json b/package.json index 85e00ae..3ad97e4 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "lint": "eslint \"{src,test}/**/*.ts\" --fix", "test": "echo \"Error: no test specified\" && exit 1", "test:connection-milestones": "npm run build:main && node test/connectionMilestonesHarness.js", - "test:transparent-background": "npm run build:main && node test/transparentBackgroundGeometryHarness.js && node test/transparentBackgroundReconstructionHarness.js", + "test:transparent-background": "npm run build:main && node test/transparentBackgroundGeometryHarness.js && node test/transparentBackgroundReconstructionHarness.js && node test/packedAlphaTransportHarness.js", "watch": "nodemon -e ts --watch src --exec \"npm run build\"", "prepare": "npm run build" }, diff --git a/src/AnamClient.ts b/src/AnamClient.ts index 6b2b451..574a6b2 100644 --- a/src/AnamClient.ts +++ b/src/AnamClient.ts @@ -37,6 +37,7 @@ import { import { AgentAudioInputStream } from './types/AgentAudioInputStream'; import { TalkMessageStream } from './types/TalkMessageStream'; import { ToolCallManager } from './modules/ToolCallManager'; +import { buildTransparentBackgroundSessionOptions } from './modules/PackedAlphaTransport'; export default class AnamClient { private publicEventEmitter: PublicEventEmitter; private internalEventEmitter: InternalEventEmitter; @@ -198,15 +199,19 @@ export default class AnamClient { return undefined; } - private buildStartSessionOptionsForClient(): StartSessionOptions | undefined { + private async buildStartSessionOptionsForClient(): Promise< + StartSessionOptions | undefined + > { const sessionOptions: StartSessionOptions = {}; if (this.clientOptions?.voiceDetection) { sessionOptions.voiceDetection = this.clientOptions.voiceDetection; } - if (this.clientOptions?.transparentBackground !== undefined) { - sessionOptions.transparentBackground = - this.clientOptions.transparentBackground; - } + Object.assign( + sessionOptions, + await buildTransparentBackgroundSessionOptions( + this.clientOptions?.transparentBackground, + ), + ); // return undefined if no options are set if (Object.keys(sessionOptions).length === 0) { return undefined; @@ -249,7 +254,7 @@ export default class AnamClient { const config = this.personaConfig; // build session options from client options const sessionOptions: StartSessionOptions | undefined = - this.buildStartSessionOptionsForClient(); + await this.buildStartSessionOptionsForClient(); // start a new session connectionMilestones?.record('start_session_request_started'); let response: StartSessionResponse; @@ -328,6 +333,7 @@ export default class AnamClient { transparentBackground: { enabled: this.clientOptions?.transparentBackground === true, keyOptions: this.clientOptions?.transparentBackgroundOptions, + transport: sessionOptions?.transparentBackgroundTransport, }, metrics: { showPeerConnectionStatsReport: diff --git a/src/modules/PackedAlphaTransport.ts b/src/modules/PackedAlphaTransport.ts new file mode 100644 index 0000000..e076f0d --- /dev/null +++ b/src/modules/PackedAlphaTransport.ts @@ -0,0 +1,120 @@ +import { StartSessionOptions } from '../types/coreApi/StartSessionOptions'; +import { + PACKED_ALPHA_TRANSPORT, + TransparentBackgroundTransport, +} from '../types/TransparentBackgroundTransport'; + +const PACKED_ALPHA_DECODING_CONFIGURATION: MediaDecodingConfiguration = { + type: 'webrtc', + video: { + // Match the RTP format we actually negotiate. This is also the form used + // by the Media Capabilities specification's WebRTC H.264 example. + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + width: 1152, + height: 1536, + bitrate: 2_500_000, + framerate: 25, + }, +}; + +type MediaCapabilitiesLike = Pick; + +/** @internal */ +export type PackedAlphaCapability = 'supported' | 'unsupported' | 'unknown'; + +/** + * Query support for the H.264 Main Level 4.0 stream used by packed-alpha-v1. + * Missing or inconclusive MediaCapabilities implementations are reported as + * unknown rather than unsupported so older browsers retain their existing + * WebRTC codec negotiation path. + * + * @internal + */ +export async function detectPackedAlphaCapability( + mediaCapabilities: MediaCapabilitiesLike | undefined = typeof navigator === + 'undefined' + ? undefined + : navigator.mediaCapabilities, +): Promise { + if (!mediaCapabilities?.decodingInfo) return 'unknown'; + + try { + const result = await mediaCapabilities.decodingInfo( + PACKED_ALPHA_DECODING_CONFIGURATION, + ); + // The packed frame doubles the decoded pixel count. A decoder that can + // technically accept Main Level 4.0 but is not expected to sustain this + // configuration should use the lower-resolution compatibility path. + return result.supported && result.smooth ? 'supported' : 'unsupported'; + } catch { + // Some otherwise-compatible browsers expose MediaCapabilities but reject + // WebRTC configurations. That is not evidence that H.264 Level 4.0 cannot + // be decoded, so let normal SDP negotiation decide. + return 'unknown'; + } +} + +/** @internal */ +export async function buildTransparentBackgroundSessionOptions( + enabled: boolean | undefined, + mediaCapabilities?: MediaCapabilitiesLike, +): Promise< + Pick< + StartSessionOptions, + 'transparentBackground' | 'transparentBackgroundTransport' + > +> { + if (enabled === undefined) return {}; + if (!enabled) return { transparentBackground: false }; + + const capability = await detectPackedAlphaCapability(mediaCapabilities); + if (capability === 'unsupported') { + console.warn( + 'Packed transparent-background video is unsupported or is not expected to decode smoothly on this device; falling back to legacy green-screen keying.', + ); + return { transparentBackground: true }; + } + + return { + transparentBackground: true, + transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, + }; +} + +/** + * Raise only H.264 Main Level 3.1 payloads to Level 4.0. Other H.264 profiles, + * codecs and already-higher levels are deliberately left untouched. + * + * @internal + */ +export function promotePackedAlphaH264Level(sdp: string): string { + const h264PayloadTypes = new Set(); + for (const line of sdp.split(/\r?\n/)) { + const match = line.match(/^a=rtpmap:(\d+)\s+H264\/90000(?:\s|$)/i); + if (match) h264PayloadTypes.add(match[1]); + } + + return sdp.replace( + /^a=fmtp:(\d+)(\s+)([^\r\n]*)$/gim, + (line, payloadType: string, separator: string, parameters: string) => { + if (!h264PayloadTypes.has(payloadType)) return line; + + const promotedParameters = parameters.replace( + /(^|;)(\s*profile-level-id\s*=\s*)4d001f(?=\s*(?:;|$))/i, + (_match, boundary: string, prefix: string) => + `${boundary}${prefix}4d0028`, + ); + return `a=fmtp:${payloadType}${separator}${promotedParameters}`; + }, + ); +} + +/** @internal */ +export function prepareOfferForTransparentBackgroundTransport( + offer: RTCSessionDescriptionInit, + transport: TransparentBackgroundTransport | undefined, +): RTCSessionDescriptionInit { + if (transport !== PACKED_ALPHA_TRANSPORT || !offer.sdp) return offer; + return { ...offer, sdp: promotePackedAlphaH264Level(offer.sdp) }; +} diff --git a/src/modules/StreamingClient.ts b/src/modules/StreamingClient.ts index 59c9d28..1281a8c 100644 --- a/src/modules/StreamingClient.ts +++ b/src/modules/StreamingClient.ts @@ -40,6 +40,8 @@ import { } from '../types/streaming/WebRtcToolCallEvent'; import { ToolCallManager } from './ToolCallManager'; import { TransparentBackgroundRenderer } from './TransparentBackgroundRenderer'; +import { prepareOfferForTransparentBackgroundTransport } from './PackedAlphaTransport'; +import { TransparentBackgroundTransport } from '../types/TransparentBackgroundTransport'; const SUCCESS_METRIC_POLLING_TIMEOUT_MS = 15000; // After this time we will stop polling for the first frame and consider the session a failure. const STATS_COLLECTION_INTERVAL_MS = 5000; @@ -90,6 +92,9 @@ export class StreamingClient { private readonly transparentBackgroundKeyOptions: | TransparentBackgroundOptions | undefined; + private readonly transparentBackgroundTransport: + | TransparentBackgroundTransport + | undefined; private videoStream: MediaStream | null = null; private audioStream: MediaStream | null = null; private inputAudioState: InputAudioState = { @@ -128,6 +133,8 @@ export class StreamingClient { options.transparentBackground?.enabled === true; this.transparentBackgroundKeyOptions = options.transparentBackground?.keyOptions; + this.transparentBackgroundTransport = + options.transparentBackground?.transport; // initialize input audio state const { inputAudio } = options; this.inputAudioState = inputAudio.inputAudioState; @@ -502,6 +509,7 @@ export class StreamingClient { this.transparentBackgroundRenderer = new TransparentBackgroundRenderer( this.videoElement, this.transparentBackgroundKeyOptions, + this.transparentBackgroundTransport, ); } } @@ -1088,7 +1096,10 @@ export class StreamingClient { this.connectionReceivedAnswer = false; this.remoteIceCandidateBuffer = []; - const offer = await this.peerConnection.createOffer({ iceRestart: true }); + const offer = prepareOfferForTransparentBackgroundTransport( + await this.peerConnection.createOffer({ iceRestart: true }), + this.transparentBackgroundTransport, + ); // Buffer local candidates that gather from setLocalDescription until the // re-offer is sent. The engine queues remote candidates only while it has // no remote description; during a restart it still holds the OLD ICE @@ -1549,7 +1560,10 @@ export class StreamingClient { try { this.connectionMilestones?.record('offer_creation_started'); const offer: RTCSessionDescriptionInit = - await this.peerConnection.createOffer(); + prepareOfferForTransparentBackgroundTransport( + await this.peerConnection.createOffer(), + this.transparentBackgroundTransport, + ); this.connectionMilestones?.record('offer_creation_completed'); await this.peerConnection.setLocalDescription(offer); this.connectionMilestones?.record('local_description_set'); diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index c53e6a9..1f6e862 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -3,6 +3,10 @@ import { sendClientMetric, } from '../lib/ClientMetrics'; import { TransparentBackgroundOptions } from '../types/TransparentBackgroundOptions'; +import { + PACKED_ALPHA_TRANSPORT, + TransparentBackgroundTransport, +} from '../types/TransparentBackgroundTransport'; // Calibrated on the held-out person-matting set after the engine's JPEG q90 // and H.264 Main/I420 path. A broad transition retains fractional-alpha hair; @@ -10,6 +14,11 @@ import { TransparentBackgroundOptions } from '../types/TransparentBackgroundOpti const DEFAULT_SIMILARITY = 0.005; const DEFAULT_SMOOTHNESS = 0.56; const DEFAULT_SPILL = 1.0; +// H.264/JPEG ringing leaves a very small non-zero alpha tail in otherwise +// uniform background pixels. Keeping it produces a faint grey/coloured veil +// over the page. The five-person transport holdout put the live p99.9 tail at +// ~0.012; 0.02 clears it while changing edge error by only 0.2%. +const BACKGROUND_ALPHA_FLOOR = 0.02; const SPILL_GUARD_START_ALPHA = 0.9; const SPILL_GUARD_END_ALPHA = 0.995; const TELEMETRY_FRAME_INTERVAL = 250; @@ -55,6 +64,7 @@ void main() { u_similarity + max(u_smoothness, 0.0001), chromaDistance ); + alpha *= step(${BACKGROUND_ALPHA_FLOOR.toFixed(3)}, alpha); vec3 premultiplied = clamp( rgb - (1.0 - alpha) * vec3(0.0, 1.0, 0.0), @@ -79,6 +89,36 @@ void main() { } `; +// packed-alpha-v1 carries already-premultiplied colour in the top half of the +// video and a grayscale alpha plane in the bottom half. Sampling the planes +// directly avoids another estimate, despill pass, or alpha cutoff in-browser. +/** @internal */ +export const PACKED_ALPHA_FRAGMENT_SHADER_SOURCE = ` +precision mediump float; + +uniform sampler2D u_frame; +varying vec2 v_texCoord; + +void main() { + vec2 colourCoord = vec2( + v_texCoord.x, + 0.5 + v_texCoord.y * 0.5 + ); + vec2 alphaCoord = vec2( + v_texCoord.x, + v_texCoord.y * 0.5 + ); + vec3 premultiplied = texture2D(u_frame, colourCoord).rgb; + vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; + float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); + + // Compression can make an individual premultiplied colour channel exceed + // alpha by a code value. Clamp only that invalid premultiplied state; do not + // estimate, despill, or threshold the transported matte. + gl_FragColor = vec4(min(premultiplied, vec3(alpha)), alpha); +} +`; + type OptionalVideoFrameCallbacks = { requestVideoFrameCallback?: ( callback: (now: DOMHighResTimeStamp) => void, @@ -93,13 +133,27 @@ interface ResolvedKeyOptions { } interface GlResources { - program: WebGLProgram; positionBuffer: WebGLBuffer; texture: WebGLTexture; - positionLocation: number; + legacyProgram: WebGLProgram; + legacyPositionLocation: number; similarityLocation: WebGLUniformLocation; smoothnessLocation: WebGLUniformLocation; spillLocation: WebGLUniformLocation; + packedAlphaProgram: WebGLProgram; + packedAlphaPositionLocation: number; +} + +export type TransparentFrameMode = + | 'green-key-v1' + | 'packed-alpha-v1' + | 'unsupported'; + +export interface TransparentFrameGeometry { + mode: TransparentFrameMode; + canvasWidth: number; + canvasHeight: number; + reason?: string; } export interface TransparentRendererDiagnostics { @@ -113,7 +167,9 @@ export class TransparentBackgroundRenderer { private readonly canvas: HTMLCanvasElement; private readonly parent: HTMLElement; private readonly keyOptions: ResolvedKeyOptions; + private readonly transport: TransparentBackgroundTransport | undefined; private readonly gl: WebGLRenderingContext; + private readonly maxTextureSize: number; private resources: GlResources; private resizeObserver: ResizeObserver | null = null; private videoFrameCallbackHandle: number | null = null; @@ -128,8 +184,13 @@ export class TransparentBackgroundRenderer { private readonly originalVideoOpacity: string; private readonly originalParentPosition: string; private changedParentPosition = false; + private lastGeometrySignature: string | null = null; - constructor(video: HTMLVideoElement, options?: TransparentBackgroundOptions) { + constructor( + video: HTMLVideoElement, + options?: TransparentBackgroundOptions, + transport?: TransparentBackgroundTransport, + ) { if (!video.parentElement) { this.report('initialization_failed', 1, { reason: 'missing_parent' }); throw new Error( @@ -140,6 +201,7 @@ export class TransparentBackgroundRenderer { this.video = video; this.parent = video.parentElement; this.keyOptions = resolveKeyOptions(options); + this.transport = transport; this.originalVideoOpacity = video.style.opacity; this.originalParentPosition = this.parent.style.position; @@ -168,6 +230,7 @@ export class TransparentBackgroundRenderer { ); } this.gl = gl; + this.maxTextureSize = gl.getParameter(gl.MAX_TEXTURE_SIZE) as number; this.resources = this.createGlResources(); this.onContextLost = this.onContextLost.bind(this); @@ -182,6 +245,8 @@ export class TransparentBackgroundRenderer { this.installOverlay(); this.report('initialized', 1, { renderer: 'webgl1', + requestedTransport: this.transport ?? 'green-key-v1', + maxTextureSize: this.maxTextureSize, scheduling: getFrameCallbackApi(this.video).requestVideoFrameCallback ? 'rvfc' : 'raf', @@ -234,6 +299,7 @@ export class TransparentBackgroundRenderer { 'webglcontextrestored', this.onContextRestored, ); + this.deleteGlResources(this.resources); this.canvas.remove(); this.video.style.opacity = this.originalVideoOpacity; if ( @@ -291,8 +357,9 @@ export class TransparentBackgroundRenderer { const computedVideoStyle = window.getComputedStyle(this.video); // Keep the canvas's replaced-element box exactly on the video box. Its - // intrinsic bitmap remains videoWidth x videoHeight, so the browser can - // apply the same object-fit/object-position algorithm to both elements. + // intrinsic bitmap is the reconstructed output plane (half the source + // height for packed alpha), so browser object-fit/object-position applies + // to the visible avatar rather than the vertically-stacked transport. // Replaced content is clipped to its own content box, which prevents // `cover` pixels escaping when the parent has visible overflow. Delegating // positioning to CSS also preserves the full grammar (lengths, @@ -347,30 +414,39 @@ export class TransparentBackgroundRenderer { const startedAt = performance.now(); const gl = this.gl; try { + const geometry = resolveTransparentFrameGeometry( + this.video.videoWidth, + this.video.videoHeight, + this.transport, + this.maxTextureSize, + ); + this.applyFrameGeometry(geometry); + if (geometry.mode === 'unsupported') return; + if ( - this.canvas.width !== this.video.videoWidth || - this.canvas.height !== this.video.videoHeight + this.canvas.width !== geometry.canvasWidth || + this.canvas.height !== geometry.canvasHeight ) { // Deliberately use source pixels, not devicePixelRatio. A DPR-scaled // backing canvas would add work without adding information. - this.canvas.width = this.video.videoWidth; - this.canvas.height = this.video.videoHeight; + this.canvas.width = geometry.canvasWidth; + this.canvas.height = geometry.canvasHeight; } gl.viewport(0, 0, this.canvas.width, this.canvas.height); gl.clearColor(0, 0, 0, 0); gl.clear(gl.COLOR_BUFFER_BIT); - gl.useProgram(this.resources.program); + const packedAlpha = geometry.mode === PACKED_ALPHA_TRANSPORT; + const program = packedAlpha + ? this.resources.packedAlphaProgram + : this.resources.legacyProgram; + const positionLocation = packedAlpha + ? this.resources.packedAlphaPositionLocation + : this.resources.legacyPositionLocation; + gl.useProgram(program); gl.bindBuffer(gl.ARRAY_BUFFER, this.resources.positionBuffer); - gl.enableVertexAttribArray(this.resources.positionLocation); - gl.vertexAttribPointer( - this.resources.positionLocation, - 2, - gl.FLOAT, - false, - 0, - 0, - ); + gl.enableVertexAttribArray(positionLocation); + gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, this.resources.texture); gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1); @@ -382,15 +458,17 @@ export class TransparentBackgroundRenderer { gl.UNSIGNED_BYTE, this.video, ); - gl.uniform1f( - this.resources.similarityLocation, - this.keyOptions.similarity, - ); - gl.uniform1f( - this.resources.smoothnessLocation, - this.keyOptions.smoothness, - ); - gl.uniform1f(this.resources.spillLocation, this.keyOptions.spill); + if (!packedAlpha) { + gl.uniform1f( + this.resources.similarityLocation, + this.keyOptions.similarity, + ); + gl.uniform1f( + this.resources.smoothnessLocation, + this.keyOptions.smoothness, + ); + gl.uniform1f(this.resources.spillLocation, this.keyOptions.spill); + } gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); } catch (error) { if (!this.renderErrorReported) { @@ -424,70 +502,142 @@ export class TransparentBackgroundRenderer { } } - private createGlResources(): GlResources { - const gl = this.gl; - const vertexShader = compileShader( - gl, - gl.VERTEX_SHADER, - VERTEX_SHADER_SOURCE, - ); - const fragmentShader = compileShader( - gl, - gl.FRAGMENT_SHADER, - FRAGMENT_SHADER_SOURCE, - ); - const program = gl.createProgram(); - if (!program) throw new Error('Unable to create WebGL program.'); - gl.attachShader(program, vertexShader); - gl.attachShader(program, fragmentShader); - gl.linkProgram(program); - gl.deleteShader(vertexShader); - gl.deleteShader(fragmentShader); - if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { - const message = gl.getProgramInfoLog(program) ?? 'unknown link error'; - gl.deleteProgram(program); - throw new Error(`Unable to link transparent renderer: ${message}`); + private applyFrameGeometry(geometry: TransparentFrameGeometry): void { + const signature = `${geometry.mode}:${this.video.videoWidth}x${this.video.videoHeight}`; + if (signature === this.lastGeometrySignature) return; + this.lastGeometrySignature = signature; + + if (geometry.mode === 'unsupported') { + // Never hide the only usable pixels when a server or intermediary + // delivers an unexpected geometry. If a later frame returns to a valid + // geometry, the renderer automatically takes over again. + this.canvas.style.opacity = '0'; + this.video.style.opacity = this.originalVideoOpacity; + console.warn( + `Transparent background received unsupported video geometry ${this.video.videoWidth}x${this.video.videoHeight}; showing the source video unchanged.`, + ); + this.report('unsupported_geometry', 1, { + width: this.video.videoWidth, + height: this.video.videoHeight, + reason: geometry.reason ?? 'unknown', + }); + return; } - const positionBuffer = gl.createBuffer(); - const texture = gl.createTexture(); - if (!positionBuffer || !texture) { - throw new Error('Unable to allocate transparent renderer resources.'); - } - gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); - gl.bufferData( - gl.ARRAY_BUFFER, - new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), - gl.STATIC_DRAW, - ); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - - const positionLocation = gl.getAttribLocation(program, 'a_position'); - const similarityLocation = gl.getUniformLocation(program, 'u_similarity'); - const smoothnessLocation = gl.getUniformLocation(program, 'u_smoothness'); - const spillLocation = gl.getUniformLocation(program, 'u_spill'); + this.canvas.style.opacity = ''; + this.video.style.opacity = '0'; if ( - positionLocation < 0 || - !similarityLocation || - !smoothnessLocation || - !spillLocation + this.transport === PACKED_ALPHA_TRANSPORT && + geometry.mode === 'green-key-v1' ) { - throw new Error('Unable to resolve transparent renderer shader inputs.'); + console.warn( + 'Packed transparent-background transport returned a legacy green-screen frame; using the compatibility keyer.', + ); + this.report('transport_fallback', 1, { + deliveredTransport: 'green-key-v1', + width: this.video.videoWidth, + height: this.video.videoHeight, + }); + } else { + this.report('transport_selected', 1, { + deliveredTransport: geometry.mode, + width: this.video.videoWidth, + height: this.video.videoHeight, + }); } + } - return { - program, - positionBuffer, - texture, - positionLocation, - similarityLocation, - smoothnessLocation, - spillLocation, - }; + private createGlResources(): GlResources { + const gl = this.gl; + let legacyProgram: WebGLProgram | null = null; + let packedAlphaProgram: WebGLProgram | null = null; + let positionBuffer: WebGLBuffer | null = null; + let texture: WebGLTexture | null = null; + + try { + legacyProgram = createProgram( + gl, + VERTEX_SHADER_SOURCE, + FRAGMENT_SHADER_SOURCE, + ); + packedAlphaProgram = createProgram( + gl, + VERTEX_SHADER_SOURCE, + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + ); + + positionBuffer = gl.createBuffer(); + texture = gl.createTexture(); + if (!positionBuffer || !texture) { + throw new Error('Unable to allocate transparent renderer resources.'); + } + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), + gl.STATIC_DRAW, + ); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + + const legacyPositionLocation = gl.getAttribLocation( + legacyProgram, + 'a_position', + ); + const packedAlphaPositionLocation = gl.getAttribLocation( + packedAlphaProgram, + 'a_position', + ); + const similarityLocation = gl.getUniformLocation( + legacyProgram, + 'u_similarity', + ); + const smoothnessLocation = gl.getUniformLocation( + legacyProgram, + 'u_smoothness', + ); + const spillLocation = gl.getUniformLocation(legacyProgram, 'u_spill'); + if ( + legacyPositionLocation < 0 || + packedAlphaPositionLocation < 0 || + !similarityLocation || + !smoothnessLocation || + !spillLocation + ) { + throw new Error( + 'Unable to resolve transparent renderer shader inputs.', + ); + } + + return { + positionBuffer, + texture, + legacyProgram, + legacyPositionLocation, + similarityLocation, + smoothnessLocation, + spillLocation, + packedAlphaProgram, + packedAlphaPositionLocation, + }; + } catch (error) { + if (positionBuffer) gl.deleteBuffer(positionBuffer); + if (texture) gl.deleteTexture(texture); + if (legacyProgram) gl.deleteProgram(legacyProgram); + if (packedAlphaProgram) gl.deleteProgram(packedAlphaProgram); + throw error; + } + } + + private deleteGlResources(resources: GlResources): void { + const gl = this.gl; + gl.deleteBuffer(resources.positionBuffer); + gl.deleteTexture(resources.texture); + gl.deleteProgram(resources.legacyProgram); + gl.deleteProgram(resources.packedAlphaProgram); } private onContextLost(event: Event): void { @@ -524,6 +674,32 @@ export class TransparentBackgroundRenderer { } } +function createProgram( + gl: WebGLRenderingContext, + vertexSource: string, + fragmentSource: string, +): WebGLProgram { + const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexSource); + let fragmentShader: WebGLShader | null = null; + try { + fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource); + const program = gl.createProgram(); + if (!program) throw new Error('Unable to create WebGL program.'); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + const message = gl.getProgramInfoLog(program) ?? 'unknown link error'; + gl.deleteProgram(program); + throw new Error(`Unable to link transparent renderer: ${message}`); + } + return program; + } finally { + gl.deleteShader(vertexShader); + if (fragmentShader) gl.deleteShader(fragmentShader); + } +} + function compileShader( gl: WebGLRenderingContext, type: number, @@ -541,6 +717,73 @@ function compileShader( return shader; } +/** + * Resolve the decoded frame layout before uploading it to WebGL. Packed + * frames are 3:4 because two 3:2 planes are stacked vertically; legacy Cara 4 + * frames are 3:2. Ratio-based matching also permits a decoder to deliver a + * proportionally downscaled frame without silently sampling the wrong plane. + * + * @internal + */ +export function resolveTransparentFrameGeometry( + width: number, + height: number, + transport: TransparentBackgroundTransport | undefined, + maxTextureSize = 2048, +): TransparentFrameGeometry { + if ( + !Number.isInteger(width) || + !Number.isInteger(height) || + width <= 0 || + height <= 0 + ) { + return { + mode: 'unsupported', + canvasWidth: 0, + canvasHeight: 0, + reason: 'invalid_dimensions', + }; + } + if (width > maxTextureSize || height > maxTextureSize) { + return { + mode: 'unsupported', + canvasWidth: width, + canvasHeight: height, + reason: 'exceeds_webgl_texture_limit', + }; + } + + if (transport !== PACKED_ALPHA_TRANSPORT) { + return { + mode: 'green-key-v1', + canvasWidth: width, + canvasHeight: height, + }; + } + + if (height % 2 === 0 && height * 3 === width * 4) { + return { + mode: PACKED_ALPHA_TRANSPORT, + canvasWidth: width, + canvasHeight: height / 2, + }; + } + if (width * 2 === height * 3) { + return { + mode: 'green-key-v1', + canvasWidth: width, + canvasHeight: height, + }; + } + + return { + mode: 'unsupported', + canvasWidth: width, + canvasHeight: height, + reason: 'unexpected_packed_alpha_aspect_ratio', + }; +} + /** @internal */ export function resolveKeyOptions( options?: TransparentBackgroundOptions, @@ -563,7 +806,8 @@ export function reconstructPremultipliedForeground( alphaValue: number, spillValue = DEFAULT_SPILL, ): [number, number, number] { - const alpha = clampUnit(alphaValue); + const unclippedAlpha = clampUnit(alphaValue); + const alpha = unclippedAlpha < BACKGROUND_ALPHA_FLOOR ? 0 : unclippedAlpha; const spill = clampUnit(spillValue); const premultiplied: [number, number, number] = [ Math.min(alpha, Math.max(0, rgb[0])), diff --git a/src/types/AnamPublicClientOptions.ts b/src/types/AnamPublicClientOptions.ts index e1e4d8d..3e75e00 100644 --- a/src/types/AnamPublicClientOptions.ts +++ b/src/types/AnamPublicClientOptions.ts @@ -27,16 +27,16 @@ export interface AnamPublicClientOptions { */ rtcConfiguration?: RTCConfiguration; /** - * Request the avatar's generated green-screen rendition and render it as a - * transparent WebGL canvas over the video element supplied to - * `streamToVideoElement`. + * Request the avatar's transparent rendition and render it as a transparent + * WebGL canvas over the video element supplied to `streamToVideoElement`. * * The underlying MediaStream remains an ordinary opaque WebRTC video. Calls - * to `stream()` therefore return the green-screen source; transparent pixels - * exist only in the SDK-managed canvas renderer. + * to `stream()` therefore return an opaque packed colour/matte video (or the + * legacy green-screen fallback); transparent pixels exist only in the + * SDK-managed canvas renderer. * @default false */ transparentBackground?: boolean; - /** Optional client-side key tuning for `transparentBackground`. */ + /** Optional tuning for the legacy green-screen compatibility keyer. */ transparentBackgroundOptions?: TransparentBackgroundOptions; } diff --git a/src/types/TransparentBackgroundTransport.ts b/src/types/TransparentBackgroundTransport.ts new file mode 100644 index 0000000..cae999d --- /dev/null +++ b/src/types/TransparentBackgroundTransport.ts @@ -0,0 +1,3 @@ +export const PACKED_ALPHA_TRANSPORT = 'packed-alpha-v1' as const; + +export type TransparentBackgroundTransport = typeof PACKED_ALPHA_TRANSPORT; diff --git a/src/types/coreApi/StartSessionOptions.ts b/src/types/coreApi/StartSessionOptions.ts index 7eda7d8..d4efdcb 100644 --- a/src/types/coreApi/StartSessionOptions.ts +++ b/src/types/coreApi/StartSessionOptions.ts @@ -1,4 +1,5 @@ import { VoiceDetectionOptions } from '../VoiceDetectionOptions'; +import { TransparentBackgroundTransport } from '../TransparentBackgroundTransport'; export interface StartSessionOptions { voiceDetection?: VoiceDetectionOptions; @@ -8,4 +9,9 @@ export interface StartSessionOptions { * canvas when streaming to a video element. */ transparentBackground?: boolean; + /** + * Selects the internal wire representation used for transparent video. + * @internal + */ + transparentBackgroundTransport?: TransparentBackgroundTransport; } diff --git a/src/types/streaming/StreamingClientOptions.ts b/src/types/streaming/StreamingClientOptions.ts index 4e9023c..94a369d 100644 --- a/src/types/streaming/StreamingClientOptions.ts +++ b/src/types/streaming/StreamingClientOptions.ts @@ -3,6 +3,7 @@ import { EngineApiRestClientOptions } from '../engineApi/EngineApiRestClientOpti import { InputAudioOptions } from './InputAudioOptions'; import { ApiGatewayConfig } from '../ApiGatewayConfig'; import { TransparentBackgroundOptions } from '../TransparentBackgroundOptions'; +import { TransparentBackgroundTransport } from '../TransparentBackgroundTransport'; export interface StreamingClientOptions { engine: EngineApiRestClientOptions; @@ -15,6 +16,7 @@ export interface StreamingClientOptions { transparentBackground?: { enabled: boolean; keyOptions?: TransparentBackgroundOptions; + transport?: TransparentBackgroundTransport; }; metrics?: { showPeerConnectionStatsReport?: boolean; diff --git a/test/packedAlphaTransportHarness.js b/test/packedAlphaTransportHarness.js new file mode 100644 index 0000000..fa9f9cb --- /dev/null +++ b/test/packedAlphaTransportHarness.js @@ -0,0 +1,160 @@ +const assert = require('assert'); +const { + buildTransparentBackgroundSessionOptions, + detectPackedAlphaCapability, + prepareOfferForTransparentBackgroundTransport, + promotePackedAlphaH264Level, +} = require('../dist/main/modules/PackedAlphaTransport'); +const { + PACKED_ALPHA_TRANSPORT, +} = require('../dist/main/types/TransparentBackgroundTransport'); + +void (async () => { + let decodingConfiguration; + const supportedMediaCapabilities = { + decodingInfo: async (configuration) => { + decodingConfiguration = configuration; + return { supported: true, smooth: true, powerEfficient: true }; + }, + }; + + assert.equal( + await detectPackedAlphaCapability(supportedMediaCapabilities), + 'supported', + ); + assert.deepEqual(decodingConfiguration, { + type: 'webrtc', + video: { + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + width: 1152, + height: 1536, + bitrate: 2500000, + framerate: 25, + }, + }); + assert.deepEqual( + await buildTransparentBackgroundSessionOptions( + true, + supportedMediaCapabilities, + ), + { + transparentBackground: true, + transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, + }, + 'supported devices must request packed-alpha-v1', + ); + + const originalWarn = console.warn; + let warning = ''; + console.warn = (message) => { + warning = message; + }; + try { + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(true, { + decodingInfo: async () => ({ + supported: false, + smooth: false, + powerEfficient: false, + }), + }), + { transparentBackground: true }, + 'an explicit unsupported result must retain the legacy green path', + ); + } finally { + console.warn = originalWarn; + } + assert.match(warning, /falling back to legacy green-screen keying/); + + assert.equal( + await detectPackedAlphaCapability({ + decodingInfo: async () => ({ + supported: true, + smooth: false, + powerEfficient: false, + }), + }), + 'unsupported', + 'a supported decoder that cannot sustain the packed resolution must use the compatibility path', + ); + + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(true, { + decodingInfo: async () => { + throw new TypeError('WebRTC decodingInfo is not implemented'); + }, + }), + { + transparentBackground: true, + transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, + }, + 'an inconclusive MediaCapabilities implementation must defer to SDP negotiation', + ); + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(false, { + decodingInfo: async () => { + throw new Error('must not be called'); + }, + }), + { transparentBackground: false }, + ); + + const ordinarySdp = [ + 'v=0', + 'm=video 9 UDP/TLS/RTP/SAVPF 96 97 98', + 'a=rtpmap:96 H264/90000', + 'a=fmtp:96 profile-level-id=4d001f;packetization-mode=1', + 'a=rtpmap:97 H264/90000', + 'a=fmtp:97 profile-level-id=42e01f;packetization-mode=1', + 'a=rtpmap:98 H264/90000', + 'a=fmtp:98 profile-level-id=4d002a;packetization-mode=1', + 'a=rtpmap:99 AV1/90000', + 'a=fmtp:99 profile-level-id=4d001f', + 'a=x-profile-level-id:4d001f', + '', + ].join('\r\n'); + const promotedSdp = promotePackedAlphaH264Level(ordinarySdp); + assert.match(promotedSdp, /profile-level-id=4d0028/); + assert.doesNotMatch( + promotedSdp, + /a=fmtp:96 profile-level-id=4d001f/, + 'the target H264 payload must no longer advertise Main Level 3.1', + ); + assert.match(promotedSdp, /profile-level-id=42e01f/); + assert.match(promotedSdp, /profile-level-id=4d002a/); + assert.match( + promotedSdp, + /a=fmtp:99 profile-level-id=4d001f/, + 'an fmtp parameter belonging to a non-H264 payload must not be rewritten', + ); + assert.match( + promotedSdp, + /a=x-profile-level-id:4d001f/, + 'an unrelated SDP attribute must not be rewritten', + ); + + const ordinaryOffer = { type: 'offer', sdp: ordinarySdp }; + assert.strictEqual( + prepareOfferForTransparentBackgroundTransport(ordinaryOffer, undefined), + ordinaryOffer, + 'ordinary sessions must not rewrite or clone their offer', + ); + const packedOffer = prepareOfferForTransparentBackgroundTransport( + ordinaryOffer, + PACKED_ALPHA_TRANSPORT, + ); + assert.notStrictEqual(packedOffer, ordinaryOffer); + assert.equal(packedOffer.type, 'offer'); + assert.equal(packedOffer.sdp, promotedSdp); + assert.equal( + ordinaryOffer.sdp, + ordinarySdp, + 'SDP rewriting must not mutate the browser-created offer', + ); + + console.log('packed alpha transport harness passed'); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/test/transparentBackgroundGeometryHarness.js b/test/transparentBackgroundGeometryHarness.js index 2275f4e..d974c2f 100644 --- a/test/transparentBackgroundGeometryHarness.js +++ b/test/transparentBackgroundGeometryHarness.js @@ -1,12 +1,16 @@ const assert = require('assert'); const { setClientMetricsDisabled } = require('../dist/main/lib/ClientMetrics'); const { + resolveTransparentFrameGeometry, TransparentBackgroundRenderer, } = require('../dist/main/modules/TransparentBackgroundRenderer'); +const { + PACKED_ALPHA_TRANSPORT, +} = require('../dist/main/types/TransparentBackgroundTransport'); setClientMetricsDisabled(true); -const createWebGlStub = () => ({ +const createWebGlStub = (deletedResources) => ({ VERTEX_SHADER: 1, FRAGMENT_SHADER: 2, COMPILE_STATUS: 3, @@ -20,6 +24,7 @@ const createWebGlStub = () => ({ TEXTURE_WRAP_S: 11, TEXTURE_WRAP_T: 12, CLAMP_TO_EDGE: 13, + MAX_TEXTURE_SIZE: 14, createShader: () => ({}), shaderSource: () => {}, compileShader: () => {}, @@ -31,18 +36,22 @@ const createWebGlStub = () => ({ linkProgram: () => {}, getProgramParameter: () => true, getProgramInfoLog: () => '', - deleteProgram: () => {}, + deleteProgram: (program) => deletedResources.programs.push(program), createBuffer: () => ({}), createTexture: () => ({}), + deleteBuffer: (buffer) => deletedResources.buffers.push(buffer), + deleteTexture: (texture) => deletedResources.textures.push(texture), bindBuffer: () => {}, bufferData: () => {}, bindTexture: () => {}, texParameteri: () => {}, getAttribLocation: () => 0, getUniformLocation: () => ({}), + getParameter: () => 2048, }); const createRendererHarness = ({ objectFit, objectPosition }) => { + const deletedResources = { programs: [], buffers: [], textures: [] }; const parent = { style: { position: '' }, }; @@ -51,7 +60,7 @@ const createRendererHarness = ({ objectFit, objectPosition }) => { dataset: {}, id: '', setAttribute: () => {}, - getContext: () => createWebGlStub(), + getContext: () => createWebGlStub(deletedResources), addEventListener: () => {}, removeEventListener: () => {}, remove: () => {}, @@ -100,7 +109,7 @@ const createRendererHarness = ({ objectFit, objectPosition }) => { }; const renderer = new TransparentBackgroundRenderer(video); - return { canvas, renderer }; + return { canvas, deletedResources, renderer }; }; const cover = createRendererHarness({ @@ -127,6 +136,55 @@ assert.deepEqual( 'cover must stay inside the video element box even when the parent overflow is visible', ); cover.renderer.destroy(); +cover.renderer.destroy(); +assert.deepEqual( + { + programs: cover.deletedResources.programs.length, + buffers: cover.deletedResources.buffers.length, + textures: cover.deletedResources.textures.length, + }, + { programs: 2, buffers: 1, textures: 1 }, + 'destroy must release each shader program, buffer, and texture exactly once', +); + +assert.deepEqual( + resolveTransparentFrameGeometry(1152, 1536, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 1152, + canvasHeight: 768, + }, + 'packed Cara 4 frames must expose a 1152x768 canvas', +); +assert.deepEqual( + resolveTransparentFrameGeometry(576, 768, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 576, + canvasHeight: 384, + }, + 'proportionally downscaled packed frames must preserve the two-plane layout', +); +assert.deepEqual( + resolveTransparentFrameGeometry(1152, 768, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'green-key-v1', + canvasWidth: 1152, + canvasHeight: 768, + }, + 'a standard Cara 4 frame must select the legacy keyer compatibility path', +); +assert.equal( + resolveTransparentFrameGeometry(1280, 720, PACKED_ALPHA_TRANSPORT, 2048).mode, + 'unsupported', + 'unexpected packed transport geometry must not be sampled as two planes', +); +assert.equal( + resolveTransparentFrameGeometry(1152, 1536, PACKED_ALPHA_TRANSPORT, 1024) + .reason, + 'exceeds_webgl_texture_limit', + 'the runtime WebGL texture limit must be enforced', +); for (const objectPosition of [ '10px 20px', diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index acc7c26..c12c84c 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -1,6 +1,7 @@ const assert = require('assert'); const { FRAGMENT_SHADER_SOURCE, + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, reconstructPremultipliedForeground, resolveKeyOptions, } = require('../dist/main/modules/TransparentBackgroundRenderer'); @@ -32,6 +33,12 @@ closeTo( 'fully transparent exact green must reconstruct to transparent black', ); +closeTo( + reconstructPremultipliedForeground([0.01, 0.99, 0.01], 0.019), + [0, 0, 0], + 'sub-threshold codec noise must become fully transparent', +); + const blonde = [0.8, 0.7, 0.55]; const blondeAlpha = 0.5; const blondeCarrier = [ @@ -64,10 +71,7 @@ const nearOpaqueGreenCarrier = [ guardEndAlpha * nearOpaqueGreen[2], ]; closeTo( - reconstructPremultipliedForeground( - nearOpaqueGreenCarrier, - guardEndAlpha, - ), + reconstructPremultipliedForeground(nearOpaqueGreenCarrier, guardEndAlpha), nearOpaqueGreen.map((channel) => channel * guardEndAlpha), 'green-spill clamp must be fully faded out by alpha 0.995', ); @@ -115,10 +119,41 @@ assert.match( /gl_FragColor = vec4\(premultiplied, alpha\)/, 'shader must submit already-premultiplied RGB', ); +assert.match( + FRAGMENT_SHADER_SOURCE, + /alpha \*= step\(0\.020, alpha\)/, + 'shader must clear the measured codec-noise alpha tail', +); assert.doesNotMatch( FRAGMENT_SHADER_SOURCE, /gl_FragColor = vec4\(rgb \* alpha, alpha\)/, 'shader must not multiply the green carrier by alpha', ); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /0\.5 \+ v_texCoord\.y \* 0\.5/, + 'packed renderer must sample colour from the top half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /v_texCoord\.y \* 0\.5/, + 'packed renderer must sample alpha from the bottom half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /gl_FragColor = vec4\(min\(premultiplied, vec3\(alpha\)\), alpha\)/, + 'packed renderer must submit the transported premultiplied colour and alpha', +); +assert.doesNotMatch( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /u_similarity|u_smoothness|u_spill|chroma\(/, + 'packed renderer must not run the legacy key or despill operations', +); +assert.doesNotMatch( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /alpha\s*\*=\s*step|smoothstep\(/, + 'packed renderer must not threshold or reshape transported alpha', +); + console.log('transparent background reconstruction harness passed'); From 7bc714cd4aca3b6045584ac91db7c6b9609df0ab Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 21 Jul 2026 23:16:02 +0100 Subject: [PATCH 06/15] Fall back when packed decoding support is unknown --- README.md | 2 +- src/modules/PackedAlphaTransport.ts | 14 +++++++------- test/packedAlphaTransportHarness.js | 12 +++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 324dccd..8e4e4b8 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ This will start a new session using the pre-configured persona id and start stre Set `transparentBackground` when creating the client. The SDK requests a packed colour/matte rendition and reconstructs it through a source-resolution -WebGL canvas over your video element. Browsers that explicitly report no +WebGL canvas over your video element. Browsers that cannot confirm smooth support for the packed H.264 stream, and servers that return the original 1152x768 frame, automatically use the legacy green-screen keyer. diff --git a/src/modules/PackedAlphaTransport.ts b/src/modules/PackedAlphaTransport.ts index e076f0d..26b2e14 100644 --- a/src/modules/PackedAlphaTransport.ts +++ b/src/modules/PackedAlphaTransport.ts @@ -26,8 +26,9 @@ export type PackedAlphaCapability = 'supported' | 'unsupported' | 'unknown'; /** * Query support for the H.264 Main Level 4.0 stream used by packed-alpha-v1. * Missing or inconclusive MediaCapabilities implementations are reported as - * unknown rather than unsupported so older browsers retain their existing - * WebRTC codec negotiation path. + * unknown. Callers conservatively retain the legacy carrier in that case: the + * server requires an explicit Main-Level-4 offer, so guessing support could + * otherwise turn a graceful quality fallback into a failed session. * * @internal */ @@ -48,9 +49,8 @@ export async function detectPackedAlphaCapability( // configuration should use the lower-resolution compatibility path. return result.supported && result.smooth ? 'supported' : 'unsupported'; } catch { - // Some otherwise-compatible browsers expose MediaCapabilities but reject - // WebRTC configurations. That is not evidence that H.264 Level 4.0 cannot - // be decoded, so let normal SDP negotiation decide. + // Some browsers expose MediaCapabilities but reject WebRTC configurations. + // Treat this as inconclusive; the caller keeps the compatibility path. return 'unknown'; } } @@ -69,9 +69,9 @@ export async function buildTransparentBackgroundSessionOptions( if (!enabled) return { transparentBackground: false }; const capability = await detectPackedAlphaCapability(mediaCapabilities); - if (capability === 'unsupported') { + if (capability !== 'supported') { console.warn( - 'Packed transparent-background video is unsupported or is not expected to decode smoothly on this device; falling back to legacy green-screen keying.', + 'Packed transparent-background video support could not be confirmed as smooth on this device; falling back to legacy green-screen keying.', ); return { transparentBackground: true }; } diff --git a/test/packedAlphaTransportHarness.js b/test/packedAlphaTransportHarness.js index fa9f9cb..f47c54a 100644 --- a/test/packedAlphaTransportHarness.js +++ b/test/packedAlphaTransportHarness.js @@ -85,11 +85,13 @@ void (async () => { throw new TypeError('WebRTC decodingInfo is not implemented'); }, }), - { - transparentBackground: true, - transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, - }, - 'an inconclusive MediaCapabilities implementation must defer to SDP negotiation', + { transparentBackground: true }, + 'an inconclusive MediaCapabilities implementation must retain the compatibility path', + ); + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(true, undefined), + { transparentBackground: true }, + 'a browser without MediaCapabilities must retain the compatibility path', ); assert.deepEqual( await buildTransparentBackgroundSessionOptions(false, { From e159476fa5c12ca77e98554bafcba5d14daf7414 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Wed, 22 Jul 2026 09:12:47 +0100 Subject: [PATCH 07/15] Add straight-alpha transparent transport --- src/modules/PackedAlphaTransport.ts | 13 ++- src/modules/TransparentBackgroundRenderer.ts | 79 ++++++++++++++++--- src/types/TransparentBackgroundTransport.ts | 5 +- test/packedAlphaTransportHarness.js | 14 +++- test/transparentBackgroundGeometryHarness.js | 17 +++- ...nsparentBackgroundReconstructionHarness.js | 17 ++++ 6 files changed, 128 insertions(+), 17 deletions(-) diff --git a/src/modules/PackedAlphaTransport.ts b/src/modules/PackedAlphaTransport.ts index 26b2e14..6b01b72 100644 --- a/src/modules/PackedAlphaTransport.ts +++ b/src/modules/PackedAlphaTransport.ts @@ -1,6 +1,7 @@ import { StartSessionOptions } from '../types/coreApi/StartSessionOptions'; import { PACKED_ALPHA_TRANSPORT, + PACKED_STRAIGHT_ALPHA_TRANSPORT, TransparentBackgroundTransport, } from '../types/TransparentBackgroundTransport'; @@ -24,7 +25,7 @@ type MediaCapabilitiesLike = Pick; export type PackedAlphaCapability = 'supported' | 'unsupported' | 'unknown'; /** - * Query support for the H.264 Main Level 4.0 stream used by packed-alpha-v1. + * Query support for the H.264 Main Level 4.0 stream used by packed-alpha-v1/v2. * Missing or inconclusive MediaCapabilities implementations are reported as * unknown. Callers conservatively retain the legacy carrier in that case: the * server requires an explicit Main-Level-4 offer, so guessing support could @@ -78,7 +79,7 @@ export async function buildTransparentBackgroundSessionOptions( return { transparentBackground: true, - transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, + transparentBackgroundTransport: PACKED_STRAIGHT_ALPHA_TRANSPORT, }; } @@ -115,6 +116,12 @@ export function prepareOfferForTransparentBackgroundTransport( offer: RTCSessionDescriptionInit, transport: TransparentBackgroundTransport | undefined, ): RTCSessionDescriptionInit { - if (transport !== PACKED_ALPHA_TRANSPORT || !offer.sdp) return offer; + if ( + transport !== PACKED_ALPHA_TRANSPORT && + transport !== PACKED_STRAIGHT_ALPHA_TRANSPORT + ) { + return offer; + } + if (!offer.sdp) return offer; return { ...offer, sdp: promotePackedAlphaH264Level(offer.sdp) }; } diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index 1f6e862..955c11d 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -5,6 +5,7 @@ import { import { TransparentBackgroundOptions } from '../types/TransparentBackgroundOptions'; import { PACKED_ALPHA_TRANSPORT, + PACKED_STRAIGHT_ALPHA_TRANSPORT, TransparentBackgroundTransport, } from '../types/TransparentBackgroundTransport'; @@ -119,6 +120,34 @@ void main() { } `; +// packed-alpha-v2 carries padded straight (unassociated) colour. Multiplying +// only after the separately decoded matte is sampled prevents 4:2:0 from +// blending foreground chroma with transparent black at moving hair edges. +/** @internal */ +export const PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE = ` +precision mediump float; + +uniform sampler2D u_frame; +varying vec2 v_texCoord; + +void main() { + vec2 colourCoord = vec2( + v_texCoord.x, + 0.5 + v_texCoord.y * 0.5 + ); + vec2 alphaCoord = vec2( + v_texCoord.x, + v_texCoord.y * 0.5 + ); + vec3 straight = texture2D(u_frame, colourCoord).rgb; + vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; + float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); + vec3 premultiplied = straight * alpha; + + gl_FragColor = vec4(min(premultiplied, vec3(alpha)), alpha); +} +`; + type OptionalVideoFrameCallbacks = { requestVideoFrameCallback?: ( callback: (now: DOMHighResTimeStamp) => void, @@ -142,11 +171,14 @@ interface GlResources { spillLocation: WebGLUniformLocation; packedAlphaProgram: WebGLProgram; packedAlphaPositionLocation: number; + packedStraightAlphaProgram: WebGLProgram; + packedStraightAlphaPositionLocation: number; } export type TransparentFrameMode = | 'green-key-v1' | 'packed-alpha-v1' + | 'packed-alpha-v2' | 'unsupported'; export interface TransparentFrameGeometry { @@ -437,12 +469,19 @@ export class TransparentBackgroundRenderer { gl.clearColor(0, 0, 0, 0); gl.clear(gl.COLOR_BUFFER_BIT); const packedAlpha = geometry.mode === PACKED_ALPHA_TRANSPORT; - const program = packedAlpha - ? this.resources.packedAlphaProgram - : this.resources.legacyProgram; - const positionLocation = packedAlpha - ? this.resources.packedAlphaPositionLocation - : this.resources.legacyPositionLocation; + const packedStraightAlpha = + geometry.mode === PACKED_STRAIGHT_ALPHA_TRANSPORT; + const packed = packedAlpha || packedStraightAlpha; + const program = packedStraightAlpha + ? this.resources.packedStraightAlphaProgram + : packedAlpha + ? this.resources.packedAlphaProgram + : this.resources.legacyProgram; + const positionLocation = packedStraightAlpha + ? this.resources.packedStraightAlphaPositionLocation + : packedAlpha + ? this.resources.packedAlphaPositionLocation + : this.resources.legacyPositionLocation; gl.useProgram(program); gl.bindBuffer(gl.ARRAY_BUFFER, this.resources.positionBuffer); gl.enableVertexAttribArray(positionLocation); @@ -458,7 +497,7 @@ export class TransparentBackgroundRenderer { gl.UNSIGNED_BYTE, this.video, ); - if (!packedAlpha) { + if (!packed) { gl.uniform1f( this.resources.similarityLocation, this.keyOptions.similarity, @@ -527,7 +566,8 @@ export class TransparentBackgroundRenderer { this.canvas.style.opacity = ''; this.video.style.opacity = '0'; if ( - this.transport === PACKED_ALPHA_TRANSPORT && + (this.transport === PACKED_ALPHA_TRANSPORT || + this.transport === PACKED_STRAIGHT_ALPHA_TRANSPORT) && geometry.mode === 'green-key-v1' ) { console.warn( @@ -551,6 +591,7 @@ export class TransparentBackgroundRenderer { const gl = this.gl; let legacyProgram: WebGLProgram | null = null; let packedAlphaProgram: WebGLProgram | null = null; + let packedStraightAlphaProgram: WebGLProgram | null = null; let positionBuffer: WebGLBuffer | null = null; let texture: WebGLTexture | null = null; @@ -565,6 +606,11 @@ export class TransparentBackgroundRenderer { VERTEX_SHADER_SOURCE, PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, ); + packedStraightAlphaProgram = createProgram( + gl, + VERTEX_SHADER_SOURCE, + PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, + ); positionBuffer = gl.createBuffer(); texture = gl.createTexture(); @@ -591,6 +637,10 @@ export class TransparentBackgroundRenderer { packedAlphaProgram, 'a_position', ); + const packedStraightAlphaPositionLocation = gl.getAttribLocation( + packedStraightAlphaProgram, + 'a_position', + ); const similarityLocation = gl.getUniformLocation( legacyProgram, 'u_similarity', @@ -603,6 +653,7 @@ export class TransparentBackgroundRenderer { if ( legacyPositionLocation < 0 || packedAlphaPositionLocation < 0 || + packedStraightAlphaPositionLocation < 0 || !similarityLocation || !smoothnessLocation || !spillLocation @@ -622,12 +673,16 @@ export class TransparentBackgroundRenderer { spillLocation, packedAlphaProgram, packedAlphaPositionLocation, + packedStraightAlphaProgram, + packedStraightAlphaPositionLocation, }; } catch (error) { if (positionBuffer) gl.deleteBuffer(positionBuffer); if (texture) gl.deleteTexture(texture); if (legacyProgram) gl.deleteProgram(legacyProgram); if (packedAlphaProgram) gl.deleteProgram(packedAlphaProgram); + if (packedStraightAlphaProgram) + gl.deleteProgram(packedStraightAlphaProgram); throw error; } } @@ -638,6 +693,7 @@ export class TransparentBackgroundRenderer { gl.deleteTexture(resources.texture); gl.deleteProgram(resources.legacyProgram); gl.deleteProgram(resources.packedAlphaProgram); + gl.deleteProgram(resources.packedStraightAlphaProgram); } private onContextLost(event: Event): void { @@ -753,7 +809,10 @@ export function resolveTransparentFrameGeometry( }; } - if (transport !== PACKED_ALPHA_TRANSPORT) { + if ( + transport !== PACKED_ALPHA_TRANSPORT && + transport !== PACKED_STRAIGHT_ALPHA_TRANSPORT + ) { return { mode: 'green-key-v1', canvasWidth: width, @@ -763,7 +822,7 @@ export function resolveTransparentFrameGeometry( if (height % 2 === 0 && height * 3 === width * 4) { return { - mode: PACKED_ALPHA_TRANSPORT, + mode: transport, canvasWidth: width, canvasHeight: height / 2, }; diff --git a/src/types/TransparentBackgroundTransport.ts b/src/types/TransparentBackgroundTransport.ts index cae999d..db7fbdb 100644 --- a/src/types/TransparentBackgroundTransport.ts +++ b/src/types/TransparentBackgroundTransport.ts @@ -1,3 +1,6 @@ export const PACKED_ALPHA_TRANSPORT = 'packed-alpha-v1' as const; +export const PACKED_STRAIGHT_ALPHA_TRANSPORT = 'packed-alpha-v2' as const; -export type TransparentBackgroundTransport = typeof PACKED_ALPHA_TRANSPORT; +export type TransparentBackgroundTransport = + | typeof PACKED_ALPHA_TRANSPORT + | typeof PACKED_STRAIGHT_ALPHA_TRANSPORT; diff --git a/test/packedAlphaTransportHarness.js b/test/packedAlphaTransportHarness.js index f47c54a..049cf52 100644 --- a/test/packedAlphaTransportHarness.js +++ b/test/packedAlphaTransportHarness.js @@ -7,6 +7,7 @@ const { } = require('../dist/main/modules/PackedAlphaTransport'); const { PACKED_ALPHA_TRANSPORT, + PACKED_STRAIGHT_ALPHA_TRANSPORT, } = require('../dist/main/types/TransparentBackgroundTransport'); void (async () => { @@ -40,9 +41,9 @@ void (async () => { ), { transparentBackground: true, - transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, + transparentBackgroundTransport: PACKED_STRAIGHT_ALPHA_TRANSPORT, }, - 'supported devices must request packed-alpha-v1', + 'supported devices must request the straight-colour packed-alpha-v2 contract', ); const originalWarn = console.warn; @@ -149,6 +150,15 @@ void (async () => { assert.notStrictEqual(packedOffer, ordinaryOffer); assert.equal(packedOffer.type, 'offer'); assert.equal(packedOffer.sdp, promotedSdp); + const packedV2Offer = prepareOfferForTransparentBackgroundTransport( + ordinaryOffer, + PACKED_STRAIGHT_ALPHA_TRANSPORT, + ); + assert.equal( + packedV2Offer.sdp, + promotedSdp, + 'v2 uses the same 1152x1536 H264 Level 4 transport geometry', + ); assert.equal( ordinaryOffer.sdp, ordinarySdp, diff --git a/test/transparentBackgroundGeometryHarness.js b/test/transparentBackgroundGeometryHarness.js index d974c2f..1eca4c0 100644 --- a/test/transparentBackgroundGeometryHarness.js +++ b/test/transparentBackgroundGeometryHarness.js @@ -6,6 +6,7 @@ const { } = require('../dist/main/modules/TransparentBackgroundRenderer'); const { PACKED_ALPHA_TRANSPORT, + PACKED_STRAIGHT_ALPHA_TRANSPORT, } = require('../dist/main/types/TransparentBackgroundTransport'); setClientMetricsDisabled(true); @@ -143,7 +144,7 @@ assert.deepEqual( buffers: cover.deletedResources.buffers.length, textures: cover.deletedResources.textures.length, }, - { programs: 2, buffers: 1, textures: 1 }, + { programs: 3, buffers: 1, textures: 1 }, 'destroy must release each shader program, buffer, and texture exactly once', ); @@ -165,6 +166,20 @@ assert.deepEqual( }, 'proportionally downscaled packed frames must preserve the two-plane layout', ); +assert.deepEqual( + resolveTransparentFrameGeometry( + 1152, + 1536, + PACKED_STRAIGHT_ALPHA_TRANSPORT, + 2048, + ), + { + mode: 'packed-alpha-v2', + canvasWidth: 1152, + canvasHeight: 768, + }, + 'v2 must preserve the same two-plane geometry while selecting straight-colour reconstruction', +); assert.deepEqual( resolveTransparentFrameGeometry(1152, 768, PACKED_ALPHA_TRANSPORT, 2048), { diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index c12c84c..546bfd1 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -2,6 +2,7 @@ const assert = require('assert'); const { FRAGMENT_SHADER_SOURCE, PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, reconstructPremultipliedForeground, resolveKeyOptions, } = require('../dist/main/modules/TransparentBackgroundRenderer'); @@ -156,4 +157,20 @@ assert.doesNotMatch( 'packed renderer must not threshold or reshape transported alpha', ); +assert.match( + PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, + /vec3 premultiplied = straight \* alpha/, + 'v2 must premultiply its padded straight colour by the explicit alpha in the client', +); +assert.match( + PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, + /gl_FragColor = vec4\(min\(premultiplied, vec3\(alpha\)\), alpha\)/, + 'v2 must submit valid premultiplied canvas colour', +); +assert.doesNotMatch( + PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, + /u_similarity|u_smoothness|u_spill|chroma\(/, + 'v2 must not re-key or despill its explicit server matte', +); + console.log('transparent background reconstruction harness passed'); From 386bd445f991fda70d3e5ee74ca2e024380ea989 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Wed, 22 Jul 2026 22:32:52 +0100 Subject: [PATCH 08/15] Add packed alpha transparent background rendering --- src/modules/PackedAlphaTransport.ts | 10 +- src/modules/TransparentBackgroundRenderer.ts | 358 +++++++++++++----- src/types/TransparentBackgroundTransport.ts | 10 +- test/packedAlphaTransportHarness.js | 22 +- test/transparentBackgroundGeometryHarness.js | 13 +- ...nsparentBackgroundReconstructionHarness.js | 120 +++++- 6 files changed, 399 insertions(+), 134 deletions(-) diff --git a/src/modules/PackedAlphaTransport.ts b/src/modules/PackedAlphaTransport.ts index 6b01b72..b41072f 100644 --- a/src/modules/PackedAlphaTransport.ts +++ b/src/modules/PackedAlphaTransport.ts @@ -1,7 +1,7 @@ import { StartSessionOptions } from '../types/coreApi/StartSessionOptions'; import { + PACKED_ALPHA_CPU_TRANSPORT, PACKED_ALPHA_TRANSPORT, - PACKED_STRAIGHT_ALPHA_TRANSPORT, TransparentBackgroundTransport, } from '../types/TransparentBackgroundTransport'; @@ -25,7 +25,7 @@ type MediaCapabilitiesLike = Pick; export type PackedAlphaCapability = 'supported' | 'unsupported' | 'unknown'; /** - * Query support for the H.264 Main Level 4.0 stream used by packed-alpha-v1/v2. + * Query support for the H.264 Main Level 4.0 packed-alpha stream. * Missing or inconclusive MediaCapabilities implementations are reported as * unknown. Callers conservatively retain the legacy carrier in that case: the * server requires an explicit Main-Level-4 offer, so guessing support could @@ -72,14 +72,14 @@ export async function buildTransparentBackgroundSessionOptions( const capability = await detectPackedAlphaCapability(mediaCapabilities); if (capability !== 'supported') { console.warn( - 'Packed transparent-background video support could not be confirmed as smooth on this device; falling back to legacy green-screen keying.', + 'Packed transparent-background video support could not be confirmed as smooth on this device; falling back to legacy adaptive chroma keying.', ); return { transparentBackground: true }; } return { transparentBackground: true, - transparentBackgroundTransport: PACKED_STRAIGHT_ALPHA_TRANSPORT, + transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, }; } @@ -118,7 +118,7 @@ export function prepareOfferForTransparentBackgroundTransport( ): RTCSessionDescriptionInit { if ( transport !== PACKED_ALPHA_TRANSPORT && - transport !== PACKED_STRAIGHT_ALPHA_TRANSPORT + transport !== PACKED_ALPHA_CPU_TRANSPORT ) { return offer; } diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index 955c11d..ab5a2be 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -4,14 +4,14 @@ import { } from '../lib/ClientMetrics'; import { TransparentBackgroundOptions } from '../types/TransparentBackgroundOptions'; import { + PACKED_ALPHA_CPU_TRANSPORT, PACKED_ALPHA_TRANSPORT, - PACKED_STRAIGHT_ALPHA_TRANSPORT, TransparentBackgroundTransport, } from '../types/TransparentBackgroundTransport'; // Calibrated on the held-out person-matting set after the engine's JPEG q90 // and H.264 Main/I420 path. A broad transition retains fractional-alpha hair; -// the exact carrier inverse below removes the green contribution from it. +// the exact carrier inverse below removes the selected carrier contribution. const DEFAULT_SIMILARITY = 0.005; const DEFAULT_SMOOTHNESS = 0.56; const DEFAULT_SPILL = 1.0; @@ -23,6 +23,39 @@ const BACKGROUND_ALPHA_FLOOR = 0.02; const SPILL_GUARD_START_ALPHA = 0.9; const SPILL_GUARD_END_ALPHA = 0.995; const TELEMETRY_FRAME_INTERVAL = 250; +const LEGACY_CARRIER_SAMPLE_SIZE = 64; +const LEGACY_CARRIER_MATCH_TOLERANCE = 48; +const LEGACY_CARRIER_MAX_DETECTION_ATTEMPTS = 3; + +export interface LegacyChromaCarrier { + name: 'green' | 'blue'; + rgb: readonly [number, number, number]; + axis: readonly [number, number, number]; + matchedSamples: number; +} + +const LEGACY_GREEN_CARRIER: LegacyChromaCarrier = { + name: 'green', + rgb: [0, 122 / 255, 51 / 255], + axis: [0, 1, 0], + matchedSamples: 0, +}; + +const LEGACY_CARRIER_CANDIDATES: readonly LegacyChromaCarrier[] = [ + LEGACY_GREEN_CARRIER, + { + name: 'blue', + rgb: [0, 71 / 255, 187 / 255], + axis: [0, 0, 1], + matchedSamples: 0, + }, + { + name: 'green', + rgb: [0, 1, 0], + axis: [0, 1, 0], + matchedSamples: 0, + }, +]; const VERTEX_SHADER_SOURCE = ` attribute vec2 a_position; @@ -34,12 +67,11 @@ void main() { } `; -// Key in chroma space so luminance variation introduced by H.264 does not -// turn a uniformly-green background into a noisy alpha plane. The source -// asset is composited over green directly in gamma-encoded sRGB, so subtracting -// the estimated green contribution recovers premultiplied foreground. Merely -// multiplying the carrier RGB by alpha would retain that green contribution -// and produce a bright fringe at translucent edges. +// Key in chroma space so luminance variation introduced by H.264 does not turn +// a uniform carrier into a noisy alpha plane. Avatar creation selects green or +// blue to avoid foreground colour collisions; a one-time decoded-border sample +// chooses the corresponding uniforms. Subtracting the gamma-encoded carrier +// contribution recovers premultiplied foreground at translucent edges. /** @internal */ export const FRAGMENT_SHADER_SOURCE = ` precision mediump float; @@ -48,6 +80,8 @@ uniform sampler2D u_frame; uniform float u_similarity; uniform float u_smoothness; uniform float u_spill; +uniform vec3 u_carrierColor; +uniform vec3 u_carrierAxis; varying vec2 v_texCoord; vec2 chroma(vec3 rgb) { @@ -58,8 +92,7 @@ vec2 chroma(vec3 rgb) { void main() { vec3 rgb = texture2D(u_frame, v_texCoord).rgb; - vec2 greenChroma = chroma(vec3(0.0, 1.0, 0.0)); - float chromaDistance = distance(chroma(rgb), greenChroma); + float chromaDistance = distance(chroma(rgb), chroma(u_carrierColor)); float alpha = smoothstep( u_similarity, u_similarity + max(u_smoothness, 0.0001), @@ -68,7 +101,7 @@ void main() { alpha *= step(${BACKGROUND_ALPHA_FLOOR.toFixed(3)}, alpha); vec3 premultiplied = clamp( - rgb - (1.0 - alpha) * vec3(0.0, 1.0, 0.0), + rgb - (1.0 - alpha) * u_carrierColor, vec3(0.0), vec3(alpha) ); @@ -77,22 +110,24 @@ void main() { ${SPILL_GUARD_END_ALPHA.toFixed(3)}, alpha ); - float greenExcess = max( - premultiplied.g - max(premultiplied.r, premultiplied.b), + float carrierValue = dot(premultiplied, u_carrierAxis); + vec3 otherChannels = premultiplied * (vec3(1.0) - u_carrierAxis); + float carrierExcess = max( + carrierValue - max(otherChannels.r, max(otherChannels.g, otherChannels.b)), 0.0 ); - premultiplied.g = max( - premultiplied.g - greenExcess * u_spill * spillGuard, - 0.0 + premultiplied = max( + premultiplied - u_carrierAxis * carrierExcess * u_spill * spillGuard, + vec3(0.0) ); gl_FragColor = vec4(premultiplied, alpha); } `; -// packed-alpha-v1 carries already-premultiplied colour in the top half of the -// video and a grayscale alpha plane in the bottom half. Sampling the planes -// directly avoids another estimate, despill pass, or alpha cutoff in-browser. +// Both packed transports carry already-premultiplied colour in the top half +// and a grayscale alpha plane in the bottom half. V1 keys pre-JPEG in M2F; v2 +// is the engine-CPU post-JPEG control. The client reconstruction is identical. /** @internal */ export const PACKED_ALPHA_FRAGMENT_SHADER_SOURCE = ` precision mediump float; @@ -120,34 +155,6 @@ void main() { } `; -// packed-alpha-v2 carries padded straight (unassociated) colour. Multiplying -// only after the separately decoded matte is sampled prevents 4:2:0 from -// blending foreground chroma with transparent black at moving hair edges. -/** @internal */ -export const PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE = ` -precision mediump float; - -uniform sampler2D u_frame; -varying vec2 v_texCoord; - -void main() { - vec2 colourCoord = vec2( - v_texCoord.x, - 0.5 + v_texCoord.y * 0.5 - ); - vec2 alphaCoord = vec2( - v_texCoord.x, - v_texCoord.y * 0.5 - ); - vec3 straight = texture2D(u_frame, colourCoord).rgb; - vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; - float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); - vec3 premultiplied = straight * alpha; - - gl_FragColor = vec4(min(premultiplied, vec3(alpha)), alpha); -} -`; - type OptionalVideoFrameCallbacks = { requestVideoFrameCallback?: ( callback: (now: DOMHighResTimeStamp) => void, @@ -169,10 +176,10 @@ interface GlResources { similarityLocation: WebGLUniformLocation; smoothnessLocation: WebGLUniformLocation; spillLocation: WebGLUniformLocation; + carrierColorLocation: WebGLUniformLocation; + carrierAxisLocation: WebGLUniformLocation; packedAlphaProgram: WebGLProgram; packedAlphaPositionLocation: number; - packedStraightAlphaProgram: WebGLProgram; - packedStraightAlphaPositionLocation: number; } export type TransparentFrameMode = @@ -217,6 +224,9 @@ export class TransparentBackgroundRenderer { private readonly originalParentPosition: string; private changedParentPosition = false; private lastGeometrySignature: string | null = null; + private legacyCarrier: LegacyChromaCarrier = LEGACY_GREEN_CARRIER; + private legacyCarrierDetected = false; + private legacyCarrierDetectionAttempts = 0; constructor( video: HTMLVideoElement, @@ -468,20 +478,15 @@ export class TransparentBackgroundRenderer { gl.viewport(0, 0, this.canvas.width, this.canvas.height); gl.clearColor(0, 0, 0, 0); gl.clear(gl.COLOR_BUFFER_BIT); - const packedAlpha = geometry.mode === PACKED_ALPHA_TRANSPORT; - const packedStraightAlpha = - geometry.mode === PACKED_STRAIGHT_ALPHA_TRANSPORT; - const packed = packedAlpha || packedStraightAlpha; - const program = packedStraightAlpha - ? this.resources.packedStraightAlphaProgram - : packedAlpha - ? this.resources.packedAlphaProgram - : this.resources.legacyProgram; - const positionLocation = packedStraightAlpha - ? this.resources.packedStraightAlphaPositionLocation - : packedAlpha - ? this.resources.packedAlphaPositionLocation - : this.resources.legacyPositionLocation; + const packedAlpha = + geometry.mode === PACKED_ALPHA_TRANSPORT || + geometry.mode === PACKED_ALPHA_CPU_TRANSPORT; + const program = packedAlpha + ? this.resources.packedAlphaProgram + : this.resources.legacyProgram; + const positionLocation = packedAlpha + ? this.resources.packedAlphaPositionLocation + : this.resources.legacyPositionLocation; gl.useProgram(program); gl.bindBuffer(gl.ARRAY_BUFFER, this.resources.positionBuffer); gl.enableVertexAttribArray(positionLocation); @@ -497,7 +502,25 @@ export class TransparentBackgroundRenderer { gl.UNSIGNED_BYTE, this.video, ); - if (!packed) { + if (!packedAlpha) { + if (!this.legacyCarrierDetected) { + const candidate = this.detectLegacyCarrierFromVideo(); + this.legacyCarrierDetectionAttempts += 1; + if ( + shouldFinalizeLegacyCarrierDetection( + candidate, + this.legacyCarrierDetectionAttempts, + ) + ) { + this.legacyCarrier = candidate; + this.legacyCarrierDetected = true; + this.report('legacy_carrier_detected', 1, { + carrier: this.legacyCarrier.name, + matchedSamples: this.legacyCarrier.matchedSamples, + attempts: this.legacyCarrierDetectionAttempts, + }); + } + } gl.uniform1f( this.resources.similarityLocation, this.keyOptions.similarity, @@ -507,6 +530,18 @@ export class TransparentBackgroundRenderer { this.keyOptions.smoothness, ); gl.uniform1f(this.resources.spillLocation, this.keyOptions.spill); + gl.uniform3f( + this.resources.carrierColorLocation, + this.legacyCarrier.rgb[0], + this.legacyCarrier.rgb[1], + this.legacyCarrier.rgb[2], + ); + gl.uniform3f( + this.resources.carrierAxisLocation, + this.legacyCarrier.axis[0], + this.legacyCarrier.axis[1], + this.legacyCarrier.axis[2], + ); } gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); } catch (error) { @@ -541,6 +576,76 @@ export class TransparentBackgroundRenderer { } } + private detectLegacyCarrierFromVideo(): LegacyChromaCarrier { + try { + const sampleCanvas = document.createElement('canvas'); + sampleCanvas.width = LEGACY_CARRIER_SAMPLE_SIZE; + sampleCanvas.height = 4; + const context = sampleCanvas.getContext('2d', { + willReadFrequently: true, + }); + if (!context) return LEGACY_GREEN_CARRIER; + + const sourceWidth = this.video.videoWidth; + const sourceHeight = this.video.videoHeight; + const border = Math.max( + 1, + Math.floor(Math.min(sourceWidth, sourceHeight) / 64), + ); + context.drawImage( + this.video, + 0, + 0, + sourceWidth, + border, + 0, + 0, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + context.drawImage( + this.video, + 0, + sourceHeight - border, + sourceWidth, + border, + 0, + 1, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + context.drawImage( + this.video, + 0, + 0, + border, + sourceHeight, + 0, + 2, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + context.drawImage( + this.video, + sourceWidth - border, + 0, + border, + sourceHeight, + 0, + 3, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + return detectLegacyChromaCarrier( + context.getImageData(0, 0, LEGACY_CARRIER_SAMPLE_SIZE, 4).data, + ); + } catch { + // WebRTC video is normally origin-clean. If a custom media source taints + // the sampling canvas, retain the historical green fallback. + return LEGACY_GREEN_CARRIER; + } + } + private applyFrameGeometry(geometry: TransparentFrameGeometry): void { const signature = `${geometry.mode}:${this.video.videoWidth}x${this.video.videoHeight}`; if (signature === this.lastGeometrySignature) return; @@ -567,11 +672,11 @@ export class TransparentBackgroundRenderer { this.video.style.opacity = '0'; if ( (this.transport === PACKED_ALPHA_TRANSPORT || - this.transport === PACKED_STRAIGHT_ALPHA_TRANSPORT) && + this.transport === PACKED_ALPHA_CPU_TRANSPORT) && geometry.mode === 'green-key-v1' ) { console.warn( - 'Packed transparent-background transport returned a legacy green-screen frame; using the compatibility keyer.', + 'Packed transparent-background transport returned a legacy chroma-carrier frame; using the adaptive compatibility keyer.', ); this.report('transport_fallback', 1, { deliveredTransport: 'green-key-v1', @@ -591,7 +696,6 @@ export class TransparentBackgroundRenderer { const gl = this.gl; let legacyProgram: WebGLProgram | null = null; let packedAlphaProgram: WebGLProgram | null = null; - let packedStraightAlphaProgram: WebGLProgram | null = null; let positionBuffer: WebGLBuffer | null = null; let texture: WebGLTexture | null = null; @@ -606,11 +710,6 @@ export class TransparentBackgroundRenderer { VERTEX_SHADER_SOURCE, PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, ); - packedStraightAlphaProgram = createProgram( - gl, - VERTEX_SHADER_SOURCE, - PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, - ); positionBuffer = gl.createBuffer(); texture = gl.createTexture(); @@ -637,10 +736,6 @@ export class TransparentBackgroundRenderer { packedAlphaProgram, 'a_position', ); - const packedStraightAlphaPositionLocation = gl.getAttribLocation( - packedStraightAlphaProgram, - 'a_position', - ); const similarityLocation = gl.getUniformLocation( legacyProgram, 'u_similarity', @@ -650,13 +745,22 @@ export class TransparentBackgroundRenderer { 'u_smoothness', ); const spillLocation = gl.getUniformLocation(legacyProgram, 'u_spill'); + const carrierColorLocation = gl.getUniformLocation( + legacyProgram, + 'u_carrierColor', + ); + const carrierAxisLocation = gl.getUniformLocation( + legacyProgram, + 'u_carrierAxis', + ); if ( legacyPositionLocation < 0 || packedAlphaPositionLocation < 0 || - packedStraightAlphaPositionLocation < 0 || !similarityLocation || !smoothnessLocation || - !spillLocation + !spillLocation || + !carrierColorLocation || + !carrierAxisLocation ) { throw new Error( 'Unable to resolve transparent renderer shader inputs.', @@ -671,18 +775,16 @@ export class TransparentBackgroundRenderer { similarityLocation, smoothnessLocation, spillLocation, + carrierColorLocation, + carrierAxisLocation, packedAlphaProgram, packedAlphaPositionLocation, - packedStraightAlphaProgram, - packedStraightAlphaPositionLocation, }; } catch (error) { if (positionBuffer) gl.deleteBuffer(positionBuffer); if (texture) gl.deleteTexture(texture); if (legacyProgram) gl.deleteProgram(legacyProgram); if (packedAlphaProgram) gl.deleteProgram(packedAlphaProgram); - if (packedStraightAlphaProgram) - gl.deleteProgram(packedStraightAlphaProgram); throw error; } } @@ -693,7 +795,6 @@ export class TransparentBackgroundRenderer { gl.deleteTexture(resources.texture); gl.deleteProgram(resources.legacyProgram); gl.deleteProgram(resources.packedAlphaProgram); - gl.deleteProgram(resources.packedStraightAlphaProgram); } private onContextLost(event: Event): void { @@ -811,7 +912,7 @@ export function resolveTransparentFrameGeometry( if ( transport !== PACKED_ALPHA_TRANSPORT && - transport !== PACKED_STRAIGHT_ALPHA_TRANSPORT + transport !== PACKED_ALPHA_CPU_TRANSPORT ) { return { mode: 'green-key-v1', @@ -854,6 +955,67 @@ export function resolveKeyOptions( }; } +/** + * Pick one legacy carrier from a small decoded border sample. The server-side + * packed path never calls this; it exists only for the compatibility keyer + * used when packed H.264 support cannot be confirmed on a device. + * + * @internal + */ +export function detectLegacyChromaCarrier( + rgbaSamples: ArrayLike, +): LegacyChromaCarrier { + if (rgbaSamples.length < 4) return { ...LEGACY_GREEN_CARRIER }; + + let winner = LEGACY_GREEN_CARRIER; + let winnerMatches = 0; + let winnerMatchedDistance = Number.POSITIVE_INFINITY; + for (const candidate of LEGACY_CARRIER_CANDIDATES) { + const carrierBytes = candidate.rgb.map((channel) => channel * 255); + let matches = 0; + let matchedDistance = 0; + for (let offset = 0; offset + 2 < rgbaSamples.length; offset += 4) { + const redDelta = Math.abs(rgbaSamples[offset] - carrierBytes[0]); + const greenDelta = Math.abs(rgbaSamples[offset + 1] - carrierBytes[1]); + const blueDelta = Math.abs(rgbaSamples[offset + 2] - carrierBytes[2]); + if ( + Math.max(redDelta, greenDelta, blueDelta) <= + LEGACY_CARRIER_MATCH_TOLERANCE + ) { + matches += 1; + matchedDistance += + redDelta * redDelta + greenDelta * greenDelta + blueDelta * blueDelta; + } + } + if ( + matches > winnerMatches || + (matches === winnerMatches && + matches > 0 && + matchedDistance < winnerMatchedDistance) + ) { + winner = candidate; + winnerMatches = matches; + winnerMatchedDistance = matchedDistance; + } + } + + return { + ...winner, + matchedSamples: winnerMatches, + }; +} + +/** @internal */ +export function shouldFinalizeLegacyCarrierDetection( + carrier: LegacyChromaCarrier, + attempts: number, +): boolean { + return ( + carrier.matchedSamples > 0 || + attempts >= LEGACY_CARRIER_MAX_DETECTION_ATTEMPTS + ); +} + /** * CPU reference for the fragment shader's carrier inversion. Kept alongside * the shader so focused tests can verify edge cases without a GPU dependency. @@ -864,23 +1026,37 @@ export function reconstructPremultipliedForeground( rgb: readonly [number, number, number], alphaValue: number, spillValue = DEFAULT_SPILL, + carrierRgb: readonly [number, number, number] = [0, 1, 0], ): [number, number, number] { const unclippedAlpha = clampUnit(alphaValue); const alpha = unclippedAlpha < BACKGROUND_ALPHA_FLOOR ? 0 : unclippedAlpha; const spill = clampUnit(spillValue); const premultiplied: [number, number, number] = [ - Math.min(alpha, Math.max(0, rgb[0])), - Math.min(alpha, Math.max(0, rgb[1] - (1 - alpha))), - Math.min(alpha, Math.max(0, rgb[2])), + Math.min(alpha, Math.max(0, rgb[0] - (1 - alpha) * carrierRgb[0])), + Math.min(alpha, Math.max(0, rgb[1] - (1 - alpha) * carrierRgb[1])), + Math.min(alpha, Math.max(0, rgb[2] - (1 - alpha) * carrierRgb[2])), ]; const spillGuard = 1 - smoothstep(SPILL_GUARD_START_ALPHA, SPILL_GUARD_END_ALPHA, alpha); - const greenExcess = Math.max( - premultiplied[1] - Math.max(premultiplied[0], premultiplied[2]), + const carrierChannel = + carrierRgb[2] > carrierRgb[1] && carrierRgb[2] > carrierRgb[0] + ? 2 + : carrierRgb[1] > carrierRgb[0] + ? 1 + : 0; + const otherChannels = [0, 1, 2].filter( + (channel) => channel !== carrierChannel, + ); + const carrierExcess = Math.max( + premultiplied[carrierChannel] - + Math.max( + premultiplied[otherChannels[0]], + premultiplied[otherChannels[1]], + ), 0, ); - premultiplied[1] = Math.max( - premultiplied[1] - greenExcess * spill * spillGuard, + premultiplied[carrierChannel] = Math.max( + premultiplied[carrierChannel] - carrierExcess * spill * spillGuard, 0, ); return premultiplied; diff --git a/src/types/TransparentBackgroundTransport.ts b/src/types/TransparentBackgroundTransport.ts index db7fbdb..475094c 100644 --- a/src/types/TransparentBackgroundTransport.ts +++ b/src/types/TransparentBackgroundTransport.ts @@ -1,6 +1,12 @@ export const PACKED_ALPHA_TRANSPORT = 'packed-alpha-v1' as const; -export const PACKED_STRAIGHT_ALPHA_TRANSPORT = 'packed-alpha-v2' as const; +/** @internal Engine-CPU cookbook-key control; carries the same premultiplied layout as v1. */ +export const PACKED_ALPHA_CPU_TRANSPORT = 'packed-alpha-v2' as const; +/** + * @deprecated V2 now carries premultiplied colour like v1; use PACKED_ALPHA_CPU_TRANSPORT. + * @internal + */ +export const PACKED_STRAIGHT_ALPHA_TRANSPORT = PACKED_ALPHA_CPU_TRANSPORT; export type TransparentBackgroundTransport = | typeof PACKED_ALPHA_TRANSPORT - | typeof PACKED_STRAIGHT_ALPHA_TRANSPORT; + | typeof PACKED_ALPHA_CPU_TRANSPORT; diff --git a/test/packedAlphaTransportHarness.js b/test/packedAlphaTransportHarness.js index 049cf52..7a3c050 100644 --- a/test/packedAlphaTransportHarness.js +++ b/test/packedAlphaTransportHarness.js @@ -6,11 +6,17 @@ const { promotePackedAlphaH264Level, } = require('../dist/main/modules/PackedAlphaTransport'); const { + PACKED_ALPHA_CPU_TRANSPORT, PACKED_ALPHA_TRANSPORT, PACKED_STRAIGHT_ALPHA_TRANSPORT, } = require('../dist/main/types/TransparentBackgroundTransport'); void (async () => { + assert.equal( + PACKED_STRAIGHT_ALPHA_TRANSPORT, + PACKED_ALPHA_CPU_TRANSPORT, + 'the deprecated packed-straight constant must remain a v2 compatibility alias', + ); let decodingConfiguration; const supportedMediaCapabilities = { decodingInfo: async (configuration) => { @@ -41,9 +47,9 @@ void (async () => { ), { transparentBackground: true, - transparentBackgroundTransport: PACKED_STRAIGHT_ALPHA_TRANSPORT, + transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, }, - 'supported devices must request the straight-colour packed-alpha-v2 contract', + 'supported devices must request the pre-JPEG premultiplied packed-alpha-v1 contract', ); const originalWarn = console.warn; @@ -61,12 +67,12 @@ void (async () => { }), }), { transparentBackground: true }, - 'an explicit unsupported result must retain the legacy green path', + 'an explicit unsupported result must retain the legacy adaptive chroma path', ); } finally { console.warn = originalWarn; } - assert.match(warning, /falling back to legacy green-screen keying/); + assert.match(warning, /falling back to legacy adaptive chroma keying/); assert.equal( await detectPackedAlphaCapability({ @@ -150,14 +156,14 @@ void (async () => { assert.notStrictEqual(packedOffer, ordinaryOffer); assert.equal(packedOffer.type, 'offer'); assert.equal(packedOffer.sdp, promotedSdp); - const packedV2Offer = prepareOfferForTransparentBackgroundTransport( + const packedCpuOffer = prepareOfferForTransparentBackgroundTransport( ordinaryOffer, - PACKED_STRAIGHT_ALPHA_TRANSPORT, + PACKED_ALPHA_CPU_TRANSPORT, ); assert.equal( - packedV2Offer.sdp, + packedCpuOffer.sdp, promotedSdp, - 'v2 uses the same 1152x1536 H264 Level 4 transport geometry', + 'the engine-CPU control uses the same packed H264 geometry', ); assert.equal( ordinaryOffer.sdp, diff --git a/test/transparentBackgroundGeometryHarness.js b/test/transparentBackgroundGeometryHarness.js index 1eca4c0..e6d24c0 100644 --- a/test/transparentBackgroundGeometryHarness.js +++ b/test/transparentBackgroundGeometryHarness.js @@ -5,8 +5,8 @@ const { TransparentBackgroundRenderer, } = require('../dist/main/modules/TransparentBackgroundRenderer'); const { + PACKED_ALPHA_CPU_TRANSPORT, PACKED_ALPHA_TRANSPORT, - PACKED_STRAIGHT_ALPHA_TRANSPORT, } = require('../dist/main/types/TransparentBackgroundTransport'); setClientMetricsDisabled(true); @@ -144,7 +144,7 @@ assert.deepEqual( buffers: cover.deletedResources.buffers.length, textures: cover.deletedResources.textures.length, }, - { programs: 3, buffers: 1, textures: 1 }, + { programs: 2, buffers: 1, textures: 1 }, 'destroy must release each shader program, buffer, and texture exactly once', ); @@ -167,18 +167,13 @@ assert.deepEqual( 'proportionally downscaled packed frames must preserve the two-plane layout', ); assert.deepEqual( - resolveTransparentFrameGeometry( - 1152, - 1536, - PACKED_STRAIGHT_ALPHA_TRANSPORT, - 2048, - ), + resolveTransparentFrameGeometry(1152, 1536, PACKED_ALPHA_CPU_TRANSPORT, 2048), { mode: 'packed-alpha-v2', canvasWidth: 1152, canvasHeight: 768, }, - 'v2 must preserve the same two-plane geometry while selecting straight-colour reconstruction', + 'the engine-CPU control must use the same premultiplied two-plane geometry', ); assert.deepEqual( resolveTransparentFrameGeometry(1152, 768, PACKED_ALPHA_TRANSPORT, 2048), diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index 546bfd1..157a40d 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -2,11 +2,14 @@ const assert = require('assert'); const { FRAGMENT_SHADER_SOURCE, PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, + detectLegacyChromaCarrier, reconstructPremultipliedForeground, resolveKeyOptions, + shouldFinalizeLegacyCarrierDetection, } = require('../dist/main/modules/TransparentBackgroundRenderer'); +const rgba = (pixels) => pixels.flatMap((pixel) => [...pixel, 255]); + const closeTo = (actual, expected, message, epsilon = 1e-7) => { assert.equal(actual.length, expected.length, message); actual.forEach((value, index) => { @@ -22,6 +25,84 @@ assert.deepEqual( { similarity: 0.005, smoothness: 0.56, spill: 1 }, 'calibrated defaults must remain explicit and covered by regression tests', ); + +const detectedGreen = detectLegacyChromaCarrier( + rgba([ + [2, 120, 53], + [0, 122, 51], + [4, 124, 50], + [210, 170, 130], + ]), +); +assert.equal(detectedGreen.name, 'green'); +closeTo( + detectedGreen.rgb, + [0, 122 / 255, 51 / 255], + 'decoded dark-green border must select the deployed green carrier', +); +assert.equal(detectedGreen.matchedSamples, 3); + +const detectedBlue = detectLegacyChromaCarrier( + rgba([ + [1, 72, 185], + [0, 71, 187], + [3, 69, 190], + [0, 220, 40], + ]), +); +assert.equal(detectedBlue.name, 'blue'); +closeTo( + detectedBlue.axis, + [0, 0, 1], + 'decoded blue border must select only the blue key channel', +); +assert.equal(detectedBlue.matchedSamples, 3); + +const legacyBrightGreen = detectLegacyChromaCarrier( + rgba([ + [0, 253, 1], + [2, 255, 0], + ]), +); +closeTo( + legacyBrightGreen.rgb, + [0, 1, 0], + 'legacy pure-green avatars must retain their original key colour', +); + +const unmatchedCarrier = detectLegacyChromaCarrier( + rgba([ + [32, 128, 224], + [180, 120, 90], + ]), +); +assert.equal(unmatchedCarrier.name, 'green'); +assert.equal(unmatchedCarrier.matchedSamples, 0); +closeTo( + unmatchedCarrier.rgb, + [0, 122 / 255, 51 / 255], + 'an inconclusive sample must fail closed to the current green carrier', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(unmatchedCarrier, 1), + false, + 'an inconclusive first decoded frame must be retried', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(unmatchedCarrier, 2), + false, + 'an inconclusive second decoded frame must still be retried', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(unmatchedCarrier, 3), + true, + 'inconclusive or tainted sampling must stop after a bounded third attempt', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(detectedBlue, 1), + true, + 'a conclusive blue carrier must lock on the first valid frame', +); assert.deepEqual( resolveKeyOptions({ similarity: 0.1, smoothness: 0.2, spill: 0 }), { similarity: 0.1, smoothness: 0.2, spill: 0 }, @@ -95,6 +176,18 @@ closeTo( 'spill=1 must apply the full guarded green-excess clamp at low alpha', ); +const blueForeground = [0.1, 0.8, 0.2]; +const blueAlpha = 0.5; +const blueKey = [0, 71 / 255, 187 / 255]; +const blueComposite = blueForeground.map( + (channel, index) => blueAlpha * channel + (1 - blueAlpha) * blueKey[index], +); +closeTo( + reconstructPremultipliedForeground(blueComposite, blueAlpha, 0, blueKey), + blueForeground.map((channel) => channel * blueAlpha), + 'blue-carrier inversion must preserve a deliberately green foreground', +); + for (const { rgb, alpha, spill } of [ { rgb: [-0.2, 1.4, 0.7], alpha: 0.3, spill: 0 }, { rgb: [0.8, 0.1, 1.2], alpha: 0.6, spill: 1 }, @@ -112,8 +205,13 @@ for (const { rgb, alpha, spill } of [ assert.match( FRAGMENT_SHADER_SOURCE, - /rgb - \(1\.0 - alpha\) \* vec3\(0\.0, 1\.0, 0\.0\)/, - 'shader must invert the gamma-encoded green carrier', + /rgb - \(1\.0 - alpha\) \* u_carrierColor/, + 'shader must invert the selected gamma-encoded carrier', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /premultiplied - u_carrierAxis \* carrierExcess/, + 'shader must despill only the selected green or blue channel', ); assert.match( FRAGMENT_SHADER_SOURCE, @@ -157,20 +255,4 @@ assert.doesNotMatch( 'packed renderer must not threshold or reshape transported alpha', ); -assert.match( - PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, - /vec3 premultiplied = straight \* alpha/, - 'v2 must premultiply its padded straight colour by the explicit alpha in the client', -); -assert.match( - PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, - /gl_FragColor = vec4\(min\(premultiplied, vec3\(alpha\)\), alpha\)/, - 'v2 must submit valid premultiplied canvas colour', -); -assert.doesNotMatch( - PACKED_STRAIGHT_ALPHA_FRAGMENT_SHADER_SOURCE, - /u_similarity|u_smoothness|u_spill|chroma\(/, - 'v2 must not re-key or despill its explicit server matte', -); - console.log('transparent background reconstruction harness passed'); From 87dabc7ffc33fa3362cfd06082b61f1a8c67db95 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Wed, 22 Jul 2026 23:08:18 +0100 Subject: [PATCH 09/15] Calibrate legacy transparent background key --- src/modules/TransparentBackgroundRenderer.ts | 98 ++++++++--- src/types/TransparentBackgroundOptions.ts | 19 ++- ...nsparentBackgroundReconstructionHarness.js | 155 +++++++++++------- 3 files changed, 177 insertions(+), 95 deletions(-) diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index ab5a2be..ede08c8 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -9,17 +9,20 @@ import { TransparentBackgroundTransport, } from '../types/TransparentBackgroundTransport'; -// Calibrated on the held-out person-matting set after the engine's JPEG q90 -// and H.264 Main/I420 path. A broad transition retains fractional-alpha hair; -// the exact carrier inverse below removes the selected carrier contribution. -const DEFAULT_SIMILARITY = 0.005; -const DEFAULT_SMOOTHNESS = 0.56; -const DEFAULT_SPILL = 1.0; +// Calibrated on whole-subject fit/validation/test splits across raw, H.264 +// 520 kbps, and H.264 2.42 Mbps stages. RGB distance preserves fractional +// hair better than the previous chroma-only key for the dark motion-safe +// carriers. The guarded colour recovery avoids amplifying codec hue errors. +const DEFAULT_SIMILARITY = 0.01; +const DEFAULT_SMOOTHNESS = 0.44; +const DEFAULT_SPILL = 0.5; // H.264/JPEG ringing leaves a very small non-zero alpha tail in otherwise // uniform background pixels. Keeping it produces a faint grey/coloured veil // over the page. The five-person transport holdout put the live p99.9 tail at // ~0.012; 0.02 clears it while changing edge error by only 0.2%. const BACKGROUND_ALPHA_FLOOR = 0.02; +const RECOVERY_BLEND_START_ALPHA = 0.12; +const RECOVERY_BLEND_END_ALPHA = 0.82; const SPILL_GUARD_START_ALPHA = 0.9; const SPILL_GUARD_END_ALPHA = 0.995; const TELEMETRY_FRAME_INTERVAL = 250; @@ -67,11 +70,9 @@ void main() { } `; -// Key in chroma space so luminance variation introduced by H.264 does not turn -// a uniform carrier into a noisy alpha plane. Avatar creation selects green or -// blue to avoid foreground colour collisions; a one-time decoded-border sample -// chooses the corresponding uniforms. Subtracting the gamma-encoded carrier -// contribution recovers premultiplied foreground at translucent edges. +// Avatar creation selects green or blue to avoid foreground colour collisions; +// a one-time decoded-border sample chooses the corresponding uniforms. The +// distance and colour-recovery constants mirror the held-out server key. /** @internal */ export const FRAGMENT_SHADER_SOURCE = ` precision mediump float; @@ -84,27 +85,28 @@ uniform vec3 u_carrierColor; uniform vec3 u_carrierAxis; varying vec2 v_texCoord; -vec2 chroma(vec3 rgb) { - float cb = -0.168736 * rgb.r - 0.331264 * rgb.g + 0.5 * rgb.b; - float cr = 0.5 * rgb.r - 0.418688 * rgb.g - 0.081312 * rgb.b; - return vec2(cb, cr); -} - void main() { vec3 rgb = texture2D(u_frame, v_texCoord).rgb; - float chromaDistance = distance(chroma(rgb), chroma(u_carrierColor)); + float carrierDistance = distance(rgb, u_carrierColor); float alpha = smoothstep( u_similarity, u_similarity + max(u_smoothness, 0.0001), - chromaDistance + carrierDistance ); alpha *= step(${BACKGROUND_ALPHA_FLOOR.toFixed(3)}, alpha); - vec3 premultiplied = clamp( + vec3 subtracted = clamp( rgb - (1.0 - alpha) * u_carrierColor, vec3(0.0), vec3(alpha) ); + vec3 observed = rgb * alpha; + float recoveryBlend = smoothstep( + ${RECOVERY_BLEND_START_ALPHA.toFixed(3)}, + ${RECOVERY_BLEND_END_ALPHA.toFixed(3)}, + alpha + ); + vec3 premultiplied = mix(observed, subtracted, recoveryBlend); float spillGuard = 1.0 - smoothstep( ${SPILL_GUARD_START_ALPHA.toFixed(3)}, ${SPILL_GUARD_END_ALPHA.toFixed(3)}, @@ -1017,8 +1019,9 @@ export function shouldFinalizeLegacyCarrierDetection( } /** - * CPU reference for the fragment shader's carrier inversion. Kept alongside - * the shader so focused tests can verify edge cases without a GPU dependency. + * CPU reference for the fragment shader's guarded colour recovery. Kept + * alongside the shader so focused tests can verify edge cases without a GPU + * dependency. * * @internal */ @@ -1026,16 +1029,31 @@ export function reconstructPremultipliedForeground( rgb: readonly [number, number, number], alphaValue: number, spillValue = DEFAULT_SPILL, - carrierRgb: readonly [number, number, number] = [0, 1, 0], + carrierRgb: readonly [number, number, number] = LEGACY_GREEN_CARRIER.rgb, ): [number, number, number] { const unclippedAlpha = clampUnit(alphaValue); const alpha = unclippedAlpha < BACKGROUND_ALPHA_FLOOR ? 0 : unclippedAlpha; const spill = clampUnit(spillValue); - const premultiplied: [number, number, number] = [ + const subtracted: [number, number, number] = [ Math.min(alpha, Math.max(0, rgb[0] - (1 - alpha) * carrierRgb[0])), Math.min(alpha, Math.max(0, rgb[1] - (1 - alpha) * carrierRgb[1])), Math.min(alpha, Math.max(0, rgb[2] - (1 - alpha) * carrierRgb[2])), ]; + const observed: [number, number, number] = [ + rgb[0] * alpha, + rgb[1] * alpha, + rgb[2] * alpha, + ]; + const recoveryBlend = smoothstep( + RECOVERY_BLEND_START_ALPHA, + RECOVERY_BLEND_END_ALPHA, + alpha, + ); + const premultiplied: [number, number, number] = [0, 1, 2].map( + (channel) => + observed[channel] * (1 - recoveryBlend) + + subtracted[channel] * recoveryBlend, + ) as [number, number, number]; const spillGuard = 1 - smoothstep(SPILL_GUARD_START_ALPHA, SPILL_GUARD_END_ALPHA, alpha); const carrierChannel = @@ -1062,6 +1080,38 @@ export function reconstructPremultipliedForeground( return premultiplied; } +/** + * CPU reference for the complete legacy fragment shader. Inputs and output + * are normalized sRGB/premultiplied RGBA respectively. + * + * @internal + */ +export function keyLegacyPixel( + rgb: readonly [number, number, number], + carrierRgb: readonly [number, number, number] = LEGACY_GREEN_CARRIER.rgb, + options?: TransparentBackgroundOptions, +): [number, number, number, number] { + const resolved = resolveKeyOptions(options); + const distance = Math.sqrt( + (rgb[0] - carrierRgb[0]) ** 2 + + (rgb[1] - carrierRgb[1]) ** 2 + + (rgb[2] - carrierRgb[2]) ** 2, + ); + const unclippedAlpha = smoothstep( + resolved.similarity, + resolved.similarity + Math.max(resolved.smoothness, 0.0001), + distance, + ); + const alpha = unclippedAlpha < BACKGROUND_ALPHA_FLOOR ? 0 : unclippedAlpha; + const premultiplied = reconstructPremultipliedForeground( + rgb, + alpha, + resolved.spill, + carrierRgb, + ); + return [...premultiplied, alpha]; +} + function clampUnit(value: number): number { if (!Number.isFinite(value)) return 0; return Math.min(1, Math.max(0, value)); diff --git a/src/types/TransparentBackgroundOptions.ts b/src/types/TransparentBackgroundOptions.ts index b5ca7d0..2d4290b 100644 --- a/src/types/TransparentBackgroundOptions.ts +++ b/src/types/TransparentBackgroundOptions.ts @@ -1,26 +1,27 @@ /** * Client-side chroma-key controls used when `transparentBackground` is on. * - * The defaults are tuned for Anam's generated exact-green avatar rendition. + * The defaults are tuned for Anam's generated motion-safe green/blue avatar + * renditions and mirror the server-side compatibility key. * Most applications should not need to change these values. */ export interface TransparentBackgroundOptions { /** - * Chroma distance that is treated as fully transparent. Lower values keep - * more green-adjacent detail; higher values remove more of the backdrop. - * @default 0.005 + * Normalized RGB distance that is treated as fully transparent. Lower values + * keep more carrier-adjacent detail; higher values remove more backdrop. + * @default 0.01 */ similarity?: number; /** * Width of the soft transition around the key threshold. - * @default 0.56 + * @default 0.44 */ smoothness?: number; /** - * Strength of guarded green-spill suppression at semi-transparent edges. - * Set to `0` to use only the exact green-carrier inverse, or `1` for the - * full clamp. Opaque foreground colours are preserved. - * @default 1 + * Strength of guarded carrier-channel spill suppression at semi-transparent + * edges. Set to `0` to disable despill or `1` for the full clamp. Opaque + * foreground colours are preserved. + * @default 0.5 */ spill?: number; } diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index 157a40d..046e276 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -3,6 +3,7 @@ const { FRAGMENT_SHADER_SOURCE, PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, detectLegacyChromaCarrier, + keyLegacyPixel, reconstructPremultipliedForeground, resolveKeyOptions, shouldFinalizeLegacyCarrierDetection, @@ -22,7 +23,7 @@ const closeTo = (actual, expected, message, epsilon = 1e-7) => { assert.deepEqual( resolveKeyOptions(), - { similarity: 0.005, smoothness: 0.56, spill: 1 }, + { similarity: 0.01, smoothness: 0.44, spill: 0.5 }, 'calibrated defaults must remain explicit and covered by regression tests', ); @@ -110,90 +111,100 @@ assert.deepEqual( ); closeTo( - reconstructPremultipliedForeground([0, 1, 0], 0), - [0, 0, 0], - 'fully transparent exact green must reconstruct to transparent black', + keyLegacyPixel([0, 122 / 255, 51 / 255]), + [0, 0, 0, 0], + 'the exact dark-green carrier must reconstruct to transparent black', ); closeTo( - reconstructPremultipliedForeground([0.01, 0.99, 0.01], 0.019), - [0, 0, 0], - 'sub-threshold codec noise must become fully transparent', + keyLegacyPixel([0.02, 122 / 255, 51 / 255]), + [0, 0, 0, 0], + 'the measured codec-noise alpha tail must be floored to transparent', ); -const blonde = [0.8, 0.7, 0.55]; -const blondeAlpha = 0.5; -const blondeCarrier = [ - blondeAlpha * blonde[0], - blondeAlpha * blonde[1] + (1 - blondeAlpha), - blondeAlpha * blonde[2], -]; closeTo( - reconstructPremultipliedForeground(blondeCarrier, blondeAlpha), - blonde.map((channel) => channel * blondeAlpha), - 'fractional blonde-like sRGB carrier must recover its premultiplied foreground', + keyLegacyPixel([0.05, 122 / 255, 51 / 255]), + [ + 0.0011645379413974457, 0.007900590739676788, 0.004658151765589783, + 0.023290758827948913, + ], + 'a just-visible carrier deviation must match the held-out RGB key fixture', ); +const darkGreen = [0, 122 / 255, 51 / 255]; closeTo( - reconstructPremultipliedForeground([0.2, 0.3, 0.4], 1), - [0.2, 0.3, 0.4], - 'opaque ordinary colour must stay unchanged', + keyLegacyPixel([0.12, 0.42, 0.18], darkGreen), + [ + 0.026715424077240528, 0.057808537750836525, 0.03477855641001681, + 0.19615563544778447, + ], + 'green-carrier guarded recovery must match the held-out CPU fixture', ); + +const blueKey = [0, 71 / 255, 187 / 255]; closeTo( - reconstructPremultipliedForeground([0.05, 0.9, 0.08], 1), - [0.05, 0.9, 0.08], - 'opaque genuine green must stay unchanged', + keyLegacyPixel([0.08, 0.35, 0.62], blueKey), + [ + 0.02658094751407277, 0.09548590370748197, 0.12334501296618385, + 0.257536997095212, + ], + 'blue-carrier guarded recovery must match the held-out CPU fixture', ); -const guardEndAlpha = 0.995; -const nearOpaqueGreen = [0.05, 0.9, 0.08]; -const nearOpaqueGreenCarrier = [ - guardEndAlpha * nearOpaqueGreen[0], - guardEndAlpha * nearOpaqueGreen[1] + (1 - guardEndAlpha), - guardEndAlpha * nearOpaqueGreen[2], -]; closeTo( - reconstructPremultipliedForeground(nearOpaqueGreenCarrier, guardEndAlpha), - nearOpaqueGreen.map((channel) => channel * guardEndAlpha), - 'green-spill clamp must be fully faded out by alpha 0.995', + keyLegacyPixel([0.05, 0.95, 0.08], darkGreen), + [0.05, 0.95, 0.08, 1], + 'opaque genuine green must stay unchanged', ); -const greenForeground = [0.1, 0.9, 0.1]; -const greenAlpha = 0.5; -const greenCarrier = [ - greenAlpha * greenForeground[0], - greenAlpha * greenForeground[1] + (1 - greenAlpha), - greenAlpha * greenForeground[2], -]; closeTo( - reconstructPremultipliedForeground(greenCarrier, greenAlpha, 0), - greenForeground.map((channel) => channel * greenAlpha), - 'spill=0 must perform only the exact carrier inverse', + reconstructPremultipliedForeground([0.01, 0.99, 0.01], 0.019), + [0, 0, 0], + 'the colour-recovery helper must apply the same background alpha floor', +); + +const guardedInput = [0.12, 0.42, 0.18]; +const guardedAlpha = 0.19615563544778447; +const withoutDespill = reconstructPremultipliedForeground( + guardedInput, + guardedAlpha, + 0, + darkGreen, +); +const withDespill = reconstructPremultipliedForeground( + guardedInput, + guardedAlpha, + 1, + darkGreen, +); +assert.ok( + withDespill[1] < withoutDespill[1], + 'despill must reduce only the carrier channel when it is excessive', ); closeTo( - reconstructPremultipliedForeground(greenCarrier, greenAlpha, 1), - [0.05, 0.05, 0.05], - 'spill=1 must apply the full guarded green-excess clamp at low alpha', + [withDespill[0], withDespill[2]], + [withoutDespill[0], withoutDespill[2]], + 'despill must preserve the non-carrier channels', ); -const blueForeground = [0.1, 0.8, 0.2]; -const blueAlpha = 0.5; -const blueKey = [0, 71 / 255, 187 / 255]; -const blueComposite = blueForeground.map( - (channel, index) => blueAlpha * channel + (1 - blueAlpha) * blueKey[index], -); +const nearOpaqueGreen = [0.05, 0.9, 0.08]; closeTo( - reconstructPremultipliedForeground(blueComposite, blueAlpha, 0, blueKey), - blueForeground.map((channel) => channel * blueAlpha), - 'blue-carrier inversion must preserve a deliberately green foreground', + reconstructPremultipliedForeground(nearOpaqueGreen, 1, 1, darkGreen), + nearOpaqueGreen, + 'recovery and despill must preserve opaque foreground colour', ); -for (const { rgb, alpha, spill } of [ - { rgb: [-0.2, 1.4, 0.7], alpha: 0.3, spill: 0 }, - { rgb: [0.8, 0.1, 1.2], alpha: 0.6, spill: 1 }, - { rgb: [0.5, 0.9, 0.2], alpha: 0.95, spill: 0.5 }, +for (const { rgb, alpha, spill, carrier } of [ + { rgb: [0.2, 0.7, 0.3], alpha: 0.3, spill: 0, carrier: darkGreen }, + { rgb: [0.8, 0.1, 1], alpha: 0.6, spill: 1, carrier: blueKey }, + { rgb: [0.5, 0.9, 0.2], alpha: 0.95, spill: 0.5, carrier: darkGreen }, ]) { - const reconstructed = reconstructPremultipliedForeground(rgb, alpha, spill); + const reconstructed = reconstructPremultipliedForeground( + rgb, + alpha, + spill, + carrier, + ); reconstructed.forEach((channel) => { assert.ok(channel >= 0, 'premultiplied channels must not be negative'); assert.ok( @@ -203,10 +214,30 @@ for (const { rgb, alpha, spill } of [ }); } +assert.match( + FRAGMENT_SHADER_SOURCE, + /distance\(rgb, u_carrierColor\)/, + 'shader must use normalized RGB distance from the detected carrier', +); assert.match( FRAGMENT_SHADER_SOURCE, /rgb - \(1\.0 - alpha\) \* u_carrierColor/, - 'shader must invert the selected gamma-encoded carrier', + 'shader must compute carrier-subtracted colour recovery', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /vec3 observed = rgb \* alpha/, + 'shader must include observed-colour premultiplication', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /mix\(observed, subtracted, recoveryBlend\)/, + 'shader must guard carrier subtraction with the calibrated blend', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /smoothstep\(\s*0\.120,\s*0\.820,\s*alpha/, + 'shader must use the calibrated guarded-recovery alpha interval', ); assert.match( FRAGMENT_SHADER_SOURCE, @@ -246,7 +277,7 @@ assert.match( ); assert.doesNotMatch( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /u_similarity|u_smoothness|u_spill|chroma\(/, + /u_similarity|u_smoothness|u_spill|carrierDistance|recoveryBlend/, 'packed renderer must not run the legacy key or despill operations', ); assert.doesNotMatch( From 14ea2baea2d593bd910224f4339d3f463bceb392 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Thu, 23 Jul 2026 11:20:45 +0100 Subject: [PATCH 10/15] Support portrait packed alpha rendering --- README.md | 17 +++-- src/modules/PackedAlphaTransport.ts | 56 ++++++++++----- src/modules/TransparentBackgroundRenderer.ts | 68 +++++++++++++++---- test/packedAlphaTransportHarness.js | 64 +++++++++++++---- test/transparentBackgroundGeometryHarness.js | 41 +++++++++++ ...nsparentBackgroundReconstructionHarness.js | 15 ++++ 6 files changed, 212 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 8e4e4b8..99bcf8b 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,10 @@ This will start a new session using the pre-configured persona id and start stre Set `transparentBackground` when creating the client. The SDK requests a packed colour/matte rendition and reconstructs it through a source-resolution -WebGL canvas over your video element. Browsers that cannot confirm smooth -support for the packed H.264 stream, and servers that return the original -1152x768 frame, automatically use the legacy green-screen keyer. +WebGL canvas over your video element. Landscape transport stacks colour over +alpha; portrait transport places colour left of alpha. Browsers that cannot +confirm smooth support for the packed H.264 stream, and servers that return an +ordinary Cara 4 frame, automatically use the legacy green-screen keyer. ```typescript const anamClient = createClient('your-session-token', { @@ -82,13 +83,15 @@ await anamClient.streamToVideoElement('video-element-id'); The supplied video must be attached to the DOM and should use `autoplay` and `playsinline` as usual. The underlying WebRTC `MediaStream` is still ordinary -opaque video: it contains vertically packed colour and alpha planes, or a -green-screen frame on the compatibility path. Transparency exists in the -SDK-managed canvas only. As a result, `stream()` returns that raw transport, +opaque video: it contains packed colour and alpha planes, or a green-screen +frame on the compatibility path. Transparency exists in the SDK-managed +canvas only. As a result, `stream()` returns that raw transport, and video-only browser features such as native controls and picture-in-picture do not automatically capture the transparent canvas. Use `getTransparentBackgroundCanvas()` if you need the managed canvas element for -layout or capture behavior. +layout or capture behavior. Session recordings and replays likewise contain +the raw packed video: landscape is vertically packed and portrait is +horizontally packed; they are not transparent RGBA recordings. To stop a session use the `stopStreaming` method. diff --git a/src/modules/PackedAlphaTransport.ts b/src/modules/PackedAlphaTransport.ts index b41072f..81097f0 100644 --- a/src/modules/PackedAlphaTransport.ts +++ b/src/modules/PackedAlphaTransport.ts @@ -5,19 +5,34 @@ import { TransparentBackgroundTransport, } from '../types/TransparentBackgroundTransport'; -const PACKED_ALPHA_DECODING_CONFIGURATION: MediaDecodingConfiguration = { - type: 'webrtc', - video: { - // Match the RTP format we actually negotiate. This is also the form used - // by the Media Capabilities specification's WebRTC H.264 example. - contentType: - 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', - width: 1152, - height: 1536, - bitrate: 2_500_000, - framerate: 25, - }, -}; +const PACKED_ALPHA_VIDEO_CONFIGURATION = { + // Match the RTP format we actually negotiate. This is also the form used by + // the Media Capabilities specification's WebRTC H.264 example. + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + bitrate: 2_500_000, + framerate: 25, +} as const; + +const PACKED_ALPHA_DECODING_CONFIGURATIONS: readonly MediaDecodingConfiguration[] = + [ + { + type: 'webrtc', + video: { + ...PACKED_ALPHA_VIDEO_CONFIGURATION, + width: 1152, + height: 1536, + }, + }, + { + type: 'webrtc', + video: { + ...PACKED_ALPHA_VIDEO_CONFIGURATION, + width: 1536, + height: 1152, + }, + }, + ]; type MediaCapabilitiesLike = Pick; @@ -25,7 +40,10 @@ type MediaCapabilitiesLike = Pick; export type PackedAlphaCapability = 'supported' | 'unsupported' | 'unknown'; /** - * Query support for the H.264 Main Level 4.0 packed-alpha stream. + * Query support for both orientations of the H.264 Main Level 4.0 packed-alpha + * stream. MediaCapabilities results can be orientation-sensitive, and the SDK + * does not know the token's requested output dimensions at this point, so both + * landscape and portrait carriers must independently report supported+smooth. * Missing or inconclusive MediaCapabilities implementations are reported as * unknown. Callers conservatively retain the legacy carrier in that case: the * server requires an explicit Main-Level-4 offer, so guessing support could @@ -42,13 +60,17 @@ export async function detectPackedAlphaCapability( if (!mediaCapabilities?.decodingInfo) return 'unknown'; try { - const result = await mediaCapabilities.decodingInfo( - PACKED_ALPHA_DECODING_CONFIGURATION, + const results = await Promise.all( + PACKED_ALPHA_DECODING_CONFIGURATIONS.map((configuration) => + mediaCapabilities.decodingInfo(configuration), + ), ); // The packed frame doubles the decoded pixel count. A decoder that can // technically accept Main Level 4.0 but is not expected to sustain this // configuration should use the lower-resolution compatibility path. - return result.supported && result.smooth ? 'supported' : 'unsupported'; + return results.every((result) => result.supported && result.smooth) + ? 'supported' + : 'unsupported'; } catch { // Some browsers expose MediaCapabilities but reject WebRTC configurations. // Treat this as inconclusive; the caller keeps the compatibility path. diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index ede08c8..a38bc44 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -127,25 +127,46 @@ void main() { } `; -// Both packed transports carry already-premultiplied colour in the top half -// and a grayscale alpha plane in the bottom half. V1 keys pre-JPEG in M2F; v2 -// is the engine-CPU post-JPEG control. The client reconstruction is identical. +// Both packed transports carry already-premultiplied colour and a grayscale +// alpha plane. Landscape stacks the planes vertically; portrait places them +// side-by-side so neither wire dimension exceeds WebGL 1's guaranteed 2048 +// texture limit. V1 keys pre-JPEG in M2F; v2 is the engine-CPU post-JPEG +// control. The client reconstruction is otherwise identical. /** @internal */ export const PACKED_ALPHA_FRAGMENT_SHADER_SOURCE = ` precision mediump float; uniform sampler2D u_frame; +uniform float u_horizontalLayout; varying vec2 v_texCoord; void main() { - vec2 colourCoord = vec2( + vec2 verticalColourCoord = vec2( v_texCoord.x, 0.5 + v_texCoord.y * 0.5 ); - vec2 alphaCoord = vec2( + vec2 verticalAlphaCoord = vec2( v_texCoord.x, v_texCoord.y * 0.5 ); + vec2 horizontalColourCoord = vec2( + v_texCoord.x * 0.5, + v_texCoord.y + ); + vec2 horizontalAlphaCoord = vec2( + 0.5 + v_texCoord.x * 0.5, + v_texCoord.y + ); + vec2 colourCoord = mix( + verticalColourCoord, + horizontalColourCoord, + u_horizontalLayout + ); + vec2 alphaCoord = mix( + verticalAlphaCoord, + horizontalAlphaCoord, + u_horizontalLayout + ); vec3 premultiplied = texture2D(u_frame, colourCoord).rgb; vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); @@ -182,6 +203,7 @@ interface GlResources { carrierAxisLocation: WebGLUniformLocation; packedAlphaProgram: WebGLProgram; packedAlphaPositionLocation: number; + packedAlphaHorizontalLayoutLocation: WebGLUniformLocation; } export type TransparentFrameMode = @@ -194,6 +216,7 @@ export interface TransparentFrameGeometry { mode: TransparentFrameMode; canvasWidth: number; canvasHeight: number; + packedLayout?: 'vertical' | 'horizontal'; reason?: string; } @@ -504,7 +527,12 @@ export class TransparentBackgroundRenderer { gl.UNSIGNED_BYTE, this.video, ); - if (!packedAlpha) { + if (packedAlpha) { + gl.uniform1f( + this.resources.packedAlphaHorizontalLayoutLocation, + geometry.packedLayout === 'horizontal' ? 1 : 0, + ); + } else { if (!this.legacyCarrierDetected) { const candidate = this.detectLegacyCarrierFromVideo(); this.legacyCarrierDetectionAttempts += 1; @@ -755,6 +783,10 @@ export class TransparentBackgroundRenderer { legacyProgram, 'u_carrierAxis', ); + const packedAlphaHorizontalLayoutLocation = gl.getUniformLocation( + packedAlphaProgram, + 'u_horizontalLayout', + ); if ( legacyPositionLocation < 0 || packedAlphaPositionLocation < 0 || @@ -762,7 +794,8 @@ export class TransparentBackgroundRenderer { !smoothnessLocation || !spillLocation || !carrierColorLocation || - !carrierAxisLocation + !carrierAxisLocation || + !packedAlphaHorizontalLayoutLocation ) { throw new Error( 'Unable to resolve transparent renderer shader inputs.', @@ -781,6 +814,7 @@ export class TransparentBackgroundRenderer { carrierAxisLocation, packedAlphaProgram, packedAlphaPositionLocation, + packedAlphaHorizontalLayoutLocation, }; } catch (error) { if (positionBuffer) gl.deleteBuffer(positionBuffer); @@ -877,10 +911,11 @@ function compileShader( } /** - * Resolve the decoded frame layout before uploading it to WebGL. Packed - * frames are 3:4 because two 3:2 planes are stacked vertically; legacy Cara 4 - * frames are 3:2. Ratio-based matching also permits a decoder to deliver a - * proportionally downscaled frame without silently sampling the wrong plane. + * Resolve the decoded frame layout before uploading it to WebGL. Landscape + * packed frames are 3:4 (two 3:2 planes stacked vertically); portrait packed + * frames are 4:3 (two 2:3 planes placed horizontally). Ratio-based matching + * also permits a decoder to deliver a proportionally downscaled frame without + * silently sampling the wrong plane. * * @internal */ @@ -928,9 +963,18 @@ export function resolveTransparentFrameGeometry( mode: transport, canvasWidth: width, canvasHeight: height / 2, + packedLayout: 'vertical', + }; + } + if (width % 2 === 0 && width * 3 === height * 4) { + return { + mode: transport, + canvasWidth: width / 2, + canvasHeight: height, + packedLayout: 'horizontal', }; } - if (width * 2 === height * 3) { + if (width * 2 === height * 3 || height * 2 === width * 3) { return { mode: 'green-key-v1', canvasWidth: width, diff --git a/test/packedAlphaTransportHarness.js b/test/packedAlphaTransportHarness.js index 7a3c050..d7d9101 100644 --- a/test/packedAlphaTransportHarness.js +++ b/test/packedAlphaTransportHarness.js @@ -17,10 +17,10 @@ void (async () => { PACKED_ALPHA_CPU_TRANSPORT, 'the deprecated packed-straight constant must remain a v2 compatibility alias', ); - let decodingConfiguration; + const decodingConfigurations = []; const supportedMediaCapabilities = { decodingInfo: async (configuration) => { - decodingConfiguration = configuration; + decodingConfigurations.push(configuration); return { supported: true, smooth: true, powerEfficient: true }; }, }; @@ -29,17 +29,34 @@ void (async () => { await detectPackedAlphaCapability(supportedMediaCapabilities), 'supported', ); - assert.deepEqual(decodingConfiguration, { - type: 'webrtc', - video: { - contentType: - 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', - width: 1152, - height: 1536, - bitrate: 2500000, - framerate: 25, - }, - }); + assert.deepEqual( + decodingConfigurations, + [ + { + type: 'webrtc', + video: { + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + width: 1152, + height: 1536, + bitrate: 2500000, + framerate: 25, + }, + }, + { + type: 'webrtc', + video: { + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + width: 1536, + height: 1152, + bitrate: 2500000, + framerate: 25, + }, + }, + ], + 'capability detection must probe both packed wire orientations', + ); assert.deepEqual( await buildTransparentBackgroundSessionOptions( true, @@ -86,6 +103,27 @@ void (async () => { 'a supported decoder that cannot sustain the packed resolution must use the compatibility path', ); + const orientationSpecificProbes = []; + assert.equal( + await detectPackedAlphaCapability({ + decodingInfo: async (configuration) => { + orientationSpecificProbes.push(configuration); + return { + supported: configuration.video.width === 1152, + smooth: true, + powerEfficient: true, + }; + }, + }), + 'unsupported', + 'portrait decode support must not be inferred from landscape support', + ); + assert.equal( + orientationSpecificProbes.length, + 2, + 'both orientations must be probed even when their results differ', + ); + assert.deepEqual( await buildTransparentBackgroundSessionOptions(true, { decodingInfo: async () => { diff --git a/test/transparentBackgroundGeometryHarness.js b/test/transparentBackgroundGeometryHarness.js index e6d24c0..e4e35d1 100644 --- a/test/transparentBackgroundGeometryHarness.js +++ b/test/transparentBackgroundGeometryHarness.js @@ -154,6 +154,7 @@ assert.deepEqual( mode: 'packed-alpha-v1', canvasWidth: 1152, canvasHeight: 768, + packedLayout: 'vertical', }, 'packed Cara 4 frames must expose a 1152x768 canvas', ); @@ -163,6 +164,7 @@ assert.deepEqual( mode: 'packed-alpha-v1', canvasWidth: 576, canvasHeight: 384, + packedLayout: 'vertical', }, 'proportionally downscaled packed frames must preserve the two-plane layout', ); @@ -172,9 +174,39 @@ assert.deepEqual( mode: 'packed-alpha-v2', canvasWidth: 1152, canvasHeight: 768, + packedLayout: 'vertical', }, 'the engine-CPU control must use the same premultiplied two-plane geometry', ); +assert.deepEqual( + resolveTransparentFrameGeometry(1536, 1152, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 768, + canvasHeight: 1152, + packedLayout: 'horizontal', + }, + 'packed Cara 4 portrait frames must expose a 768x1152 canvas', +); +assert.deepEqual( + resolveTransparentFrameGeometry(768, 576, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 384, + canvasHeight: 576, + packedLayout: 'horizontal', + }, + 'the portrait half-resolution ABR rung must preserve the two-plane layout', +); +assert.deepEqual( + resolveTransparentFrameGeometry(1536, 1152, undefined, 2048), + { + mode: 'green-key-v1', + canvasWidth: 1536, + canvasHeight: 1152, + }, + 'ordinary 4:3 video must never be unpacked without a negotiated packed transport', +); assert.deepEqual( resolveTransparentFrameGeometry(1152, 768, PACKED_ALPHA_TRANSPORT, 2048), { @@ -184,6 +216,15 @@ assert.deepEqual( }, 'a standard Cara 4 frame must select the legacy keyer compatibility path', ); +assert.deepEqual( + resolveTransparentFrameGeometry(768, 1152, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'green-key-v1', + canvasWidth: 768, + canvasHeight: 1152, + }, + 'a standard portrait Cara 4 frame must select the legacy keyer compatibility path', +); assert.equal( resolveTransparentFrameGeometry(1280, 720, PACKED_ALPHA_TRANSPORT, 2048).mode, 'unsupported', diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index 046e276..88bbb43 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -270,6 +270,21 @@ assert.match( /v_texCoord\.y \* 0\.5/, 'packed renderer must sample alpha from the bottom half', ); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /v_texCoord\.x \* 0\.5/, + 'packed renderer must sample portrait colour from the left half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /0\.5 \+ v_texCoord\.x \* 0\.5/, + 'packed renderer must sample portrait alpha from the right half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /u_horizontalLayout/, + 'packed renderer must select the orientation negotiated by frame geometry', +); assert.match( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, /gl_FragColor = vec4\(min\(premultiplied, vec3\(alpha\)\), alpha\)/, From 53ac21796cc1dfca0abc900b69240224cc8ebb98 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Thu, 23 Jul 2026 22:46:39 +0100 Subject: [PATCH 11/15] Clear packed alpha decoder pedestal --- src/modules/TransparentBackgroundRenderer.ts | 39 ++++++++++++++++++- ...nsparentBackgroundReconstructionHarness.js | 29 ++++++++++++-- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index a38bc44..f6bc245 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -21,6 +21,12 @@ const DEFAULT_SPILL = 0.5; // over the page. The five-person transport holdout put the live p99.9 tail at // ~0.012; 0.02 clears it while changing edge error by only 0.2%. const BACKGROUND_ALPHA_FLOOR = 0.02; +// Some browser hardware-decode paths expose limited-range video black to the +// WebGL texture as roughly 12-16/255 instead of zero. Packed alpha transports +// that decoder pedestal literally, painting a pale rectangle over the page +// even though the server matte is zero. Clear only that sub-visible pedestal; +// unlike feathering this leaves every alpha value above the floor unchanged. +const PACKED_ALPHA_BACKGROUND_FLOOR = 0.07; const RECOVERY_BLEND_START_ALPHA = 0.12; const RECOVERY_BLEND_END_ALPHA = 0.82; const SPILL_GUARD_START_ALPHA = 0.9; @@ -170,10 +176,17 @@ void main() { vec3 premultiplied = texture2D(u_frame, colourCoord).rgb; vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); + float foreground = step( + ${PACKED_ALPHA_BACKGROUND_FLOOR.toFixed(3)}, + alpha + ); + alpha *= foreground; + premultiplied *= foreground; // Compression can make an individual premultiplied colour channel exceed - // alpha by a code value. Clamp only that invalid premultiplied state; do not - // estimate, despill, or threshold the transported matte. + // alpha by a code value. Clamp only that invalid premultiplied state after + // clearing the decoded-video black pedestal; do not estimate or despill the + // transported matte. gl_FragColor = vec4(min(premultiplied, vec3(alpha)), alpha); } `; @@ -1124,6 +1137,28 @@ export function reconstructPremultipliedForeground( return premultiplied; } +/** + * CPU reference for packed-alpha black-pedestal removal. + * + * @internal + */ +export function reconstructPackedAlphaPixel( + premultipliedRgb: readonly [number, number, number], + alphaValue: number, +): [number, number, number, number] { + const unclippedAlpha = clampUnit(alphaValue); + if (unclippedAlpha < PACKED_ALPHA_BACKGROUND_FLOOR) { + return [0, 0, 0, 0]; + } + + return [ + Math.min(clampUnit(premultipliedRgb[0]), unclippedAlpha), + Math.min(clampUnit(premultipliedRgb[1]), unclippedAlpha), + Math.min(clampUnit(premultipliedRgb[2]), unclippedAlpha), + unclippedAlpha, + ]; +} + /** * CPU reference for the complete legacy fragment shader. Inputs and output * are normalized sRGB/premultiplied RGBA respectively. diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index 88bbb43..c905e41 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -4,6 +4,7 @@ const { PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, detectLegacyChromaCarrier, keyLegacyPixel, + reconstructPackedAlphaPixel, reconstructPremultipliedForeground, resolveKeyOptions, shouldFinalizeLegacyCarrierDetection, @@ -290,15 +291,35 @@ assert.match( /gl_FragColor = vec4\(min\(premultiplied, vec3\(alpha\)\), alpha\)/, 'packed renderer must submit the transported premultiplied colour and alpha', ); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /foreground = step\(\s*0\.070,\s*alpha\s*\)/, + 'packed renderer must clear the decoded-video black pedestal', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /premultiplied \*= foreground/, + 'packed renderer must clear pedestal colour together with pedestal alpha', +); assert.doesNotMatch( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, /u_similarity|u_smoothness|u_spill|carrierDistance|recoveryBlend/, 'packed renderer must not run the legacy key or despill operations', ); -assert.doesNotMatch( - PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /alpha\s*\*=\s*step|smoothstep\(/, - 'packed renderer must not threshold or reshape transported alpha', +closeTo( + reconstructPackedAlphaPixel([0.02, 0.03, 0.05], 16 / 255), + [0, 0, 0, 0], + 'decoded limited-range black must reconstruct to transparent black', +); +closeTo( + reconstructPackedAlphaPixel([0.03, 0.08, 0.2], 0.2), + [0.03, 0.08, 0.2, 0.2], + 'packed alpha above the pedestal must remain byte-for-byte unchanged', +); +closeTo( + reconstructPackedAlphaPixel([0.4, 0.1, 0.3], 0.25), + [0.25, 0.1, 0.25, 0.25], + 'invalid premultiplied channels must still clamp to alpha', ); console.log('transparent background reconstruction harness passed'); From 2de222757c17659d5b0a71d80584b92818720966 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Sat, 25 Jul 2026 00:06:38 +0100 Subject: [PATCH 12/15] Use engine CPU packed alpha transport --- src/modules/PackedAlphaTransport.ts | 2 +- test/packedAlphaTransportHarness.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/PackedAlphaTransport.ts b/src/modules/PackedAlphaTransport.ts index 81097f0..0e7a9bb 100644 --- a/src/modules/PackedAlphaTransport.ts +++ b/src/modules/PackedAlphaTransport.ts @@ -101,7 +101,7 @@ export async function buildTransparentBackgroundSessionOptions( return { transparentBackground: true, - transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, + transparentBackgroundTransport: PACKED_ALPHA_CPU_TRANSPORT, }; } diff --git a/test/packedAlphaTransportHarness.js b/test/packedAlphaTransportHarness.js index d7d9101..cf89e99 100644 --- a/test/packedAlphaTransportHarness.js +++ b/test/packedAlphaTransportHarness.js @@ -64,9 +64,9 @@ void (async () => { ), { transparentBackground: true, - transparentBackgroundTransport: PACKED_ALPHA_TRANSPORT, + transparentBackgroundTransport: PACKED_ALPHA_CPU_TRANSPORT, }, - 'supported devices must request the pre-JPEG premultiplied packed-alpha-v1 contract', + 'supported devices must request the engine-CPU RGB-distance packed-alpha-v2 contract', ); const originalWarn = console.warn; From f820c4a2011995bd198b768af891ef30769d61d9 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Sat, 25 Jul 2026 21:05:14 +0100 Subject: [PATCH 13/15] Feather transparent edges after decode --- src/modules/TransparentBackgroundRenderer.ts | 104 ++++++++++++++---- ...nsparentBackgroundReconstructionHarness.js | 23 +++- 2 files changed, 101 insertions(+), 26 deletions(-) diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index f6bc245..242a44b 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -144,44 +144,92 @@ precision mediump float; uniform sampler2D u_frame; uniform float u_horizontalLayout; +uniform vec2 u_outputTexel; varying vec2 v_texCoord; +float unpackAlpha(vec2 outputCoord) { + vec2 verticalAlphaCoord = vec2( + outputCoord.x, + outputCoord.y * 0.5 + ); + vec2 horizontalAlphaCoord = vec2( + 0.5 + outputCoord.x * 0.5, + outputCoord.y + ); + vec2 alphaCoord = mix( + verticalAlphaCoord, + horizontalAlphaCoord, + u_horizontalLayout + ); + vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; + float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); + return alpha * step(${PACKED_ALPHA_BACKGROUND_FLOOR.toFixed(3)}, alpha); +} + +float min3(float a, float b, float c) { + return min(a, min(b, c)); +} + void main() { vec2 verticalColourCoord = vec2( v_texCoord.x, 0.5 + v_texCoord.y * 0.5 ); - vec2 verticalAlphaCoord = vec2( - v_texCoord.x, - v_texCoord.y * 0.5 - ); vec2 horizontalColourCoord = vec2( v_texCoord.x * 0.5, v_texCoord.y ); - vec2 horizontalAlphaCoord = vec2( - 0.5 + v_texCoord.x * 0.5, - v_texCoord.y - ); vec2 colourCoord = mix( verticalColourCoord, horizontalColourCoord, u_horizontalLayout ); - vec2 alphaCoord = mix( - verticalAlphaCoord, - horizontalAlphaCoord, - u_horizontalLayout - ); vec3 premultiplied = texture2D(u_frame, colourCoord).rgb; - vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; - float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); - float foreground = step( - ${PACKED_ALPHA_BACKGROUND_FLOOR.toFixed(3)}, - alpha - ); - alpha *= foreground; - premultiplied *= foreground; + + // Apply the selected one-sided 1.5 px feather after decode, in output-plane + // pixels. ABR may halve the packed carrier before it reaches the browser; + // post-decode treatment keeps the visible feather stable at every rung. + float a00 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, -2.0)); + float a01 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, -2.0)); + float a02 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, -2.0)); + float a03 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, -2.0)); + float a04 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, -2.0)); + float a10 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, -1.0)); + float a11 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, -1.0)); + float a12 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, -1.0)); + float a13 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, -1.0)); + float a14 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, -1.0)); + float a20 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, 0.0)); + float a21 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, 0.0)); + float a22 = unpackAlpha(v_texCoord); + float a23 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, 0.0)); + float a24 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, 0.0)); + float a30 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, 1.0)); + float a31 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, 1.0)); + float a32 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, 1.0)); + float a33 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, 1.0)); + float a34 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, 1.0)); + float a40 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, 2.0)); + float a41 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, 2.0)); + float a42 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, 2.0)); + float a43 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, 2.0)); + float a44 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, 2.0)); + + float e00 = min3(min3(a00, a01, a02), min3(a10, a11, a12), min3(a20, a21, a22)); + float e01 = min3(min3(a01, a02, a03), min3(a11, a12, a13), min3(a21, a22, a23)); + float e02 = min3(min3(a02, a03, a04), min3(a12, a13, a14), min3(a22, a23, a24)); + float e10 = min3(min3(a10, a11, a12), min3(a20, a21, a22), min3(a30, a31, a32)); + float e11 = min3(min3(a11, a12, a13), min3(a21, a22, a23), min3(a31, a32, a33)); + float e12 = min3(min3(a12, a13, a14), min3(a22, a23, a24), min3(a32, a33, a34)); + float e20 = min3(min3(a20, a21, a22), min3(a30, a31, a32), min3(a40, a41, a42)); + float e21 = min3(min3(a21, a22, a23), min3(a31, a32, a33), min3(a41, a42, a43)); + float e22 = min3(min3(a22, a23, a24), min3(a32, a33, a34), min3(a42, a43, a44)); + + float b0 = (49.0 * e00 + 158.0 * e01 + 49.0 * e02) / 256.0; + float b1 = (49.0 * e10 + 158.0 * e11 + 49.0 * e12) / 256.0; + float b2 = (49.0 * e20 + 158.0 * e21 + 49.0 * e22) / 256.0; + float alpha = min(a22, (49.0 * b0 + 158.0 * b1 + 49.0 * b2) / 256.0); + premultiplied *= alpha / max(a22, 0.0001); // Compression can make an individual premultiplied colour channel exceed // alpha by a code value. Clamp only that invalid premultiplied state after @@ -217,6 +265,7 @@ interface GlResources { packedAlphaProgram: WebGLProgram; packedAlphaPositionLocation: number; packedAlphaHorizontalLayoutLocation: WebGLUniformLocation; + packedAlphaOutputTexelLocation: WebGLUniformLocation; } export type TransparentFrameMode = @@ -545,6 +594,11 @@ export class TransparentBackgroundRenderer { this.resources.packedAlphaHorizontalLayoutLocation, geometry.packedLayout === 'horizontal' ? 1 : 0, ); + gl.uniform2f( + this.resources.packedAlphaOutputTexelLocation, + 1 / this.canvas.width, + 1 / this.canvas.height, + ); } else { if (!this.legacyCarrierDetected) { const candidate = this.detectLegacyCarrierFromVideo(); @@ -800,6 +854,10 @@ export class TransparentBackgroundRenderer { packedAlphaProgram, 'u_horizontalLayout', ); + const packedAlphaOutputTexelLocation = gl.getUniformLocation( + packedAlphaProgram, + 'u_outputTexel', + ); if ( legacyPositionLocation < 0 || packedAlphaPositionLocation < 0 || @@ -808,7 +866,8 @@ export class TransparentBackgroundRenderer { !spillLocation || !carrierColorLocation || !carrierAxisLocation || - !packedAlphaHorizontalLayoutLocation + !packedAlphaHorizontalLayoutLocation || + !packedAlphaOutputTexelLocation ) { throw new Error( 'Unable to resolve transparent renderer shader inputs.', @@ -828,6 +887,7 @@ export class TransparentBackgroundRenderer { packedAlphaProgram, packedAlphaPositionLocation, packedAlphaHorizontalLayoutLocation, + packedAlphaOutputTexelLocation, }; } catch (error) { if (positionBuffer) gl.deleteBuffer(positionBuffer); diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index c905e41..423bc3d 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -278,7 +278,7 @@ assert.match( ); assert.match( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /0\.5 \+ v_texCoord\.x \* 0\.5/, + /0\.5 \+ outputCoord\.x \* 0\.5/, 'packed renderer must sample portrait alpha from the right half', ); assert.match( @@ -293,13 +293,28 @@ assert.match( ); assert.match( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /foreground = step\(\s*0\.070,\s*alpha\s*\)/, + /alpha \* step\(0\.070, alpha\)/, 'packed renderer must clear the decoded-video black pedestal', ); assert.match( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /premultiplied \*= foreground/, - 'packed renderer must clear pedestal colour together with pedestal alpha', + /uniform vec2 u_outputTexel/, + 'packed renderer must express the feather in decoded output pixels', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /float e11 = min3/, + 'packed renderer must erode only the outside edge', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /49\.0 \* b0 \+ 158\.0 \* b1 \+ 49\.0 \* b2/, + 'packed renderer must apply the selected three-tap feather', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /premultiplied \*= alpha \/ max\(a22, 0\.0001\)/, + 'packed renderer must keep colour premultiplied when alpha contracts', ); assert.doesNotMatch( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, From 0c81f718a6cf191b71b6ffc34ee3456c9a129311 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Sat, 25 Jul 2026 21:51:21 +0100 Subject: [PATCH 14/15] Use server-feathered transparent alpha --- src/modules/TransparentBackgroundRenderer.ts | 71 ++----------------- ...nsparentBackgroundReconstructionHarness.js | 20 ++---- 2 files changed, 10 insertions(+), 81 deletions(-) diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index 242a44b..a839603 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -144,7 +144,6 @@ precision mediump float; uniform sampler2D u_frame; uniform float u_horizontalLayout; -uniform vec2 u_outputTexel; varying vec2 v_texCoord; float unpackAlpha(vec2 outputCoord) { @@ -166,10 +165,6 @@ float unpackAlpha(vec2 outputCoord) { return alpha * step(${PACKED_ALPHA_BACKGROUND_FLOOR.toFixed(3)}, alpha); } -float min3(float a, float b, float c) { - return min(a, min(b, c)); -} - void main() { vec2 verticalColourCoord = vec2( v_texCoord.x, @@ -185,56 +180,12 @@ void main() { u_horizontalLayout ); vec3 premultiplied = texture2D(u_frame, colourCoord).rgb; - - // Apply the selected one-sided 1.5 px feather after decode, in output-plane - // pixels. ABR may halve the packed carrier before it reaches the browser; - // post-decode treatment keeps the visible feather stable at every rung. - float a00 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, -2.0)); - float a01 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, -2.0)); - float a02 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, -2.0)); - float a03 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, -2.0)); - float a04 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, -2.0)); - float a10 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, -1.0)); - float a11 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, -1.0)); - float a12 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, -1.0)); - float a13 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, -1.0)); - float a14 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, -1.0)); - float a20 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, 0.0)); - float a21 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, 0.0)); - float a22 = unpackAlpha(v_texCoord); - float a23 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, 0.0)); - float a24 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, 0.0)); - float a30 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, 1.0)); - float a31 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, 1.0)); - float a32 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, 1.0)); - float a33 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, 1.0)); - float a34 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, 1.0)); - float a40 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-2.0, 2.0)); - float a41 = unpackAlpha(v_texCoord + u_outputTexel * vec2(-1.0, 2.0)); - float a42 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 0.0, 2.0)); - float a43 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 1.0, 2.0)); - float a44 = unpackAlpha(v_texCoord + u_outputTexel * vec2( 2.0, 2.0)); - - float e00 = min3(min3(a00, a01, a02), min3(a10, a11, a12), min3(a20, a21, a22)); - float e01 = min3(min3(a01, a02, a03), min3(a11, a12, a13), min3(a21, a22, a23)); - float e02 = min3(min3(a02, a03, a04), min3(a12, a13, a14), min3(a22, a23, a24)); - float e10 = min3(min3(a10, a11, a12), min3(a20, a21, a22), min3(a30, a31, a32)); - float e11 = min3(min3(a11, a12, a13), min3(a21, a22, a23), min3(a31, a32, a33)); - float e12 = min3(min3(a12, a13, a14), min3(a22, a23, a24), min3(a32, a33, a34)); - float e20 = min3(min3(a20, a21, a22), min3(a30, a31, a32), min3(a40, a41, a42)); - float e21 = min3(min3(a21, a22, a23), min3(a31, a32, a33), min3(a41, a42, a43)); - float e22 = min3(min3(a22, a23, a24), min3(a32, a33, a34), min3(a42, a43, a44)); - - float b0 = (49.0 * e00 + 158.0 * e01 + 49.0 * e02) / 256.0; - float b1 = (49.0 * e10 + 158.0 * e11 + 49.0 * e12) / 256.0; - float b2 = (49.0 * e20 + 158.0 * e21 + 49.0 * e22) / 256.0; - float alpha = min(a22, (49.0 * b0 + 158.0 * b1 + 49.0 * b2) / 256.0); - premultiplied *= alpha / max(a22, 0.0001); + float alpha = unpackAlpha(v_texCoord); // Compression can make an individual premultiplied colour channel exceed - // alpha by a code value. Clamp only that invalid premultiplied state after - // clearing the decoded-video black pedestal; do not estimate or despill the - // transported matte. + // alpha by a code value. The engine has already applied the canonical matte + // feather, so the client only clears the decoded-video black pedestal and + // clamps that invalid premultiplied state. gl_FragColor = vec4(min(premultiplied, vec3(alpha)), alpha); } `; @@ -265,7 +216,6 @@ interface GlResources { packedAlphaProgram: WebGLProgram; packedAlphaPositionLocation: number; packedAlphaHorizontalLayoutLocation: WebGLUniformLocation; - packedAlphaOutputTexelLocation: WebGLUniformLocation; } export type TransparentFrameMode = @@ -594,11 +544,6 @@ export class TransparentBackgroundRenderer { this.resources.packedAlphaHorizontalLayoutLocation, geometry.packedLayout === 'horizontal' ? 1 : 0, ); - gl.uniform2f( - this.resources.packedAlphaOutputTexelLocation, - 1 / this.canvas.width, - 1 / this.canvas.height, - ); } else { if (!this.legacyCarrierDetected) { const candidate = this.detectLegacyCarrierFromVideo(); @@ -854,10 +799,6 @@ export class TransparentBackgroundRenderer { packedAlphaProgram, 'u_horizontalLayout', ); - const packedAlphaOutputTexelLocation = gl.getUniformLocation( - packedAlphaProgram, - 'u_outputTexel', - ); if ( legacyPositionLocation < 0 || packedAlphaPositionLocation < 0 || @@ -866,8 +807,7 @@ export class TransparentBackgroundRenderer { !spillLocation || !carrierColorLocation || !carrierAxisLocation || - !packedAlphaHorizontalLayoutLocation || - !packedAlphaOutputTexelLocation + !packedAlphaHorizontalLayoutLocation ) { throw new Error( 'Unable to resolve transparent renderer shader inputs.', @@ -887,7 +827,6 @@ export class TransparentBackgroundRenderer { packedAlphaProgram, packedAlphaPositionLocation, packedAlphaHorizontalLayoutLocation, - packedAlphaOutputTexelLocation, }; } catch (error) { if (positionBuffer) gl.deleteBuffer(positionBuffer); diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index 423bc3d..8449366 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -298,23 +298,13 @@ assert.match( ); assert.match( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /uniform vec2 u_outputTexel/, - 'packed renderer must express the feather in decoded output pixels', + /float alpha = unpackAlpha\(v_texCoord\)/, + 'packed renderer must use the server-feathered transported alpha directly', ); -assert.match( - PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /float e11 = min3/, - 'packed renderer must erode only the outside edge', -); -assert.match( - PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /49\.0 \* b0 \+ 158\.0 \* b1 \+ 49\.0 \* b2/, - 'packed renderer must apply the selected three-tap feather', -); -assert.match( +assert.doesNotMatch( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, - /premultiplied \*= alpha \/ max\(a22, 0\.0001\)/, - 'packed renderer must keep colour premultiplied when alpha contracts', + /u_outputTexel|min3|49\.0 \* b0|premultiplied \*=/, + 'packed renderer must not apply an additional client-side feather', ); assert.doesNotMatch( PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, From c6296032b1d9810a401e08bf612746085db94a94 Mon Sep 17 00:00:00 2001 From: Ben Carr Date: Tue, 4 Aug 2026 12:59:24 +0100 Subject: [PATCH 15/15] Calibrate decoded transparent carrier --- src/modules/TransparentBackgroundRenderer.ts | 33 +++++++++++++++ ...nsparentBackgroundReconstructionHarness.js | 42 +++++++++++++++++-- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts index a839603..e0e724d 100644 --- a/src/modules/TransparentBackgroundRenderer.ts +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -1028,10 +1028,12 @@ export function detectLegacyChromaCarrier( let winner = LEGACY_GREEN_CARRIER; let winnerMatches = 0; let winnerMatchedDistance = Number.POSITIVE_INFINITY; + let winnerMatchedPixels: Array = []; for (const candidate of LEGACY_CARRIER_CANDIDATES) { const carrierBytes = candidate.rgb.map((channel) => channel * 255); let matches = 0; let matchedDistance = 0; + const matchedPixels: Array = []; for (let offset = 0; offset + 2 < rgbaSamples.length; offset += 4) { const redDelta = Math.abs(rgbaSamples[offset] - carrierBytes[0]); const greenDelta = Math.abs(rgbaSamples[offset + 1] - carrierBytes[1]); @@ -1041,6 +1043,11 @@ export function detectLegacyChromaCarrier( LEGACY_CARRIER_MATCH_TOLERANCE ) { matches += 1; + matchedPixels.push([ + rgbaSamples[offset], + rgbaSamples[offset + 1], + rgbaSamples[offset + 2], + ]); matchedDistance += redDelta * redDelta + greenDelta * greenDelta + blueDelta * blueDelta; } @@ -1054,15 +1061,41 @@ export function detectLegacyChromaCarrier( winner = candidate; winnerMatches = matches; winnerMatchedDistance = matchedDistance; + winnerMatchedPixels = matchedPixels; } } + // Browser video decode is not colour-exact. In particular, limited-range + // H.264 paths can lift or shift every carrier pixel by enough that keying + // against the ideal server RGB leaves a faint, frame-shaped alpha veil. + // The matched border pixels are the carrier after that decode path. Their + // per-channel medians give us a robust local key colour while ignoring the + // foreground pixels that can touch one edge of the frame. + const decodedRgb = + winnerMatches > 0 + ? ([0, 1, 2].map((channel) => + medianByte(winnerMatchedPixels.map((pixel) => pixel[channel])), + ) as [number, number, number]) + : winner.rgb; + return { ...winner, + rgb: decodedRgb, matchedSamples: winnerMatches, }; } +function medianByte(values: readonly number[]): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + const median = + sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; + return clampUnit(median / 255); +} + /** @internal */ export function shouldFinalizeLegacyCarrierDetection( carrier: LegacyChromaCarrier, diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js index 8449366..c61d2c9 100644 --- a/test/transparentBackgroundReconstructionHarness.js +++ b/test/transparentBackgroundReconstructionHarness.js @@ -39,8 +39,8 @@ const detectedGreen = detectLegacyChromaCarrier( assert.equal(detectedGreen.name, 'green'); closeTo( detectedGreen.rgb, - [0, 122 / 255, 51 / 255], - 'decoded dark-green border must select the deployed green carrier', + [2 / 255, 122 / 255, 51 / 255], + 'decoded dark-green border must calibrate the selected carrier', ); assert.equal(detectedGreen.matchedSamples, 3); @@ -59,6 +59,11 @@ closeTo( 'decoded blue border must select only the blue key channel', ); assert.equal(detectedBlue.matchedSamples, 3); +closeTo( + detectedBlue.rgb, + [1 / 255, 71 / 255, 187 / 255], + 'decoded blue border must calibrate the selected carrier', +); const legacyBrightGreen = detectLegacyChromaCarrier( rgba([ @@ -68,8 +73,37 @@ const legacyBrightGreen = detectLegacyChromaCarrier( ); closeTo( legacyBrightGreen.rgb, - [0, 1, 0], - 'legacy pure-green avatars must retain their original key colour', + [1 / 255, 254 / 255, 0.5 / 255], + 'legacy pure-green avatars must calibrate around their decoded key colour', +); + +const liftedDecodedCarrier = detectLegacyChromaCarrier( + rgba([ + [13, 130, 62], + [15, 132, 64], + [14, 131, 63], + [210, 170, 130], + ]), +); +closeTo( + liftedDecodedCarrier.rgb, + [14 / 255, 131 / 255, 63 / 255], + 'limited-range decode lift must be absorbed into the local carrier key', +); +assert.ok( + keyLegacyPixel( + [14 / 255, 131 / 255, 63 / 255], + [0, 122 / 255, 51 / 255], + )[3] > 0.02, + 'the ideal server key reproduces the visible frame-shaped alpha veil', +); +closeTo( + keyLegacyPixel( + [14 / 255, 131 / 255, 63 / 255], + liftedDecodedCarrier.rgb, + ), + [0, 0, 0, 0], + 'the decoded local key must reconstruct the lifted carrier as transparent', ); const unmatchedCarrier = detectLegacyChromaCarrier(