diff --git a/README.md b/README.md index b5f69fd..99bcf8b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,35 @@ await anamClient.streamToVideoElement('video-element-id'); This will start a new session using the pre-configured persona id and start streaming video element in the DOM with the matching element id. +### Transparent avatar background + +Set `transparentBackground` when creating the client. The SDK requests a +packed colour/matte rendition and reconstructs it through a source-resolution +WebGL canvas over your video element. Landscape transport stacks colour over +alpha; portrait transport places colour left of alpha. Browsers that cannot +confirm smooth support for the packed H.264 stream, and servers that return an +ordinary Cara 4 frame, automatically use the legacy green-screen keyer. + +```typescript +const anamClient = createClient('your-session-token', { + transparentBackground: true, +}); + +await anamClient.streamToVideoElement('video-element-id'); +``` + +The supplied video must be attached to the DOM and should use `autoplay` and +`playsinline` as usual. The underlying WebRTC `MediaStream` is still ordinary +opaque video: it contains packed colour and alpha planes, or a green-screen +frame on the compatibility path. Transparency exists in the SDK-managed +canvas only. As a result, `stream()` returns that raw transport, +and video-only browser features such as native controls and picture-in-picture +do not automatically capture the transparent canvas. Use +`getTransparentBackgroundCanvas()` if you need the managed canvas element for +layout or capture behavior. Session recordings and replays likewise contain +the raw packed video: landscape is vertically packed and portrait is +horizontally packed; they are not transparent RGBA recordings. + To stop a session use the `stopStreaming` method. ```typescript diff --git a/package.json b/package.json index 9e727b3..3ad97e4 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "lint": "eslint \"{src,test}/**/*.ts\" --fix", "test": "echo \"Error: no test specified\" && exit 1", "test:connection-milestones": "npm run build:main && node test/connectionMilestonesHarness.js", + "test:transparent-background": "npm run build:main && node test/transparentBackgroundGeometryHarness.js && node test/transparentBackgroundReconstructionHarness.js && node test/packedAlphaTransportHarness.js", "watch": "nodemon -e ts --watch src --exec \"npm run build\"", "prepare": "npm run build" }, diff --git a/src/AnamClient.ts b/src/AnamClient.ts index 8a3c693..574a6b2 100644 --- a/src/AnamClient.ts +++ b/src/AnamClient.ts @@ -37,6 +37,7 @@ import { import { AgentAudioInputStream } from './types/AgentAudioInputStream'; import { TalkMessageStream } from './types/TalkMessageStream'; import { ToolCallManager } from './modules/ToolCallManager'; +import { buildTransparentBackgroundSessionOptions } from './modules/PackedAlphaTransport'; export default class AnamClient { private publicEventEmitter: PublicEventEmitter; private internalEventEmitter: InternalEventEmitter; @@ -198,11 +199,19 @@ export default class AnamClient { return undefined; } - private buildStartSessionOptionsForClient(): StartSessionOptions | undefined { + private async buildStartSessionOptionsForClient(): Promise< + StartSessionOptions | undefined + > { const sessionOptions: StartSessionOptions = {}; if (this.clientOptions?.voiceDetection) { sessionOptions.voiceDetection = this.clientOptions.voiceDetection; } + Object.assign( + sessionOptions, + await buildTransparentBackgroundSessionOptions( + this.clientOptions?.transparentBackground, + ), + ); // return undefined if no options are set if (Object.keys(sessionOptions).length === 0) { return undefined; @@ -245,7 +254,7 @@ export default class AnamClient { const config = this.personaConfig; // build session options from client options const sessionOptions: StartSessionOptions | undefined = - this.buildStartSessionOptionsForClient(); + await this.buildStartSessionOptionsForClient(); // start a new session connectionMilestones?.record('start_session_request_started'); let response: StartSessionResponse; @@ -321,6 +330,11 @@ export default class AnamClient { disableInputAudio: this.clientOptions?.disableInputAudio, }, apiGateway: this.clientOptions?.api?.apiGateway, + transparentBackground: { + enabled: this.clientOptions?.transparentBackground === true, + keyOptions: this.clientOptions?.transparentBackgroundOptions, + transport: sessionOptions?.transparentBackgroundTransport, + }, metrics: { showPeerConnectionStatsReport: this.clientOptions?.metrics?.showPeerConnectionStatsReport ?? @@ -492,7 +506,6 @@ export default class AnamClient { }); throw new Error('Already streaming'); } - this._isStreaming = true; if (!this.streamingClient) { connectionMilestones.publishFailure({ failureStage: 'streaming_client_missing', @@ -502,16 +515,29 @@ export default class AnamClient { try { this.streamingClient.setMediaStreamTargetById(videoElementId); + this._isStreaming = true; this.streamingClient.startConnection(); } catch (error) { connectionMilestones.publishFailure({ failureStage: 'start_connection', ...getErrorMilestoneTags(error), }); + // Renderer initialization (notably WebGL capability detection) happens + // after the server has allocated a session. Do not leak that session when + // the local render target cannot be created. + await this.stopStreaming(); throw error; } } + /** + * Returns the SDK-managed alpha canvas when transparent background rendering + * is enabled and `streamToVideoElement` has installed its render target. + */ + public getTransparentBackgroundCanvas(): HTMLCanvasElement | null { + return this.streamingClient?.getTransparentBackgroundCanvas() ?? null; + } + /** * Send a talk command to make the persona speak the provided content. * @param content - The text content for the persona to speak diff --git a/src/lib/ClientMetrics.ts b/src/lib/ClientMetrics.ts index 1f5ebd8..50b9465 100644 --- a/src/lib/ClientMetrics.ts +++ b/src/lib/ClientMetrics.ts @@ -13,6 +13,7 @@ export enum ClientMetricMeasurement { CLIENT_METRIC_MEASUREMENT_SESSION_ATTEMPT = 'client_session_attempt', CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS = 'client_session_success', CLIENT_METRIC_MEASUREMENT_ICE_RESTART = 'client_ice_restart', + CLIENT_METRIC_MEASUREMENT_TRANSPARENT_RENDERER = 'client_transparent_renderer', } const CLIENT_METRICS_MAX_BATCH_SIZE = 50; diff --git a/src/modules/PackedAlphaTransport.ts b/src/modules/PackedAlphaTransport.ts new file mode 100644 index 0000000..0e7a9bb --- /dev/null +++ b/src/modules/PackedAlphaTransport.ts @@ -0,0 +1,149 @@ +import { StartSessionOptions } from '../types/coreApi/StartSessionOptions'; +import { + PACKED_ALPHA_CPU_TRANSPORT, + PACKED_ALPHA_TRANSPORT, + TransparentBackgroundTransport, +} from '../types/TransparentBackgroundTransport'; + +const PACKED_ALPHA_VIDEO_CONFIGURATION = { + // Match the RTP format we actually negotiate. This is also the form used by + // the Media Capabilities specification's WebRTC H.264 example. + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + bitrate: 2_500_000, + framerate: 25, +} as const; + +const PACKED_ALPHA_DECODING_CONFIGURATIONS: readonly MediaDecodingConfiguration[] = + [ + { + type: 'webrtc', + video: { + ...PACKED_ALPHA_VIDEO_CONFIGURATION, + width: 1152, + height: 1536, + }, + }, + { + type: 'webrtc', + video: { + ...PACKED_ALPHA_VIDEO_CONFIGURATION, + width: 1536, + height: 1152, + }, + }, + ]; + +type MediaCapabilitiesLike = Pick; + +/** @internal */ +export type PackedAlphaCapability = 'supported' | 'unsupported' | 'unknown'; + +/** + * Query support for both orientations of the H.264 Main Level 4.0 packed-alpha + * stream. MediaCapabilities results can be orientation-sensitive, and the SDK + * does not know the token's requested output dimensions at this point, so both + * landscape and portrait carriers must independently report supported+smooth. + * Missing or inconclusive MediaCapabilities implementations are reported as + * unknown. Callers conservatively retain the legacy carrier in that case: the + * server requires an explicit Main-Level-4 offer, so guessing support could + * otherwise turn a graceful quality fallback into a failed session. + * + * @internal + */ +export async function detectPackedAlphaCapability( + mediaCapabilities: MediaCapabilitiesLike | undefined = typeof navigator === + 'undefined' + ? undefined + : navigator.mediaCapabilities, +): Promise { + if (!mediaCapabilities?.decodingInfo) return 'unknown'; + + try { + const results = await Promise.all( + PACKED_ALPHA_DECODING_CONFIGURATIONS.map((configuration) => + mediaCapabilities.decodingInfo(configuration), + ), + ); + // The packed frame doubles the decoded pixel count. A decoder that can + // technically accept Main Level 4.0 but is not expected to sustain this + // configuration should use the lower-resolution compatibility path. + return results.every((result) => result.supported && result.smooth) + ? 'supported' + : 'unsupported'; + } catch { + // Some browsers expose MediaCapabilities but reject WebRTC configurations. + // Treat this as inconclusive; the caller keeps the compatibility path. + return 'unknown'; + } +} + +/** @internal */ +export async function buildTransparentBackgroundSessionOptions( + enabled: boolean | undefined, + mediaCapabilities?: MediaCapabilitiesLike, +): Promise< + Pick< + StartSessionOptions, + 'transparentBackground' | 'transparentBackgroundTransport' + > +> { + if (enabled === undefined) return {}; + if (!enabled) return { transparentBackground: false }; + + const capability = await detectPackedAlphaCapability(mediaCapabilities); + if (capability !== 'supported') { + console.warn( + 'Packed transparent-background video support could not be confirmed as smooth on this device; falling back to legacy adaptive chroma keying.', + ); + return { transparentBackground: true }; + } + + return { + transparentBackground: true, + transparentBackgroundTransport: PACKED_ALPHA_CPU_TRANSPORT, + }; +} + +/** + * Raise only H.264 Main Level 3.1 payloads to Level 4.0. Other H.264 profiles, + * codecs and already-higher levels are deliberately left untouched. + * + * @internal + */ +export function promotePackedAlphaH264Level(sdp: string): string { + const h264PayloadTypes = new Set(); + for (const line of sdp.split(/\r?\n/)) { + const match = line.match(/^a=rtpmap:(\d+)\s+H264\/90000(?:\s|$)/i); + if (match) h264PayloadTypes.add(match[1]); + } + + return sdp.replace( + /^a=fmtp:(\d+)(\s+)([^\r\n]*)$/gim, + (line, payloadType: string, separator: string, parameters: string) => { + if (!h264PayloadTypes.has(payloadType)) return line; + + const promotedParameters = parameters.replace( + /(^|;)(\s*profile-level-id\s*=\s*)4d001f(?=\s*(?:;|$))/i, + (_match, boundary: string, prefix: string) => + `${boundary}${prefix}4d0028`, + ); + return `a=fmtp:${payloadType}${separator}${promotedParameters}`; + }, + ); +} + +/** @internal */ +export function prepareOfferForTransparentBackgroundTransport( + offer: RTCSessionDescriptionInit, + transport: TransparentBackgroundTransport | undefined, +): RTCSessionDescriptionInit { + if ( + transport !== PACKED_ALPHA_TRANSPORT && + transport !== PACKED_ALPHA_CPU_TRANSPORT + ) { + return offer; + } + if (!offer.sdp) return offer; + return { ...offer, sdp: promotePackedAlphaH264Level(offer.sdp) }; +} diff --git a/src/modules/StreamingClient.ts b/src/modules/StreamingClient.ts index 967d77d..1281a8c 100644 --- a/src/modules/StreamingClient.ts +++ b/src/modules/StreamingClient.ts @@ -27,6 +27,7 @@ import { WebRtcPersonaConfigUpdateAppliedEvent, WebRtcTextMessageEvent, WebRtcReasoningTextMessageEvent, + TransparentBackgroundOptions, } from '../types'; import { AgentAudioInputStream } from '../types/AgentAudioInputStream'; import { ToolCallResultReceivedPayload } from '../types/toolCalling/ToolCallPayload'; @@ -38,6 +39,9 @@ import { WebRtcToolCallStartedEvent, } from '../types/streaming/WebRtcToolCallEvent'; import { ToolCallManager } from './ToolCallManager'; +import { TransparentBackgroundRenderer } from './TransparentBackgroundRenderer'; +import { prepareOfferForTransparentBackgroundTransport } from './PackedAlphaTransport'; +import { TransparentBackgroundTransport } from '../types/TransparentBackgroundTransport'; const SUCCESS_METRIC_POLLING_TIMEOUT_MS = 15000; // After this time we will stop polling for the first frame and consider the session a failure. const STATS_COLLECTION_INTERVAL_MS = 5000; @@ -82,6 +86,15 @@ export class StreamingClient { private inputAudioStream: MediaStream | null = null; private dataChannel: RTCDataChannel | null = null; private videoElement: HTMLVideoElement | null = null; + private transparentBackgroundRenderer: TransparentBackgroundRenderer | null = + null; + private readonly transparentBackgroundEnabled: boolean; + private readonly transparentBackgroundKeyOptions: + | TransparentBackgroundOptions + | undefined; + private readonly transparentBackgroundTransport: + | TransparentBackgroundTransport + | undefined; private videoStream: MediaStream | null = null; private audioStream: MediaStream | null = null; private inputAudioState: InputAudioState = { @@ -116,6 +129,12 @@ export class StreamingClient { this.toolCallManager = toolCallManager; this.connectionMilestones = connectionMilestones; this.apiGatewayConfig = options.apiGateway; + this.transparentBackgroundEnabled = + options.transparentBackground?.enabled === true; + this.transparentBackgroundKeyOptions = + options.transparentBackground?.keyOptions; + this.transparentBackgroundTransport = + options.transparentBackground?.transport; // initialize input audio state const { inputAudio } = options; this.inputAudioState = inputAudio.inputAudioState; @@ -479,10 +498,27 @@ export class StreamingClient { `StreamingClient: video element with id ${videoElementId} not found`, ); } - this.videoElement = videoElement as HTMLVideoElement; + if (!(videoElement instanceof HTMLVideoElement)) { + throw new Error( + `StreamingClient: element ${videoElementId} must be a video element`, + ); + } + this.videoElement = videoElement; + if (this.transparentBackgroundEnabled) { + this.transparentBackgroundRenderer?.destroy(); + this.transparentBackgroundRenderer = new TransparentBackgroundRenderer( + this.videoElement, + this.transparentBackgroundKeyOptions, + this.transparentBackgroundTransport, + ); + } } } + public getTransparentBackgroundCanvas(): HTMLCanvasElement | null { + return this.transparentBackgroundRenderer?.getCanvas() ?? null; + } + public startConnection() { try { if (this.peerConnection) { @@ -1060,7 +1096,10 @@ export class StreamingClient { this.connectionReceivedAnswer = false; this.remoteIceCandidateBuffer = []; - const offer = await this.peerConnection.createOffer({ iceRestart: true }); + const offer = prepareOfferForTransparentBackgroundTransport( + await this.peerConnection.createOffer({ iceRestart: true }), + this.transparentBackgroundTransport, + ); // Buffer local candidates that gather from setLocalDescription until the // re-offer is sent. The engine queues remote candidates only while it has // no remote description; during a restart it still holds the OLD ICE @@ -1206,12 +1245,18 @@ export class StreamingClient { ); if (this.videoElement) { this.videoElement.srcObject = this.videoStream; - const handle = this.videoElement.requestVideoFrameCallback(() => { - // unregister the callback after the first frame - this.videoElement?.cancelVideoFrameCallback(handle); + this.transparentBackgroundRenderer?.start(); + const onFirstFrame = () => { this.publicEventEmitter.emit(AnamEvent.VIDEO_PLAY_STARTED); this.recordSessionSuccess('videoElement'); - }); + }; + if (this.videoElement.requestVideoFrameCallback) { + this.videoElement.requestVideoFrameCallback(onFirstFrame); + } else { + this.videoElement.addEventListener('loadeddata', onFirstFrame, { + once: true, + }); + } } } else if (event.track.kind === 'audio') { this.connectionMilestones?.record('audio_track_received'); @@ -1515,7 +1560,10 @@ export class StreamingClient { try { this.connectionMilestones?.record('offer_creation_started'); const offer: RTCSessionDescriptionInit = - await this.peerConnection.createOffer(); + prepareOfferForTransparentBackgroundTransport( + await this.peerConnection.createOffer(), + this.transparentBackgroundTransport, + ); this.connectionMilestones?.record('offer_creation_completed'); await this.peerConnection.setLocalDescription(offer); this.connectionMilestones?.record('local_description_set'); @@ -1569,6 +1617,12 @@ export class StreamingClient { } this.successMetricFired = false; + this.transparentBackgroundRenderer?.destroy(); + this.transparentBackgroundRenderer = null; + if (this.videoElement) { + this.videoElement.srcObject = null; + } + // stop the input audio stream try { if (this.inputAudioStream) { diff --git a/src/modules/TransparentBackgroundRenderer.ts b/src/modules/TransparentBackgroundRenderer.ts new file mode 100644 index 0000000..e0e724d --- /dev/null +++ b/src/modules/TransparentBackgroundRenderer.ts @@ -0,0 +1,1246 @@ +import { + ClientMetricMeasurement, + sendClientMetric, +} from '../lib/ClientMetrics'; +import { TransparentBackgroundOptions } from '../types/TransparentBackgroundOptions'; +import { + PACKED_ALPHA_CPU_TRANSPORT, + PACKED_ALPHA_TRANSPORT, + TransparentBackgroundTransport, +} from '../types/TransparentBackgroundTransport'; + +// Calibrated on whole-subject fit/validation/test splits across raw, H.264 +// 520 kbps, and H.264 2.42 Mbps stages. RGB distance preserves fractional +// hair better than the previous chroma-only key for the dark motion-safe +// carriers. The guarded colour recovery avoids amplifying codec hue errors. +const DEFAULT_SIMILARITY = 0.01; +const DEFAULT_SMOOTHNESS = 0.44; +const DEFAULT_SPILL = 0.5; +// H.264/JPEG ringing leaves a very small non-zero alpha tail in otherwise +// uniform background pixels. Keeping it produces a faint grey/coloured veil +// over the page. The five-person transport holdout put the live p99.9 tail at +// ~0.012; 0.02 clears it while changing edge error by only 0.2%. +const BACKGROUND_ALPHA_FLOOR = 0.02; +// Some browser hardware-decode paths expose limited-range video black to the +// WebGL texture as roughly 12-16/255 instead of zero. Packed alpha transports +// that decoder pedestal literally, painting a pale rectangle over the page +// even though the server matte is zero. Clear only that sub-visible pedestal; +// unlike feathering this leaves every alpha value above the floor unchanged. +const PACKED_ALPHA_BACKGROUND_FLOOR = 0.07; +const RECOVERY_BLEND_START_ALPHA = 0.12; +const RECOVERY_BLEND_END_ALPHA = 0.82; +const SPILL_GUARD_START_ALPHA = 0.9; +const SPILL_GUARD_END_ALPHA = 0.995; +const TELEMETRY_FRAME_INTERVAL = 250; +const LEGACY_CARRIER_SAMPLE_SIZE = 64; +const LEGACY_CARRIER_MATCH_TOLERANCE = 48; +const LEGACY_CARRIER_MAX_DETECTION_ATTEMPTS = 3; + +export interface LegacyChromaCarrier { + name: 'green' | 'blue'; + rgb: readonly [number, number, number]; + axis: readonly [number, number, number]; + matchedSamples: number; +} + +const LEGACY_GREEN_CARRIER: LegacyChromaCarrier = { + name: 'green', + rgb: [0, 122 / 255, 51 / 255], + axis: [0, 1, 0], + matchedSamples: 0, +}; + +const LEGACY_CARRIER_CANDIDATES: readonly LegacyChromaCarrier[] = [ + LEGACY_GREEN_CARRIER, + { + name: 'blue', + rgb: [0, 71 / 255, 187 / 255], + axis: [0, 0, 1], + matchedSamples: 0, + }, + { + name: 'green', + rgb: [0, 1, 0], + axis: [0, 1, 0], + matchedSamples: 0, + }, +]; + +const VERTEX_SHADER_SOURCE = ` +attribute vec2 a_position; +varying vec2 v_texCoord; + +void main() { + gl_Position = vec4(a_position, 0.0, 1.0); + v_texCoord = (a_position + 1.0) * 0.5; +} +`; + +// Avatar creation selects green or blue to avoid foreground colour collisions; +// a one-time decoded-border sample chooses the corresponding uniforms. The +// distance and colour-recovery constants mirror the held-out server key. +/** @internal */ +export const FRAGMENT_SHADER_SOURCE = ` +precision mediump float; + +uniform sampler2D u_frame; +uniform float u_similarity; +uniform float u_smoothness; +uniform float u_spill; +uniform vec3 u_carrierColor; +uniform vec3 u_carrierAxis; +varying vec2 v_texCoord; + +void main() { + vec3 rgb = texture2D(u_frame, v_texCoord).rgb; + float carrierDistance = distance(rgb, u_carrierColor); + float alpha = smoothstep( + u_similarity, + u_similarity + max(u_smoothness, 0.0001), + carrierDistance + ); + alpha *= step(${BACKGROUND_ALPHA_FLOOR.toFixed(3)}, alpha); + + vec3 subtracted = clamp( + rgb - (1.0 - alpha) * u_carrierColor, + vec3(0.0), + vec3(alpha) + ); + vec3 observed = rgb * alpha; + float recoveryBlend = smoothstep( + ${RECOVERY_BLEND_START_ALPHA.toFixed(3)}, + ${RECOVERY_BLEND_END_ALPHA.toFixed(3)}, + alpha + ); + vec3 premultiplied = mix(observed, subtracted, recoveryBlend); + float spillGuard = 1.0 - smoothstep( + ${SPILL_GUARD_START_ALPHA.toFixed(3)}, + ${SPILL_GUARD_END_ALPHA.toFixed(3)}, + alpha + ); + float carrierValue = dot(premultiplied, u_carrierAxis); + vec3 otherChannels = premultiplied * (vec3(1.0) - u_carrierAxis); + float carrierExcess = max( + carrierValue - max(otherChannels.r, max(otherChannels.g, otherChannels.b)), + 0.0 + ); + premultiplied = max( + premultiplied - u_carrierAxis * carrierExcess * u_spill * spillGuard, + vec3(0.0) + ); + + gl_FragColor = vec4(premultiplied, alpha); +} +`; + +// Both packed transports carry already-premultiplied colour and a grayscale +// alpha plane. Landscape stacks the planes vertically; portrait places them +// side-by-side so neither wire dimension exceeds WebGL 1's guaranteed 2048 +// texture limit. V1 keys pre-JPEG in M2F; v2 is the engine-CPU post-JPEG +// control. The client reconstruction is otherwise identical. +/** @internal */ +export const PACKED_ALPHA_FRAGMENT_SHADER_SOURCE = ` +precision mediump float; + +uniform sampler2D u_frame; +uniform float u_horizontalLayout; +varying vec2 v_texCoord; + +float unpackAlpha(vec2 outputCoord) { + vec2 verticalAlphaCoord = vec2( + outputCoord.x, + outputCoord.y * 0.5 + ); + vec2 horizontalAlphaCoord = vec2( + 0.5 + outputCoord.x * 0.5, + outputCoord.y + ); + vec2 alphaCoord = mix( + verticalAlphaCoord, + horizontalAlphaCoord, + u_horizontalLayout + ); + vec3 alphaRgb = texture2D(u_frame, alphaCoord).rgb; + float alpha = dot(alphaRgb, vec3(0.2126, 0.7152, 0.0722)); + return alpha * step(${PACKED_ALPHA_BACKGROUND_FLOOR.toFixed(3)}, alpha); +} + +void main() { + vec2 verticalColourCoord = vec2( + v_texCoord.x, + 0.5 + v_texCoord.y * 0.5 + ); + vec2 horizontalColourCoord = vec2( + v_texCoord.x * 0.5, + v_texCoord.y + ); + vec2 colourCoord = mix( + verticalColourCoord, + horizontalColourCoord, + u_horizontalLayout + ); + vec3 premultiplied = texture2D(u_frame, colourCoord).rgb; + float alpha = unpackAlpha(v_texCoord); + + // Compression can make an individual premultiplied colour channel exceed + // alpha by a code value. The engine has already applied the canonical matte + // feather, so the client only clears the decoded-video black pedestal and + // clamps that invalid premultiplied state. + gl_FragColor = vec4(min(premultiplied, vec3(alpha)), alpha); +} +`; + +type OptionalVideoFrameCallbacks = { + requestVideoFrameCallback?: ( + callback: (now: DOMHighResTimeStamp) => void, + ) => number; + cancelVideoFrameCallback?: (handle: number) => void; +}; + +interface ResolvedKeyOptions { + similarity: number; + smoothness: number; + spill: number; +} + +interface GlResources { + positionBuffer: WebGLBuffer; + texture: WebGLTexture; + legacyProgram: WebGLProgram; + legacyPositionLocation: number; + similarityLocation: WebGLUniformLocation; + smoothnessLocation: WebGLUniformLocation; + spillLocation: WebGLUniformLocation; + carrierColorLocation: WebGLUniformLocation; + carrierAxisLocation: WebGLUniformLocation; + packedAlphaProgram: WebGLProgram; + packedAlphaPositionLocation: number; + packedAlphaHorizontalLayoutLocation: WebGLUniformLocation; +} + +export type TransparentFrameMode = + | 'green-key-v1' + | 'packed-alpha-v1' + | 'packed-alpha-v2' + | 'unsupported'; + +export interface TransparentFrameGeometry { + mode: TransparentFrameMode; + canvasWidth: number; + canvasHeight: number; + packedLayout?: 'vertical' | 'horizontal'; + reason?: string; +} + +export interface TransparentRendererDiagnostics { + framesRendered: number; + averageSubmissionMs: number; + maxSubmissionMs: number; +} + +export class TransparentBackgroundRenderer { + private readonly video: HTMLVideoElement; + private readonly canvas: HTMLCanvasElement; + private readonly parent: HTMLElement; + private readonly keyOptions: ResolvedKeyOptions; + private readonly transport: TransparentBackgroundTransport | undefined; + private readonly gl: WebGLRenderingContext; + private readonly maxTextureSize: number; + private resources: GlResources; + private resizeObserver: ResizeObserver | null = null; + private videoFrameCallbackHandle: number | null = null; + private animationFrameHandle: number | null = null; + private destroyed = false; + private started = false; + private contextLost = false; + private renderErrorReported = false; + private frameCount = 0; + private submissionTimeTotalMs = 0; + private submissionTimeMaxMs = 0; + private readonly originalVideoOpacity: string; + private readonly originalParentPosition: string; + private changedParentPosition = false; + private lastGeometrySignature: string | null = null; + private legacyCarrier: LegacyChromaCarrier = LEGACY_GREEN_CARRIER; + private legacyCarrierDetected = false; + private legacyCarrierDetectionAttempts = 0; + + constructor( + video: HTMLVideoElement, + options?: TransparentBackgroundOptions, + transport?: TransparentBackgroundTransport, + ) { + if (!video.parentElement) { + this.report('initialization_failed', 1, { reason: 'missing_parent' }); + throw new Error( + 'Transparent background requires the target video element to be attached to the DOM.', + ); + } + + this.video = video; + this.parent = video.parentElement; + this.keyOptions = resolveKeyOptions(options); + this.transport = transport; + this.originalVideoOpacity = video.style.opacity; + this.originalParentPosition = this.parent.style.position; + + this.canvas = document.createElement('canvas'); + this.canvas.setAttribute('aria-hidden', 'true'); + this.canvas.dataset.anamTransparentBackground = 'true'; + if (video.id) { + this.canvas.id = `${video.id}--anam-transparent`; + } + + const gl = this.canvas.getContext('webgl', { + alpha: true, + antialias: false, + depth: false, + stencil: false, + premultipliedAlpha: true, + preserveDrawingBuffer: false, + powerPreference: 'high-performance', + }); + if (!gl) { + this.report('initialization_failed', 1, { + reason: 'webgl_unavailable', + }); + throw new Error( + 'Transparent background is unavailable because WebGL could not be initialized on this device.', + ); + } + this.gl = gl; + this.maxTextureSize = gl.getParameter(gl.MAX_TEXTURE_SIZE) as number; + this.resources = this.createGlResources(); + + this.onContextLost = this.onContextLost.bind(this); + this.onContextRestored = this.onContextRestored.bind(this); + this.syncOverlayGeometry = this.syncOverlayGeometry.bind(this); + this.canvas.addEventListener('webglcontextlost', this.onContextLost); + this.canvas.addEventListener( + 'webglcontextrestored', + this.onContextRestored, + ); + + this.installOverlay(); + this.report('initialized', 1, { + renderer: 'webgl1', + requestedTransport: this.transport ?? 'green-key-v1', + maxTextureSize: this.maxTextureSize, + scheduling: getFrameCallbackApi(this.video).requestVideoFrameCallback + ? 'rvfc' + : 'raf', + }); + } + + public start(): void { + if (this.destroyed || this.started) return; + this.started = true; + this.scheduleNextFrame(); + } + + public getCanvas(): HTMLCanvasElement { + return this.canvas; + } + + public getDiagnostics(): TransparentRendererDiagnostics { + return { + framesRendered: this.frameCount, + averageSubmissionMs: + this.frameCount === 0 + ? 0 + : roundMetric(this.submissionTimeTotalMs / this.frameCount), + maxSubmissionMs: roundMetric(this.submissionTimeMaxMs), + }; + } + + public destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + + const frameCallbackApi = getFrameCallbackApi(this.video); + if ( + this.videoFrameCallbackHandle !== null && + frameCallbackApi.cancelVideoFrameCallback + ) { + frameCallbackApi.cancelVideoFrameCallback(this.videoFrameCallbackHandle); + } + if (this.animationFrameHandle !== null) { + cancelAnimationFrame(this.animationFrameHandle); + } + this.videoFrameCallbackHandle = null; + this.animationFrameHandle = null; + + this.resizeObserver?.disconnect(); + this.resizeObserver = null; + window.removeEventListener('resize', this.syncOverlayGeometry); + this.canvas.removeEventListener('webglcontextlost', this.onContextLost); + this.canvas.removeEventListener( + 'webglcontextrestored', + this.onContextRestored, + ); + this.deleteGlResources(this.resources); + this.canvas.remove(); + this.video.style.opacity = this.originalVideoOpacity; + if ( + this.changedParentPosition && + this.parent.style.position === 'relative' + ) { + this.parent.style.position = this.originalParentPosition; + } + + if (this.frameCount > 0) { + this.report( + 'renderer_summary', + this.submissionTimeTotalMs / this.frameCount, + { + frames: this.frameCount, + maxSubmissionMs: roundMetric(this.submissionTimeMaxMs), + }, + ); + } + } + + private installOverlay(): void { + const parentStyle = window.getComputedStyle(this.parent); + if (parentStyle.position === 'static') { + this.parent.style.position = 'relative'; + this.changedParentPosition = true; + } + + Object.assign(this.canvas.style, { + position: 'absolute', + pointerEvents: 'none', + background: 'transparent', + }); + // Keep the canvas immediately above the source video in DOM paint order. + // Appending it to the parent would put it above later sibling UI (for + // example call controls) unless every overlay supplied an explicit z-index. + this.video.insertAdjacentElement('afterend', this.canvas); + this.syncOverlayGeometry(); + + // Opacity keeps the source element active for media playback and iOS + // frame delivery while the transparent canvas supplies the visible pixels. + this.video.style.opacity = '0'; + + if (typeof ResizeObserver !== 'undefined') { + this.resizeObserver = new ResizeObserver(this.syncOverlayGeometry); + this.resizeObserver.observe(this.video); + this.resizeObserver.observe(this.parent); + } else { + window.addEventListener('resize', this.syncOverlayGeometry); + } + } + + private syncOverlayGeometry(): void { + if (this.destroyed) return; + const computedVideoStyle = window.getComputedStyle(this.video); + + // Keep the canvas's replaced-element box exactly on the video box. Its + // intrinsic bitmap is the reconstructed output plane (half the source + // height for packed alpha), so browser object-fit/object-position applies + // to the visible avatar rather than the vertically-stacked transport. + // Replaced content is clipped to its own content box, which prevents + // `cover` pixels escaping when the parent has visible overflow. Delegating + // positioning to CSS also preserves the full grammar (lengths, + // edge offsets, and calc()), rather than approximating it in JavaScript. + Object.assign(this.canvas.style, { + left: `${this.video.offsetLeft}px`, + top: `${this.video.offsetTop}px`, + width: `${this.video.offsetWidth}px`, + height: `${this.video.offsetHeight}px`, + objectFit: computedVideoStyle.objectFit, + objectPosition: computedVideoStyle.objectPosition, + borderRadius: computedVideoStyle.borderRadius, + clipPath: computedVideoStyle.clipPath, + transform: computedVideoStyle.transform, + transformOrigin: computedVideoStyle.transformOrigin, + zIndex: computedVideoStyle.zIndex, + }); + } + + private scheduleNextFrame(): void { + if (this.destroyed) return; + + const frameCallbackApi = getFrameCallbackApi(this.video); + if (frameCallbackApi.requestVideoFrameCallback) { + this.videoFrameCallbackHandle = + frameCallbackApi.requestVideoFrameCallback(() => { + this.videoFrameCallbackHandle = null; + this.drawFrame(); + this.scheduleNextFrame(); + }); + return; + } + + this.animationFrameHandle = requestAnimationFrame(() => { + this.animationFrameHandle = null; + this.drawFrame(); + this.scheduleNextFrame(); + }); + } + + private drawFrame(): void { + if ( + this.destroyed || + this.contextLost || + this.video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA || + this.video.videoWidth === 0 || + this.video.videoHeight === 0 + ) { + return; + } + + const startedAt = performance.now(); + const gl = this.gl; + try { + const geometry = resolveTransparentFrameGeometry( + this.video.videoWidth, + this.video.videoHeight, + this.transport, + this.maxTextureSize, + ); + this.applyFrameGeometry(geometry); + if (geometry.mode === 'unsupported') return; + + if ( + this.canvas.width !== geometry.canvasWidth || + this.canvas.height !== geometry.canvasHeight + ) { + // Deliberately use source pixels, not devicePixelRatio. A DPR-scaled + // backing canvas would add work without adding information. + this.canvas.width = geometry.canvasWidth; + this.canvas.height = geometry.canvasHeight; + } + + gl.viewport(0, 0, this.canvas.width, this.canvas.height); + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + const packedAlpha = + geometry.mode === PACKED_ALPHA_TRANSPORT || + geometry.mode === PACKED_ALPHA_CPU_TRANSPORT; + const program = packedAlpha + ? this.resources.packedAlphaProgram + : this.resources.legacyProgram; + const positionLocation = packedAlpha + ? this.resources.packedAlphaPositionLocation + : this.resources.legacyPositionLocation; + gl.useProgram(program); + gl.bindBuffer(gl.ARRAY_BUFFER, this.resources.positionBuffer); + gl.enableVertexAttribArray(positionLocation); + gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.resources.texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1); + gl.texImage2D( + gl.TEXTURE_2D, + 0, + gl.RGBA, + gl.RGBA, + gl.UNSIGNED_BYTE, + this.video, + ); + if (packedAlpha) { + gl.uniform1f( + this.resources.packedAlphaHorizontalLayoutLocation, + geometry.packedLayout === 'horizontal' ? 1 : 0, + ); + } else { + if (!this.legacyCarrierDetected) { + const candidate = this.detectLegacyCarrierFromVideo(); + this.legacyCarrierDetectionAttempts += 1; + if ( + shouldFinalizeLegacyCarrierDetection( + candidate, + this.legacyCarrierDetectionAttempts, + ) + ) { + this.legacyCarrier = candidate; + this.legacyCarrierDetected = true; + this.report('legacy_carrier_detected', 1, { + carrier: this.legacyCarrier.name, + matchedSamples: this.legacyCarrier.matchedSamples, + attempts: this.legacyCarrierDetectionAttempts, + }); + } + } + gl.uniform1f( + this.resources.similarityLocation, + this.keyOptions.similarity, + ); + gl.uniform1f( + this.resources.smoothnessLocation, + this.keyOptions.smoothness, + ); + gl.uniform1f(this.resources.spillLocation, this.keyOptions.spill); + gl.uniform3f( + this.resources.carrierColorLocation, + this.legacyCarrier.rgb[0], + this.legacyCarrier.rgb[1], + this.legacyCarrier.rgb[2], + ); + gl.uniform3f( + this.resources.carrierAxisLocation, + this.legacyCarrier.axis[0], + this.legacyCarrier.axis[1], + this.legacyCarrier.axis[2], + ); + } + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + } catch (error) { + if (!this.renderErrorReported) { + this.renderErrorReported = true; + console.warn( + 'Transparent background renderer failed to process a frame.', + error, + ); + this.report('render_error', 1, { + reason: error instanceof Error ? error.name : 'unknown', + }); + } + return; + } + + const submissionMs = performance.now() - startedAt; + this.frameCount += 1; + this.submissionTimeTotalMs += submissionMs; + this.submissionTimeMaxMs = Math.max(this.submissionTimeMaxMs, submissionMs); + if (this.frameCount === 1) { + this.report('first_frame', submissionMs); + } else if (this.frameCount % TELEMETRY_FRAME_INTERVAL === 0) { + this.report( + 'submission_sample', + this.submissionTimeTotalMs / this.frameCount, + { + frames: this.frameCount, + maxSubmissionMs: roundMetric(this.submissionTimeMaxMs), + }, + ); + } + } + + private detectLegacyCarrierFromVideo(): LegacyChromaCarrier { + try { + const sampleCanvas = document.createElement('canvas'); + sampleCanvas.width = LEGACY_CARRIER_SAMPLE_SIZE; + sampleCanvas.height = 4; + const context = sampleCanvas.getContext('2d', { + willReadFrequently: true, + }); + if (!context) return LEGACY_GREEN_CARRIER; + + const sourceWidth = this.video.videoWidth; + const sourceHeight = this.video.videoHeight; + const border = Math.max( + 1, + Math.floor(Math.min(sourceWidth, sourceHeight) / 64), + ); + context.drawImage( + this.video, + 0, + 0, + sourceWidth, + border, + 0, + 0, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + context.drawImage( + this.video, + 0, + sourceHeight - border, + sourceWidth, + border, + 0, + 1, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + context.drawImage( + this.video, + 0, + 0, + border, + sourceHeight, + 0, + 2, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + context.drawImage( + this.video, + sourceWidth - border, + 0, + border, + sourceHeight, + 0, + 3, + LEGACY_CARRIER_SAMPLE_SIZE, + 1, + ); + return detectLegacyChromaCarrier( + context.getImageData(0, 0, LEGACY_CARRIER_SAMPLE_SIZE, 4).data, + ); + } catch { + // WebRTC video is normally origin-clean. If a custom media source taints + // the sampling canvas, retain the historical green fallback. + return LEGACY_GREEN_CARRIER; + } + } + + private applyFrameGeometry(geometry: TransparentFrameGeometry): void { + const signature = `${geometry.mode}:${this.video.videoWidth}x${this.video.videoHeight}`; + if (signature === this.lastGeometrySignature) return; + this.lastGeometrySignature = signature; + + if (geometry.mode === 'unsupported') { + // Never hide the only usable pixels when a server or intermediary + // delivers an unexpected geometry. If a later frame returns to a valid + // geometry, the renderer automatically takes over again. + this.canvas.style.opacity = '0'; + this.video.style.opacity = this.originalVideoOpacity; + console.warn( + `Transparent background received unsupported video geometry ${this.video.videoWidth}x${this.video.videoHeight}; showing the source video unchanged.`, + ); + this.report('unsupported_geometry', 1, { + width: this.video.videoWidth, + height: this.video.videoHeight, + reason: geometry.reason ?? 'unknown', + }); + return; + } + + this.canvas.style.opacity = ''; + this.video.style.opacity = '0'; + if ( + (this.transport === PACKED_ALPHA_TRANSPORT || + this.transport === PACKED_ALPHA_CPU_TRANSPORT) && + geometry.mode === 'green-key-v1' + ) { + console.warn( + 'Packed transparent-background transport returned a legacy chroma-carrier frame; using the adaptive compatibility keyer.', + ); + this.report('transport_fallback', 1, { + deliveredTransport: 'green-key-v1', + width: this.video.videoWidth, + height: this.video.videoHeight, + }); + } else { + this.report('transport_selected', 1, { + deliveredTransport: geometry.mode, + width: this.video.videoWidth, + height: this.video.videoHeight, + }); + } + } + + private createGlResources(): GlResources { + const gl = this.gl; + let legacyProgram: WebGLProgram | null = null; + let packedAlphaProgram: WebGLProgram | null = null; + let positionBuffer: WebGLBuffer | null = null; + let texture: WebGLTexture | null = null; + + try { + legacyProgram = createProgram( + gl, + VERTEX_SHADER_SOURCE, + FRAGMENT_SHADER_SOURCE, + ); + packedAlphaProgram = createProgram( + gl, + VERTEX_SHADER_SOURCE, + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + ); + + positionBuffer = gl.createBuffer(); + texture = gl.createTexture(); + if (!positionBuffer || !texture) { + throw new Error('Unable to allocate transparent renderer resources.'); + } + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), + gl.STATIC_DRAW, + ); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + + const legacyPositionLocation = gl.getAttribLocation( + legacyProgram, + 'a_position', + ); + const packedAlphaPositionLocation = gl.getAttribLocation( + packedAlphaProgram, + 'a_position', + ); + const similarityLocation = gl.getUniformLocation( + legacyProgram, + 'u_similarity', + ); + const smoothnessLocation = gl.getUniformLocation( + legacyProgram, + 'u_smoothness', + ); + const spillLocation = gl.getUniformLocation(legacyProgram, 'u_spill'); + const carrierColorLocation = gl.getUniformLocation( + legacyProgram, + 'u_carrierColor', + ); + const carrierAxisLocation = gl.getUniformLocation( + legacyProgram, + 'u_carrierAxis', + ); + const packedAlphaHorizontalLayoutLocation = gl.getUniformLocation( + packedAlphaProgram, + 'u_horizontalLayout', + ); + if ( + legacyPositionLocation < 0 || + packedAlphaPositionLocation < 0 || + !similarityLocation || + !smoothnessLocation || + !spillLocation || + !carrierColorLocation || + !carrierAxisLocation || + !packedAlphaHorizontalLayoutLocation + ) { + throw new Error( + 'Unable to resolve transparent renderer shader inputs.', + ); + } + + return { + positionBuffer, + texture, + legacyProgram, + legacyPositionLocation, + similarityLocation, + smoothnessLocation, + spillLocation, + carrierColorLocation, + carrierAxisLocation, + packedAlphaProgram, + packedAlphaPositionLocation, + packedAlphaHorizontalLayoutLocation, + }; + } catch (error) { + if (positionBuffer) gl.deleteBuffer(positionBuffer); + if (texture) gl.deleteTexture(texture); + if (legacyProgram) gl.deleteProgram(legacyProgram); + if (packedAlphaProgram) gl.deleteProgram(packedAlphaProgram); + throw error; + } + } + + private deleteGlResources(resources: GlResources): void { + const gl = this.gl; + gl.deleteBuffer(resources.positionBuffer); + gl.deleteTexture(resources.texture); + gl.deleteProgram(resources.legacyProgram); + gl.deleteProgram(resources.packedAlphaProgram); + } + + private onContextLost(event: Event): void { + event.preventDefault(); + this.contextLost = true; + this.report('context_lost', 1); + } + + private onContextRestored(): void { + try { + this.resources = this.createGlResources(); + this.contextLost = false; + this.renderErrorReported = false; + this.report('context_restored', 1); + } catch (error) { + console.warn( + 'Transparent background WebGL context did not restore.', + error, + ); + this.report('context_restore_failed', 1); + } + } + + private report( + event: string, + value: number, + tags: Record = {}, + ): void { + void sendClientMetric( + ClientMetricMeasurement.CLIENT_METRIC_MEASUREMENT_TRANSPARENT_RENDERER, + roundMetric(value), + { event, ...tags }, + ); + } +} + +function createProgram( + gl: WebGLRenderingContext, + vertexSource: string, + fragmentSource: string, +): WebGLProgram { + const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexSource); + let fragmentShader: WebGLShader | null = null; + try { + fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource); + const program = gl.createProgram(); + if (!program) throw new Error('Unable to create WebGL program.'); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + const message = gl.getProgramInfoLog(program) ?? 'unknown link error'; + gl.deleteProgram(program); + throw new Error(`Unable to link transparent renderer: ${message}`); + } + return program; + } finally { + gl.deleteShader(vertexShader); + if (fragmentShader) gl.deleteShader(fragmentShader); + } +} + +function compileShader( + gl: WebGLRenderingContext, + type: number, + source: string, +): WebGLShader { + const shader = gl.createShader(type); + if (!shader) throw new Error('Unable to create WebGL shader.'); + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + const message = gl.getShaderInfoLog(shader) ?? 'unknown compile error'; + gl.deleteShader(shader); + throw new Error(`Unable to compile transparent renderer: ${message}`); + } + return shader; +} + +/** + * Resolve the decoded frame layout before uploading it to WebGL. Landscape + * packed frames are 3:4 (two 3:2 planes stacked vertically); portrait packed + * frames are 4:3 (two 2:3 planes placed horizontally). Ratio-based matching + * also permits a decoder to deliver a proportionally downscaled frame without + * silently sampling the wrong plane. + * + * @internal + */ +export function resolveTransparentFrameGeometry( + width: number, + height: number, + transport: TransparentBackgroundTransport | undefined, + maxTextureSize = 2048, +): TransparentFrameGeometry { + if ( + !Number.isInteger(width) || + !Number.isInteger(height) || + width <= 0 || + height <= 0 + ) { + return { + mode: 'unsupported', + canvasWidth: 0, + canvasHeight: 0, + reason: 'invalid_dimensions', + }; + } + if (width > maxTextureSize || height > maxTextureSize) { + return { + mode: 'unsupported', + canvasWidth: width, + canvasHeight: height, + reason: 'exceeds_webgl_texture_limit', + }; + } + + if ( + transport !== PACKED_ALPHA_TRANSPORT && + transport !== PACKED_ALPHA_CPU_TRANSPORT + ) { + return { + mode: 'green-key-v1', + canvasWidth: width, + canvasHeight: height, + }; + } + + if (height % 2 === 0 && height * 3 === width * 4) { + return { + mode: transport, + canvasWidth: width, + canvasHeight: height / 2, + packedLayout: 'vertical', + }; + } + if (width % 2 === 0 && width * 3 === height * 4) { + return { + mode: transport, + canvasWidth: width / 2, + canvasHeight: height, + packedLayout: 'horizontal', + }; + } + if (width * 2 === height * 3 || height * 2 === width * 3) { + return { + mode: 'green-key-v1', + canvasWidth: width, + canvasHeight: height, + }; + } + + return { + mode: 'unsupported', + canvasWidth: width, + canvasHeight: height, + reason: 'unexpected_packed_alpha_aspect_ratio', + }; +} + +/** @internal */ +export function resolveKeyOptions( + options?: TransparentBackgroundOptions, +): ResolvedKeyOptions { + return { + similarity: clampUnit(options?.similarity ?? DEFAULT_SIMILARITY), + smoothness: clampUnit(options?.smoothness ?? DEFAULT_SMOOTHNESS), + spill: clampUnit(options?.spill ?? DEFAULT_SPILL), + }; +} + +/** + * Pick one legacy carrier from a small decoded border sample. The server-side + * packed path never calls this; it exists only for the compatibility keyer + * used when packed H.264 support cannot be confirmed on a device. + * + * @internal + */ +export function detectLegacyChromaCarrier( + rgbaSamples: ArrayLike, +): LegacyChromaCarrier { + if (rgbaSamples.length < 4) return { ...LEGACY_GREEN_CARRIER }; + + let winner = LEGACY_GREEN_CARRIER; + let winnerMatches = 0; + let winnerMatchedDistance = Number.POSITIVE_INFINITY; + let winnerMatchedPixels: Array = []; + for (const candidate of LEGACY_CARRIER_CANDIDATES) { + const carrierBytes = candidate.rgb.map((channel) => channel * 255); + let matches = 0; + let matchedDistance = 0; + const matchedPixels: Array = []; + for (let offset = 0; offset + 2 < rgbaSamples.length; offset += 4) { + const redDelta = Math.abs(rgbaSamples[offset] - carrierBytes[0]); + const greenDelta = Math.abs(rgbaSamples[offset + 1] - carrierBytes[1]); + const blueDelta = Math.abs(rgbaSamples[offset + 2] - carrierBytes[2]); + if ( + Math.max(redDelta, greenDelta, blueDelta) <= + LEGACY_CARRIER_MATCH_TOLERANCE + ) { + matches += 1; + matchedPixels.push([ + rgbaSamples[offset], + rgbaSamples[offset + 1], + rgbaSamples[offset + 2], + ]); + matchedDistance += + redDelta * redDelta + greenDelta * greenDelta + blueDelta * blueDelta; + } + } + if ( + matches > winnerMatches || + (matches === winnerMatches && + matches > 0 && + matchedDistance < winnerMatchedDistance) + ) { + winner = candidate; + winnerMatches = matches; + winnerMatchedDistance = matchedDistance; + winnerMatchedPixels = matchedPixels; + } + } + + // Browser video decode is not colour-exact. In particular, limited-range + // H.264 paths can lift or shift every carrier pixel by enough that keying + // against the ideal server RGB leaves a faint, frame-shaped alpha veil. + // The matched border pixels are the carrier after that decode path. Their + // per-channel medians give us a robust local key colour while ignoring the + // foreground pixels that can touch one edge of the frame. + const decodedRgb = + winnerMatches > 0 + ? ([0, 1, 2].map((channel) => + medianByte(winnerMatchedPixels.map((pixel) => pixel[channel])), + ) as [number, number, number]) + : winner.rgb; + + return { + ...winner, + rgb: decodedRgb, + matchedSamples: winnerMatches, + }; +} + +function medianByte(values: readonly number[]): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + const median = + sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; + return clampUnit(median / 255); +} + +/** @internal */ +export function shouldFinalizeLegacyCarrierDetection( + carrier: LegacyChromaCarrier, + attempts: number, +): boolean { + return ( + carrier.matchedSamples > 0 || + attempts >= LEGACY_CARRIER_MAX_DETECTION_ATTEMPTS + ); +} + +/** + * CPU reference for the fragment shader's guarded colour recovery. Kept + * alongside the shader so focused tests can verify edge cases without a GPU + * dependency. + * + * @internal + */ +export function reconstructPremultipliedForeground( + rgb: readonly [number, number, number], + alphaValue: number, + spillValue = DEFAULT_SPILL, + carrierRgb: readonly [number, number, number] = LEGACY_GREEN_CARRIER.rgb, +): [number, number, number] { + const unclippedAlpha = clampUnit(alphaValue); + const alpha = unclippedAlpha < BACKGROUND_ALPHA_FLOOR ? 0 : unclippedAlpha; + const spill = clampUnit(spillValue); + const subtracted: [number, number, number] = [ + Math.min(alpha, Math.max(0, rgb[0] - (1 - alpha) * carrierRgb[0])), + Math.min(alpha, Math.max(0, rgb[1] - (1 - alpha) * carrierRgb[1])), + Math.min(alpha, Math.max(0, rgb[2] - (1 - alpha) * carrierRgb[2])), + ]; + const observed: [number, number, number] = [ + rgb[0] * alpha, + rgb[1] * alpha, + rgb[2] * alpha, + ]; + const recoveryBlend = smoothstep( + RECOVERY_BLEND_START_ALPHA, + RECOVERY_BLEND_END_ALPHA, + alpha, + ); + const premultiplied: [number, number, number] = [0, 1, 2].map( + (channel) => + observed[channel] * (1 - recoveryBlend) + + subtracted[channel] * recoveryBlend, + ) as [number, number, number]; + const spillGuard = + 1 - smoothstep(SPILL_GUARD_START_ALPHA, SPILL_GUARD_END_ALPHA, alpha); + const carrierChannel = + carrierRgb[2] > carrierRgb[1] && carrierRgb[2] > carrierRgb[0] + ? 2 + : carrierRgb[1] > carrierRgb[0] + ? 1 + : 0; + const otherChannels = [0, 1, 2].filter( + (channel) => channel !== carrierChannel, + ); + const carrierExcess = Math.max( + premultiplied[carrierChannel] - + Math.max( + premultiplied[otherChannels[0]], + premultiplied[otherChannels[1]], + ), + 0, + ); + premultiplied[carrierChannel] = Math.max( + premultiplied[carrierChannel] - carrierExcess * spill * spillGuard, + 0, + ); + return premultiplied; +} + +/** + * CPU reference for packed-alpha black-pedestal removal. + * + * @internal + */ +export function reconstructPackedAlphaPixel( + premultipliedRgb: readonly [number, number, number], + alphaValue: number, +): [number, number, number, number] { + const unclippedAlpha = clampUnit(alphaValue); + if (unclippedAlpha < PACKED_ALPHA_BACKGROUND_FLOOR) { + return [0, 0, 0, 0]; + } + + return [ + Math.min(clampUnit(premultipliedRgb[0]), unclippedAlpha), + Math.min(clampUnit(premultipliedRgb[1]), unclippedAlpha), + Math.min(clampUnit(premultipliedRgb[2]), unclippedAlpha), + unclippedAlpha, + ]; +} + +/** + * CPU reference for the complete legacy fragment shader. Inputs and output + * are normalized sRGB/premultiplied RGBA respectively. + * + * @internal + */ +export function keyLegacyPixel( + rgb: readonly [number, number, number], + carrierRgb: readonly [number, number, number] = LEGACY_GREEN_CARRIER.rgb, + options?: TransparentBackgroundOptions, +): [number, number, number, number] { + const resolved = resolveKeyOptions(options); + const distance = Math.sqrt( + (rgb[0] - carrierRgb[0]) ** 2 + + (rgb[1] - carrierRgb[1]) ** 2 + + (rgb[2] - carrierRgb[2]) ** 2, + ); + const unclippedAlpha = smoothstep( + resolved.similarity, + resolved.similarity + Math.max(resolved.smoothness, 0.0001), + distance, + ); + const alpha = unclippedAlpha < BACKGROUND_ALPHA_FLOOR ? 0 : unclippedAlpha; + const premultiplied = reconstructPremultipliedForeground( + rgb, + alpha, + resolved.spill, + carrierRgb, + ); + return [...premultiplied, alpha]; +} + +function clampUnit(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.min(1, Math.max(0, value)); +} + +function smoothstep(edge0: number, edge1: number, value: number): number { + const t = clampUnit((value - edge0) / (edge1 - edge0)); + return t * t * (3 - 2 * t); +} + +function roundMetric(value: number): number { + return Math.round(value * 1000) / 1000; +} + +function getFrameCallbackApi( + video: HTMLVideoElement, +): OptionalVideoFrameCallbacks { + // Older WebViews can lack rVFC even though it is present in the current DOM + // TypeScript definitions, so runtime detection is still required. + return video as unknown as OptionalVideoFrameCallbacks; +} diff --git a/src/types/AnamPublicClientOptions.ts b/src/types/AnamPublicClientOptions.ts index eb6c9cc..3e75e00 100644 --- a/src/types/AnamPublicClientOptions.ts +++ b/src/types/AnamPublicClientOptions.ts @@ -1,6 +1,7 @@ import { ApiOptions } from '../types'; import { ConnectionMilestoneMetricsOptions } from './ConnectionMilestoneMetricsOptions'; import { VoiceDetectionOptions } from './VoiceDetectionOptions'; +import { TransparentBackgroundOptions } from './TransparentBackgroundOptions'; export interface AnamPublicClientOptions { api?: ApiOptions; @@ -25,4 +26,17 @@ export interface AnamPublicClientOptions { * precedence over `rtcConfiguration.iceServers`. */ rtcConfiguration?: RTCConfiguration; + /** + * Request the avatar's transparent rendition and render it as a transparent + * WebGL canvas over the video element supplied to `streamToVideoElement`. + * + * The underlying MediaStream remains an ordinary opaque WebRTC video. Calls + * to `stream()` therefore return an opaque packed colour/matte video (or the + * legacy green-screen fallback); transparent pixels exist only in the + * SDK-managed canvas renderer. + * @default false + */ + transparentBackground?: boolean; + /** Optional tuning for the legacy green-screen compatibility keyer. */ + transparentBackgroundOptions?: TransparentBackgroundOptions; } diff --git a/src/types/TransparentBackgroundOptions.ts b/src/types/TransparentBackgroundOptions.ts new file mode 100644 index 0000000..2d4290b --- /dev/null +++ b/src/types/TransparentBackgroundOptions.ts @@ -0,0 +1,27 @@ +/** + * Client-side chroma-key controls used when `transparentBackground` is on. + * + * The defaults are tuned for Anam's generated motion-safe green/blue avatar + * renditions and mirror the server-side compatibility key. + * Most applications should not need to change these values. + */ +export interface TransparentBackgroundOptions { + /** + * Normalized RGB distance that is treated as fully transparent. Lower values + * keep more carrier-adjacent detail; higher values remove more backdrop. + * @default 0.01 + */ + similarity?: number; + /** + * Width of the soft transition around the key threshold. + * @default 0.44 + */ + smoothness?: number; + /** + * Strength of guarded carrier-channel spill suppression at semi-transparent + * edges. Set to `0` to disable despill or `1` for the full clamp. Opaque + * foreground colours are preserved. + * @default 0.5 + */ + spill?: number; +} diff --git a/src/types/TransparentBackgroundTransport.ts b/src/types/TransparentBackgroundTransport.ts new file mode 100644 index 0000000..475094c --- /dev/null +++ b/src/types/TransparentBackgroundTransport.ts @@ -0,0 +1,12 @@ +export const PACKED_ALPHA_TRANSPORT = 'packed-alpha-v1' as const; +/** @internal Engine-CPU cookbook-key control; carries the same premultiplied layout as v1. */ +export const PACKED_ALPHA_CPU_TRANSPORT = 'packed-alpha-v2' as const; +/** + * @deprecated V2 now carries premultiplied colour like v1; use PACKED_ALPHA_CPU_TRANSPORT. + * @internal + */ +export const PACKED_STRAIGHT_ALPHA_TRANSPORT = PACKED_ALPHA_CPU_TRANSPORT; + +export type TransparentBackgroundTransport = + | typeof PACKED_ALPHA_TRANSPORT + | typeof PACKED_ALPHA_CPU_TRANSPORT; diff --git a/src/types/coreApi/StartSessionOptions.ts b/src/types/coreApi/StartSessionOptions.ts index 69e500d..d4efdcb 100644 --- a/src/types/coreApi/StartSessionOptions.ts +++ b/src/types/coreApi/StartSessionOptions.ts @@ -1,5 +1,17 @@ import { VoiceDetectionOptions } from '../VoiceDetectionOptions'; +import { TransparentBackgroundTransport } from '../TransparentBackgroundTransport'; export interface StartSessionOptions { voiceDetection?: VoiceDetectionOptions; + /** + * Selects the avatar's private green-screen rendition for this session. + * The JavaScript SDK separately composites that rendition into a transparent + * canvas when streaming to a video element. + */ + transparentBackground?: boolean; + /** + * Selects the internal wire representation used for transparent video. + * @internal + */ + transparentBackgroundTransport?: TransparentBackgroundTransport; } diff --git a/src/types/index.ts b/src/types/index.ts index e16de16..4167c98 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,6 @@ export type { AnamClientOptions } from './AnamClientOptions'; export type { ConnectionMilestoneMetricsOptions } from './ConnectionMilestoneMetricsOptions'; +export type { TransparentBackgroundOptions } from './TransparentBackgroundOptions'; export type * from './signalling'; export { SignalMessageAction } from './signalling'; // need to export this explicitly to avoid enum import issues export type * from './streaming'; diff --git a/src/types/streaming/StreamingClientOptions.ts b/src/types/streaming/StreamingClientOptions.ts index 0ae8326..94a369d 100644 --- a/src/types/streaming/StreamingClientOptions.ts +++ b/src/types/streaming/StreamingClientOptions.ts @@ -2,6 +2,8 @@ import { SignallingClientOptions } from '../../types'; import { EngineApiRestClientOptions } from '../engineApi/EngineApiRestClientOptions'; import { InputAudioOptions } from './InputAudioOptions'; import { ApiGatewayConfig } from '../ApiGatewayConfig'; +import { TransparentBackgroundOptions } from '../TransparentBackgroundOptions'; +import { TransparentBackgroundTransport } from '../TransparentBackgroundTransport'; export interface StreamingClientOptions { engine: EngineApiRestClientOptions; @@ -11,6 +13,11 @@ export interface StreamingClientOptions { rtcConfiguration?: RTCConfiguration; inputAudio: InputAudioOptions; apiGateway?: ApiGatewayConfig; + transparentBackground?: { + enabled: boolean; + keyOptions?: TransparentBackgroundOptions; + transport?: TransparentBackgroundTransport; + }; metrics?: { showPeerConnectionStatsReport?: boolean; peerConnectionStatsReportOutputFormat?: 'console' | 'json'; diff --git a/test/packedAlphaTransportHarness.js b/test/packedAlphaTransportHarness.js new file mode 100644 index 0000000..cf89e99 --- /dev/null +++ b/test/packedAlphaTransportHarness.js @@ -0,0 +1,216 @@ +const assert = require('assert'); +const { + buildTransparentBackgroundSessionOptions, + detectPackedAlphaCapability, + prepareOfferForTransparentBackgroundTransport, + promotePackedAlphaH264Level, +} = require('../dist/main/modules/PackedAlphaTransport'); +const { + PACKED_ALPHA_CPU_TRANSPORT, + PACKED_ALPHA_TRANSPORT, + PACKED_STRAIGHT_ALPHA_TRANSPORT, +} = require('../dist/main/types/TransparentBackgroundTransport'); + +void (async () => { + assert.equal( + PACKED_STRAIGHT_ALPHA_TRANSPORT, + PACKED_ALPHA_CPU_TRANSPORT, + 'the deprecated packed-straight constant must remain a v2 compatibility alias', + ); + const decodingConfigurations = []; + const supportedMediaCapabilities = { + decodingInfo: async (configuration) => { + decodingConfigurations.push(configuration); + return { supported: true, smooth: true, powerEfficient: true }; + }, + }; + + assert.equal( + await detectPackedAlphaCapability(supportedMediaCapabilities), + 'supported', + ); + assert.deepEqual( + decodingConfigurations, + [ + { + type: 'webrtc', + video: { + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + width: 1152, + height: 1536, + bitrate: 2500000, + framerate: 25, + }, + }, + { + type: 'webrtc', + video: { + contentType: + 'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028', + width: 1536, + height: 1152, + bitrate: 2500000, + framerate: 25, + }, + }, + ], + 'capability detection must probe both packed wire orientations', + ); + assert.deepEqual( + await buildTransparentBackgroundSessionOptions( + true, + supportedMediaCapabilities, + ), + { + transparentBackground: true, + transparentBackgroundTransport: PACKED_ALPHA_CPU_TRANSPORT, + }, + 'supported devices must request the engine-CPU RGB-distance packed-alpha-v2 contract', + ); + + const originalWarn = console.warn; + let warning = ''; + console.warn = (message) => { + warning = message; + }; + try { + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(true, { + decodingInfo: async () => ({ + supported: false, + smooth: false, + powerEfficient: false, + }), + }), + { transparentBackground: true }, + 'an explicit unsupported result must retain the legacy adaptive chroma path', + ); + } finally { + console.warn = originalWarn; + } + assert.match(warning, /falling back to legacy adaptive chroma keying/); + + assert.equal( + await detectPackedAlphaCapability({ + decodingInfo: async () => ({ + supported: true, + smooth: false, + powerEfficient: false, + }), + }), + 'unsupported', + 'a supported decoder that cannot sustain the packed resolution must use the compatibility path', + ); + + const orientationSpecificProbes = []; + assert.equal( + await detectPackedAlphaCapability({ + decodingInfo: async (configuration) => { + orientationSpecificProbes.push(configuration); + return { + supported: configuration.video.width === 1152, + smooth: true, + powerEfficient: true, + }; + }, + }), + 'unsupported', + 'portrait decode support must not be inferred from landscape support', + ); + assert.equal( + orientationSpecificProbes.length, + 2, + 'both orientations must be probed even when their results differ', + ); + + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(true, { + decodingInfo: async () => { + throw new TypeError('WebRTC decodingInfo is not implemented'); + }, + }), + { transparentBackground: true }, + 'an inconclusive MediaCapabilities implementation must retain the compatibility path', + ); + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(true, undefined), + { transparentBackground: true }, + 'a browser without MediaCapabilities must retain the compatibility path', + ); + assert.deepEqual( + await buildTransparentBackgroundSessionOptions(false, { + decodingInfo: async () => { + throw new Error('must not be called'); + }, + }), + { transparentBackground: false }, + ); + + const ordinarySdp = [ + 'v=0', + 'm=video 9 UDP/TLS/RTP/SAVPF 96 97 98', + 'a=rtpmap:96 H264/90000', + 'a=fmtp:96 profile-level-id=4d001f;packetization-mode=1', + 'a=rtpmap:97 H264/90000', + 'a=fmtp:97 profile-level-id=42e01f;packetization-mode=1', + 'a=rtpmap:98 H264/90000', + 'a=fmtp:98 profile-level-id=4d002a;packetization-mode=1', + 'a=rtpmap:99 AV1/90000', + 'a=fmtp:99 profile-level-id=4d001f', + 'a=x-profile-level-id:4d001f', + '', + ].join('\r\n'); + const promotedSdp = promotePackedAlphaH264Level(ordinarySdp); + assert.match(promotedSdp, /profile-level-id=4d0028/); + assert.doesNotMatch( + promotedSdp, + /a=fmtp:96 profile-level-id=4d001f/, + 'the target H264 payload must no longer advertise Main Level 3.1', + ); + assert.match(promotedSdp, /profile-level-id=42e01f/); + assert.match(promotedSdp, /profile-level-id=4d002a/); + assert.match( + promotedSdp, + /a=fmtp:99 profile-level-id=4d001f/, + 'an fmtp parameter belonging to a non-H264 payload must not be rewritten', + ); + assert.match( + promotedSdp, + /a=x-profile-level-id:4d001f/, + 'an unrelated SDP attribute must not be rewritten', + ); + + const ordinaryOffer = { type: 'offer', sdp: ordinarySdp }; + assert.strictEqual( + prepareOfferForTransparentBackgroundTransport(ordinaryOffer, undefined), + ordinaryOffer, + 'ordinary sessions must not rewrite or clone their offer', + ); + const packedOffer = prepareOfferForTransparentBackgroundTransport( + ordinaryOffer, + PACKED_ALPHA_TRANSPORT, + ); + assert.notStrictEqual(packedOffer, ordinaryOffer); + assert.equal(packedOffer.type, 'offer'); + assert.equal(packedOffer.sdp, promotedSdp); + const packedCpuOffer = prepareOfferForTransparentBackgroundTransport( + ordinaryOffer, + PACKED_ALPHA_CPU_TRANSPORT, + ); + assert.equal( + packedCpuOffer.sdp, + promotedSdp, + 'the engine-CPU control uses the same packed H264 geometry', + ); + assert.equal( + ordinaryOffer.sdp, + ordinarySdp, + 'SDP rewriting must not mutate the browser-created offer', + ); + + console.log('packed alpha transport harness passed'); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/test/transparentBackgroundGeometryHarness.js b/test/transparentBackgroundGeometryHarness.js new file mode 100644 index 0000000..e4e35d1 --- /dev/null +++ b/test/transparentBackgroundGeometryHarness.js @@ -0,0 +1,257 @@ +const assert = require('assert'); +const { setClientMetricsDisabled } = require('../dist/main/lib/ClientMetrics'); +const { + resolveTransparentFrameGeometry, + TransparentBackgroundRenderer, +} = require('../dist/main/modules/TransparentBackgroundRenderer'); +const { + PACKED_ALPHA_CPU_TRANSPORT, + PACKED_ALPHA_TRANSPORT, +} = require('../dist/main/types/TransparentBackgroundTransport'); + +setClientMetricsDisabled(true); + +const createWebGlStub = (deletedResources) => ({ + VERTEX_SHADER: 1, + FRAGMENT_SHADER: 2, + COMPILE_STATUS: 3, + LINK_STATUS: 4, + ARRAY_BUFFER: 5, + STATIC_DRAW: 6, + TEXTURE_2D: 7, + TEXTURE_MIN_FILTER: 8, + TEXTURE_MAG_FILTER: 9, + LINEAR: 10, + TEXTURE_WRAP_S: 11, + TEXTURE_WRAP_T: 12, + CLAMP_TO_EDGE: 13, + MAX_TEXTURE_SIZE: 14, + createShader: () => ({}), + shaderSource: () => {}, + compileShader: () => {}, + getShaderParameter: () => true, + getShaderInfoLog: () => '', + deleteShader: () => {}, + createProgram: () => ({}), + attachShader: () => {}, + linkProgram: () => {}, + getProgramParameter: () => true, + getProgramInfoLog: () => '', + deleteProgram: (program) => deletedResources.programs.push(program), + createBuffer: () => ({}), + createTexture: () => ({}), + deleteBuffer: (buffer) => deletedResources.buffers.push(buffer), + deleteTexture: (texture) => deletedResources.textures.push(texture), + bindBuffer: () => {}, + bufferData: () => {}, + bindTexture: () => {}, + texParameteri: () => {}, + getAttribLocation: () => 0, + getUniformLocation: () => ({}), + getParameter: () => 2048, +}); + +const createRendererHarness = ({ objectFit, objectPosition }) => { + const deletedResources = { programs: [], buffers: [], textures: [] }; + const parent = { + style: { position: '' }, + }; + const canvas = { + style: {}, + dataset: {}, + id: '', + setAttribute: () => {}, + getContext: () => createWebGlStub(deletedResources), + addEventListener: () => {}, + removeEventListener: () => {}, + remove: () => {}, + }; + const video = { + parentElement: parent, + id: 'persona-video', + style: { opacity: '' }, + offsetLeft: 13, + offsetTop: 17, + offsetWidth: 100, + offsetHeight: 100, + videoWidth: 200, + videoHeight: 100, + insertAdjacentElement: (position, element) => { + assert.equal(position, 'afterend'); + assert.equal(element, canvas); + }, + }; + + global.document = { + createElement: (tagName) => { + assert.equal(tagName, 'canvas'); + return canvas; + }, + }; + global.window = { + getComputedStyle: (element) => + element === parent + ? { position: 'relative', overflow: 'visible' } + : { + objectFit, + objectPosition, + borderRadius: '12px', + clipPath: 'none', + transform: 'none', + transformOrigin: '50% 50%', + zIndex: 'auto', + }, + addEventListener: () => {}, + removeEventListener: () => {}, + }; + global.ResizeObserver = class { + observe() {} + disconnect() {} + }; + + const renderer = new TransparentBackgroundRenderer(video); + return { canvas, deletedResources, renderer }; +}; + +const cover = createRendererHarness({ + objectFit: 'cover', + objectPosition: '50% 50%', +}); +assert.deepEqual( + { + left: cover.canvas.style.left, + top: cover.canvas.style.top, + width: cover.canvas.style.width, + height: cover.canvas.style.height, + objectFit: cover.canvas.style.objectFit, + objectPosition: cover.canvas.style.objectPosition, + }, + { + left: '13px', + top: '17px', + width: '100px', + height: '100px', + objectFit: 'cover', + objectPosition: '50% 50%', + }, + 'cover must stay inside the video element box even when the parent overflow is visible', +); +cover.renderer.destroy(); +cover.renderer.destroy(); +assert.deepEqual( + { + programs: cover.deletedResources.programs.length, + buffers: cover.deletedResources.buffers.length, + textures: cover.deletedResources.textures.length, + }, + { programs: 2, buffers: 1, textures: 1 }, + 'destroy must release each shader program, buffer, and texture exactly once', +); + +assert.deepEqual( + resolveTransparentFrameGeometry(1152, 1536, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 1152, + canvasHeight: 768, + packedLayout: 'vertical', + }, + 'packed Cara 4 frames must expose a 1152x768 canvas', +); +assert.deepEqual( + resolveTransparentFrameGeometry(576, 768, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 576, + canvasHeight: 384, + packedLayout: 'vertical', + }, + 'proportionally downscaled packed frames must preserve the two-plane layout', +); +assert.deepEqual( + resolveTransparentFrameGeometry(1152, 1536, PACKED_ALPHA_CPU_TRANSPORT, 2048), + { + mode: 'packed-alpha-v2', + canvasWidth: 1152, + canvasHeight: 768, + packedLayout: 'vertical', + }, + 'the engine-CPU control must use the same premultiplied two-plane geometry', +); +assert.deepEqual( + resolveTransparentFrameGeometry(1536, 1152, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 768, + canvasHeight: 1152, + packedLayout: 'horizontal', + }, + 'packed Cara 4 portrait frames must expose a 768x1152 canvas', +); +assert.deepEqual( + resolveTransparentFrameGeometry(768, 576, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'packed-alpha-v1', + canvasWidth: 384, + canvasHeight: 576, + packedLayout: 'horizontal', + }, + 'the portrait half-resolution ABR rung must preserve the two-plane layout', +); +assert.deepEqual( + resolveTransparentFrameGeometry(1536, 1152, undefined, 2048), + { + mode: 'green-key-v1', + canvasWidth: 1536, + canvasHeight: 1152, + }, + 'ordinary 4:3 video must never be unpacked without a negotiated packed transport', +); +assert.deepEqual( + resolveTransparentFrameGeometry(1152, 768, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'green-key-v1', + canvasWidth: 1152, + canvasHeight: 768, + }, + 'a standard Cara 4 frame must select the legacy keyer compatibility path', +); +assert.deepEqual( + resolveTransparentFrameGeometry(768, 1152, PACKED_ALPHA_TRANSPORT, 2048), + { + mode: 'green-key-v1', + canvasWidth: 768, + canvasHeight: 1152, + }, + 'a standard portrait Cara 4 frame must select the legacy keyer compatibility path', +); +assert.equal( + resolveTransparentFrameGeometry(1280, 720, PACKED_ALPHA_TRANSPORT, 2048).mode, + 'unsupported', + 'unexpected packed transport geometry must not be sampled as two planes', +); +assert.equal( + resolveTransparentFrameGeometry(1152, 1536, PACKED_ALPHA_TRANSPORT, 1024) + .reason, + 'exceeds_webgl_texture_limit', + 'the runtime WebGL texture limit must be enforced', +); + +for (const objectPosition of [ + '10px 20px', + 'calc(100% - 12px) calc(50% + 4px)', + 'right 10px bottom 20px', +]) { + const harness = createRendererHarness({ + objectFit: 'contain', + objectPosition, + }); + assert.equal( + harness.canvas.style.objectPosition, + objectPosition, + `object-position must be delegated to CSS without rewriting ${objectPosition}`, + ); + harness.renderer.destroy(); +} + +console.log('transparent background geometry harness passed'); diff --git a/test/transparentBackgroundReconstructionHarness.js b/test/transparentBackgroundReconstructionHarness.js new file mode 100644 index 0000000..c61d2c9 --- /dev/null +++ b/test/transparentBackgroundReconstructionHarness.js @@ -0,0 +1,364 @@ +const assert = require('assert'); +const { + FRAGMENT_SHADER_SOURCE, + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + detectLegacyChromaCarrier, + keyLegacyPixel, + reconstructPackedAlphaPixel, + reconstructPremultipliedForeground, + resolveKeyOptions, + shouldFinalizeLegacyCarrierDetection, +} = require('../dist/main/modules/TransparentBackgroundRenderer'); + +const rgba = (pixels) => pixels.flatMap((pixel) => [...pixel, 255]); + +const closeTo = (actual, expected, message, epsilon = 1e-7) => { + assert.equal(actual.length, expected.length, message); + actual.forEach((value, index) => { + assert.ok( + Math.abs(value - expected[index]) <= epsilon, + `${message}: channel ${index} expected ${expected[index]}, got ${value}`, + ); + }); +}; + +assert.deepEqual( + resolveKeyOptions(), + { similarity: 0.01, smoothness: 0.44, spill: 0.5 }, + 'calibrated defaults must remain explicit and covered by regression tests', +); + +const detectedGreen = detectLegacyChromaCarrier( + rgba([ + [2, 120, 53], + [0, 122, 51], + [4, 124, 50], + [210, 170, 130], + ]), +); +assert.equal(detectedGreen.name, 'green'); +closeTo( + detectedGreen.rgb, + [2 / 255, 122 / 255, 51 / 255], + 'decoded dark-green border must calibrate the selected carrier', +); +assert.equal(detectedGreen.matchedSamples, 3); + +const detectedBlue = detectLegacyChromaCarrier( + rgba([ + [1, 72, 185], + [0, 71, 187], + [3, 69, 190], + [0, 220, 40], + ]), +); +assert.equal(detectedBlue.name, 'blue'); +closeTo( + detectedBlue.axis, + [0, 0, 1], + 'decoded blue border must select only the blue key channel', +); +assert.equal(detectedBlue.matchedSamples, 3); +closeTo( + detectedBlue.rgb, + [1 / 255, 71 / 255, 187 / 255], + 'decoded blue border must calibrate the selected carrier', +); + +const legacyBrightGreen = detectLegacyChromaCarrier( + rgba([ + [0, 253, 1], + [2, 255, 0], + ]), +); +closeTo( + legacyBrightGreen.rgb, + [1 / 255, 254 / 255, 0.5 / 255], + 'legacy pure-green avatars must calibrate around their decoded key colour', +); + +const liftedDecodedCarrier = detectLegacyChromaCarrier( + rgba([ + [13, 130, 62], + [15, 132, 64], + [14, 131, 63], + [210, 170, 130], + ]), +); +closeTo( + liftedDecodedCarrier.rgb, + [14 / 255, 131 / 255, 63 / 255], + 'limited-range decode lift must be absorbed into the local carrier key', +); +assert.ok( + keyLegacyPixel( + [14 / 255, 131 / 255, 63 / 255], + [0, 122 / 255, 51 / 255], + )[3] > 0.02, + 'the ideal server key reproduces the visible frame-shaped alpha veil', +); +closeTo( + keyLegacyPixel( + [14 / 255, 131 / 255, 63 / 255], + liftedDecodedCarrier.rgb, + ), + [0, 0, 0, 0], + 'the decoded local key must reconstruct the lifted carrier as transparent', +); + +const unmatchedCarrier = detectLegacyChromaCarrier( + rgba([ + [32, 128, 224], + [180, 120, 90], + ]), +); +assert.equal(unmatchedCarrier.name, 'green'); +assert.equal(unmatchedCarrier.matchedSamples, 0); +closeTo( + unmatchedCarrier.rgb, + [0, 122 / 255, 51 / 255], + 'an inconclusive sample must fail closed to the current green carrier', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(unmatchedCarrier, 1), + false, + 'an inconclusive first decoded frame must be retried', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(unmatchedCarrier, 2), + false, + 'an inconclusive second decoded frame must still be retried', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(unmatchedCarrier, 3), + true, + 'inconclusive or tainted sampling must stop after a bounded third attempt', +); +assert.equal( + shouldFinalizeLegacyCarrierDetection(detectedBlue, 1), + true, + 'a conclusive blue carrier must lock on the first valid frame', +); +assert.deepEqual( + resolveKeyOptions({ similarity: 0.1, smoothness: 0.2, spill: 0 }), + { similarity: 0.1, smoothness: 0.2, spill: 0 }, + 'all public key controls must remain configurable, including inverse-only spill=0', +); + +closeTo( + keyLegacyPixel([0, 122 / 255, 51 / 255]), + [0, 0, 0, 0], + 'the exact dark-green carrier must reconstruct to transparent black', +); + +closeTo( + keyLegacyPixel([0.02, 122 / 255, 51 / 255]), + [0, 0, 0, 0], + 'the measured codec-noise alpha tail must be floored to transparent', +); + +closeTo( + keyLegacyPixel([0.05, 122 / 255, 51 / 255]), + [ + 0.0011645379413974457, 0.007900590739676788, 0.004658151765589783, + 0.023290758827948913, + ], + 'a just-visible carrier deviation must match the held-out RGB key fixture', +); + +const darkGreen = [0, 122 / 255, 51 / 255]; +closeTo( + keyLegacyPixel([0.12, 0.42, 0.18], darkGreen), + [ + 0.026715424077240528, 0.057808537750836525, 0.03477855641001681, + 0.19615563544778447, + ], + 'green-carrier guarded recovery must match the held-out CPU fixture', +); + +const blueKey = [0, 71 / 255, 187 / 255]; +closeTo( + keyLegacyPixel([0.08, 0.35, 0.62], blueKey), + [ + 0.02658094751407277, 0.09548590370748197, 0.12334501296618385, + 0.257536997095212, + ], + 'blue-carrier guarded recovery must match the held-out CPU fixture', +); + +closeTo( + keyLegacyPixel([0.05, 0.95, 0.08], darkGreen), + [0.05, 0.95, 0.08, 1], + 'opaque genuine green must stay unchanged', +); + +closeTo( + reconstructPremultipliedForeground([0.01, 0.99, 0.01], 0.019), + [0, 0, 0], + 'the colour-recovery helper must apply the same background alpha floor', +); + +const guardedInput = [0.12, 0.42, 0.18]; +const guardedAlpha = 0.19615563544778447; +const withoutDespill = reconstructPremultipliedForeground( + guardedInput, + guardedAlpha, + 0, + darkGreen, +); +const withDespill = reconstructPremultipliedForeground( + guardedInput, + guardedAlpha, + 1, + darkGreen, +); +assert.ok( + withDespill[1] < withoutDespill[1], + 'despill must reduce only the carrier channel when it is excessive', +); +closeTo( + [withDespill[0], withDespill[2]], + [withoutDespill[0], withoutDespill[2]], + 'despill must preserve the non-carrier channels', +); + +const nearOpaqueGreen = [0.05, 0.9, 0.08]; +closeTo( + reconstructPremultipliedForeground(nearOpaqueGreen, 1, 1, darkGreen), + nearOpaqueGreen, + 'recovery and despill must preserve opaque foreground colour', +); + +for (const { rgb, alpha, spill, carrier } of [ + { rgb: [0.2, 0.7, 0.3], alpha: 0.3, spill: 0, carrier: darkGreen }, + { rgb: [0.8, 0.1, 1], alpha: 0.6, spill: 1, carrier: blueKey }, + { rgb: [0.5, 0.9, 0.2], alpha: 0.95, spill: 0.5, carrier: darkGreen }, +]) { + const reconstructed = reconstructPremultipliedForeground( + rgb, + alpha, + spill, + carrier, + ); + reconstructed.forEach((channel) => { + assert.ok(channel >= 0, 'premultiplied channels must not be negative'); + assert.ok( + channel <= alpha + 1e-7, + 'premultiplied channels must not exceed alpha', + ); + }); +} + +assert.match( + FRAGMENT_SHADER_SOURCE, + /distance\(rgb, u_carrierColor\)/, + 'shader must use normalized RGB distance from the detected carrier', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /rgb - \(1\.0 - alpha\) \* u_carrierColor/, + 'shader must compute carrier-subtracted colour recovery', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /vec3 observed = rgb \* alpha/, + 'shader must include observed-colour premultiplication', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /mix\(observed, subtracted, recoveryBlend\)/, + 'shader must guard carrier subtraction with the calibrated blend', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /smoothstep\(\s*0\.120,\s*0\.820,\s*alpha/, + 'shader must use the calibrated guarded-recovery alpha interval', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /premultiplied - u_carrierAxis \* carrierExcess/, + 'shader must despill only the selected green or blue channel', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /gl_FragColor = vec4\(premultiplied, alpha\)/, + 'shader must submit already-premultiplied RGB', +); +assert.match( + FRAGMENT_SHADER_SOURCE, + /alpha \*= step\(0\.020, alpha\)/, + 'shader must clear the measured codec-noise alpha tail', +); +assert.doesNotMatch( + FRAGMENT_SHADER_SOURCE, + /gl_FragColor = vec4\(rgb \* alpha, alpha\)/, + 'shader must not multiply the green carrier by alpha', +); + +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /0\.5 \+ v_texCoord\.y \* 0\.5/, + 'packed renderer must sample colour from the top half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /v_texCoord\.y \* 0\.5/, + 'packed renderer must sample alpha from the bottom half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /v_texCoord\.x \* 0\.5/, + 'packed renderer must sample portrait colour from the left half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /0\.5 \+ outputCoord\.x \* 0\.5/, + 'packed renderer must sample portrait alpha from the right half', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /u_horizontalLayout/, + 'packed renderer must select the orientation negotiated by frame geometry', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /gl_FragColor = vec4\(min\(premultiplied, vec3\(alpha\)\), alpha\)/, + 'packed renderer must submit the transported premultiplied colour and alpha', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /alpha \* step\(0\.070, alpha\)/, + 'packed renderer must clear the decoded-video black pedestal', +); +assert.match( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /float alpha = unpackAlpha\(v_texCoord\)/, + 'packed renderer must use the server-feathered transported alpha directly', +); +assert.doesNotMatch( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /u_outputTexel|min3|49\.0 \* b0|premultiplied \*=/, + 'packed renderer must not apply an additional client-side feather', +); +assert.doesNotMatch( + PACKED_ALPHA_FRAGMENT_SHADER_SOURCE, + /u_similarity|u_smoothness|u_spill|carrierDistance|recoveryBlend/, + 'packed renderer must not run the legacy key or despill operations', +); +closeTo( + reconstructPackedAlphaPixel([0.02, 0.03, 0.05], 16 / 255), + [0, 0, 0, 0], + 'decoded limited-range black must reconstruct to transparent black', +); +closeTo( + reconstructPackedAlphaPixel([0.03, 0.08, 0.2], 0.2), + [0.03, 0.08, 0.2, 0.2], + 'packed alpha above the pedestal must remain byte-for-byte unchanged', +); +closeTo( + reconstructPackedAlphaPixel([0.4, 0.1, 0.3], 0.25), + [0.25, 0.1, 0.25, 0.25], + 'invalid premultiplied channels must still clamp to alpha', +); + +console.log('transparent background reconstruction harness passed');