Add transparent background rendering

README.md

@@ -64,6 +64,35 @@ await anamClient.streamToVideoElement('video-element-id');
 
 This will start a new session using the pre-configured persona id and start streaming video element in the DOM with the matching element id.
 
+### Transparent avatar background
+
+Set `transparentBackground` when creating the client. The SDK requests a
+packed colour/matte rendition and reconstructs it through a source-resolution
+WebGL canvas over your video element. Landscape transport stacks colour over
+alpha; portrait transport places colour left of alpha. Browsers that cannot
+confirm smooth support for the packed H.264 stream, and servers that return an
+ordinary Cara 4 frame, automatically use the legacy green-screen keyer.
+
+```typescript
+const anamClient = createClient('your-session-token', {
+  transparentBackground: true,
+});
+
+await anamClient.streamToVideoElement('video-element-id');
+```
+
+The supplied video must be attached to the DOM and should use `autoplay` and
+`playsinline` as usual. The underlying WebRTC `MediaStream` is still ordinary
+opaque video: it contains packed colour and alpha planes, or a green-screen
+frame on the compatibility path. Transparency exists in the SDK-managed
+canvas only. As a result, `stream()` returns that raw transport,
+and video-only browser features such as native controls and picture-in-picture
+do not automatically capture the transparent canvas. Use
+`getTransparentBackgroundCanvas()` if you need the managed canvas element for
+layout or capture behavior. Session recordings and replays likewise contain
+the raw packed video: landscape is vertically packed and portrait is
+horizontally packed; they are not transparent RGBA recordings.
+
 To stop a session use the `stopStreaming` method.
 
 ```typescript

package.json

@@ -39,6 +39,7 @@
     "lint": "eslint \"{src,test}/**/*.ts\" --fix",
     "test": "echo \"Error: no test specified\" && exit 1",
     "test:connection-milestones": "npm run build:main && node test/connectionMilestonesHarness.js",
+    "test:transparent-background": "npm run build:main && node test/transparentBackgroundGeometryHarness.js && node test/transparentBackgroundReconstructionHarness.js && node test/packedAlphaTransportHarness.js",
     "watch": "nodemon -e ts --watch src --exec \"npm run build\"",
     "prepare": "npm run build"
   },

src/AnamClient.ts

@@ -37,6 +37,7 @@ import {
 import { AgentAudioInputStream } from './types/AgentAudioInputStream';
 import { TalkMessageStream } from './types/TalkMessageStream';
 import { ToolCallManager } from './modules/ToolCallManager';
+import { buildTransparentBackgroundSessionOptions } from './modules/PackedAlphaTransport';
 export default class AnamClient {
   private publicEventEmitter: PublicEventEmitter;
   private internalEventEmitter: InternalEventEmitter;
@@ -198,11 +199,19 @@ export default class AnamClient {
     return undefined;
   }
 
-  private buildStartSessionOptionsForClient(): StartSessionOptions | undefined {
+  private async buildStartSessionOptionsForClient(): Promise<
+    StartSessionOptions | undefined
+  > {
     const sessionOptions: StartSessionOptions = {};
     if (this.clientOptions?.voiceDetection) {
       sessionOptions.voiceDetection = this.clientOptions.voiceDetection;
     }
+    Object.assign(
+      sessionOptions,
+      await buildTransparentBackgroundSessionOptions(
+        this.clientOptions?.transparentBackground,
+      ),
+    );
     // return undefined if no options are set
     if (Object.keys(sessionOptions).length === 0) {
       return undefined;
@@ -245,7 +254,7 @@ export default class AnamClient {
     const config = this.personaConfig;
     // build session options from client options
     const sessionOptions: StartSessionOptions | undefined =
-      this.buildStartSessionOptionsForClient();
+      await this.buildStartSessionOptionsForClient();
     // start a new session
     connectionMilestones?.record('start_session_request_started');
     let response: StartSessionResponse;
@@ -321,6 +330,11 @@ export default class AnamClient {
             disableInputAudio: this.clientOptions?.disableInputAudio,
           },
           apiGateway: this.clientOptions?.api?.apiGateway,
+          transparentBackground: {
+            enabled: this.clientOptions?.transparentBackground === true,
+            keyOptions: this.clientOptions?.transparentBackgroundOptions,
+            transport: sessionOptions?.transparentBackgroundTransport,
+          },
           metrics: {
             showPeerConnectionStatsReport:
               this.clientOptions?.metrics?.showPeerConnectionStatsReport ??
@@ -492,7 +506,6 @@ export default class AnamClient {
       });
       throw new Error('Already streaming');
     }
-    this._isStreaming = true;
     if (!this.streamingClient) {
       connectionMilestones.publishFailure({
         failureStage: 'streaming_client_missing',
@@ -502,16 +515,29 @@ export default class AnamClient {
 
     try {
       this.streamingClient.setMediaStreamTargetById(videoElementId);
+      this._isStreaming = true;
       this.streamingClient.startConnection();
     } catch (error) {
       connectionMilestones.publishFailure({
         failureStage: 'start_connection',
         ...getErrorMilestoneTags(error),
       });
+      // Renderer initialization (notably WebGL capability detection) happens
+      // after the server has allocated a session. Do not leak that session when
+      // the local render target cannot be created.
+      await this.stopStreaming();
       throw error;
     }
   }
 
+  /**
+   * Returns the SDK-managed alpha canvas when transparent background rendering
+   * is enabled and `streamToVideoElement` has installed its render target.
+   */
+  public getTransparentBackgroundCanvas(): HTMLCanvasElement | null {
+    return this.streamingClient?.getTransparentBackgroundCanvas() ?? null;
+  }
+
   /**
    * Send a talk command to make the persona speak the provided content.
    * @param content - The text content for the persona to speak

src/lib/ClientMetrics.ts

@@ -13,6 +13,7 @@ export enum ClientMetricMeasurement {
   CLIENT_METRIC_MEASUREMENT_SESSION_ATTEMPT = 'client_session_attempt',
   CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS = 'client_session_success',
   CLIENT_METRIC_MEASUREMENT_ICE_RESTART = 'client_ice_restart',
+  CLIENT_METRIC_MEASUREMENT_TRANSPARENT_RENDERER = 'client_transparent_renderer',
 }
 
 const CLIENT_METRICS_MAX_BATCH_SIZE = 50;

src/modules/PackedAlphaTransport.ts

@@ -0,0 +1,149 @@
+import { StartSessionOptions } from '../types/coreApi/StartSessionOptions';
+import {
+  PACKED_ALPHA_CPU_TRANSPORT,
+  PACKED_ALPHA_TRANSPORT,
+  TransparentBackgroundTransport,
+} from '../types/TransparentBackgroundTransport';
+
+const PACKED_ALPHA_VIDEO_CONFIGURATION = {
+  // Match the RTP format we actually negotiate. This is also the form used by
+  // the Media Capabilities specification's WebRTC H.264 example.
+  contentType:
+    'video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0028',
+  bitrate: 2_500_000,
+  framerate: 25,
+} as const;
+
+const PACKED_ALPHA_DECODING_CONFIGURATIONS: readonly MediaDecodingConfiguration[] =
+  [
+    {
+      type: 'webrtc',
+      video: {
+        ...PACKED_ALPHA_VIDEO_CONFIGURATION,
+        width: 1152,
+        height: 1536,
+      },
+    },
+    {
+      type: 'webrtc',
+      video: {
+        ...PACKED_ALPHA_VIDEO_CONFIGURATION,
+        width: 1536,
+        height: 1152,
+      },
+    },
+  ];
+
+type MediaCapabilitiesLike = Pick<MediaCapabilities, 'decodingInfo'>;
+
+/** @internal */
+export type PackedAlphaCapability = 'supported' | 'unsupported' | 'unknown';
+
+/**
+ * Query support for both orientations of the H.264 Main Level 4.0 packed-alpha
+ * stream. MediaCapabilities results can be orientation-sensitive, and the SDK
+ * does not know the token's requested output dimensions at this point, so both
+ * landscape and portrait carriers must independently report supported+smooth.
+ * Missing or inconclusive MediaCapabilities implementations are reported as
+ * unknown. Callers conservatively retain the legacy carrier in that case: the
+ * server requires an explicit Main-Level-4 offer, so guessing support could
+ * otherwise turn a graceful quality fallback into a failed session.
+ *
+ * @internal
+ */
+export async function detectPackedAlphaCapability(
+  mediaCapabilities: MediaCapabilitiesLike | undefined = typeof navigator ===
+  'undefined'
+    ? undefined
+    : navigator.mediaCapabilities,
+): Promise<PackedAlphaCapability> {
+  if (!mediaCapabilities?.decodingInfo) return 'unknown';
+
+  try {
+    const results = await Promise.all(
+      PACKED_ALPHA_DECODING_CONFIGURATIONS.map((configuration) =>
+        mediaCapabilities.decodingInfo(configuration),
+      ),
+    );
+    // The packed frame doubles the decoded pixel count. A decoder that can
+    // technically accept Main Level 4.0 but is not expected to sustain this
+    // configuration should use the lower-resolution compatibility path.
+    return results.every((result) => result.supported && result.smooth)
+      ? 'supported'
+      : 'unsupported';
+  } catch {
+    // Some browsers expose MediaCapabilities but reject WebRTC configurations.
+    // Treat this as inconclusive; the caller keeps the compatibility path.
+    return 'unknown';
+  }
+}
+
+/** @internal */
+export async function buildTransparentBackgroundSessionOptions(
+  enabled: boolean | undefined,
+  mediaCapabilities?: MediaCapabilitiesLike,
+): Promise<
+  Pick<
+    StartSessionOptions,
+    'transparentBackground' | 'transparentBackgroundTransport'
+  >
+> {
+  if (enabled === undefined) return {};
+  if (!enabled) return { transparentBackground: false };
+
+  const capability = await detectPackedAlphaCapability(mediaCapabilities);
+  if (capability !== 'supported') {
+    console.warn(
+      'Packed transparent-background video support could not be confirmed as smooth on this device; falling back to legacy adaptive chroma keying.',
+    );
+    return { transparentBackground: true };
+  }
+
+  return {
+    transparentBackground: true,
+    transparentBackgroundTransport: PACKED_ALPHA_CPU_TRANSPORT,
+  };
+}
+
+/**
+ * Raise only H.264 Main Level 3.1 payloads to Level 4.0. Other H.264 profiles,
+ * codecs and already-higher levels are deliberately left untouched.
+ *
+ * @internal
+ */
+export function promotePackedAlphaH264Level(sdp: string): string {
+  const h264PayloadTypes = new Set<string>();
+  for (const line of sdp.split(/\r?\n/)) {
+    const match = line.match(/^a=rtpmap:(\d+)\s+H264\/90000(?:\s|$)/i);
+    if (match) h264PayloadTypes.add(match[1]);
+  }
+
+  return sdp.replace(
+    /^a=fmtp:(\d+)(\s+)([^\r\n]*)$/gim,
+    (line, payloadType: string, separator: string, parameters: string) => {
+      if (!h264PayloadTypes.has(payloadType)) return line;
+
+      const promotedParameters = parameters.replace(
+        /(^|;)(\s*profile-level-id\s*=\s*)4d001f(?=\s*(?:;|$))/i,
+        (_match, boundary: string, prefix: string) =>
+          `${boundary}${prefix}4d0028`,
+      );
+      return `a=fmtp:${payloadType}${separator}${promotedParameters}`;
+    },
+  );
+}
+
+/** @internal */
+export function prepareOfferForTransparentBackgroundTransport(
+  offer: RTCSessionDescriptionInit,
+  transport: TransparentBackgroundTransport | undefined,
+): RTCSessionDescriptionInit {
+  if (
+    transport !== PACKED_ALPHA_TRANSPORT &&
+    transport !== PACKED_ALPHA_CPU_TRANSPORT
+  ) {
+    return offer;
+  }
+  if (!offer.sdp) return offer;
+  return { ...offer, sdp: promotePackedAlphaH264Level(offer.sdp) };
+}