diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f999b41 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +ELEVENLABS_API_KEY=replace_with_your_elevenlabs_api_key +OFSPECTRUM_API_KEY=replace_with_your_ofspectrum_api_key +OFSPECTRUM_TOKEN_ID=replace_with_your_ofspectrum_token_id diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a7eb7f..a4d240b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- Added `Ofspectrum(ElevenLabs(...))`, a transparent synchronous ElevenLabs wrapper that watermarks known complete-audio responses while preserving native streaming, JSON, metadata, job, and status behavior. +- Added lazy `client.watermark.config(...)` controls, explicit future audio-method registration, and non-persisting OfSpectrum encode defaults. +- Added a credentialed ElevenLabs TTS latency benchmark reporting average, p50, and p95 for native, wrapped, encode-only, and paired added latency. + ## 1.3.1 - 2026-08-24 - `open_stream_pool(..., keepalive_interval_seconds=120)` heartbeats every pooled connection so idle Neo model sessions are not retired. diff --git a/README.md b/README.md index 2c768e4..bc09f27 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,71 @@ the unpaid requirement. ## Audio Watermarking +### ElevenLabs transparent wrapper (first phase) + +Install the optional ElevenLabs dependency and configure the watermark identity: + +```bash +pip install "ofspectrum[elevenlabs]" +cp .env.example .env +``` + +Set `ELEVENLABS_API_KEY`, `OFSPECTRUM_API_KEY`, and +`OFSPECTRUM_TOKEN_ID` in `.env`. The local `.env` file is ignored by Git. + +Wrap the native synchronous client once; existing resource calls retain the +ElevenLabs shape: + +```python +from dotenv import load_dotenv +from elevenlabs.client import ElevenLabs +from ofspectrum import Ofspectrum + +load_dotenv() + +client = Ofspectrum(ElevenLabs()) +audio = client.text_to_speech.convert( + text="Hello from ElevenLabs", + voice_id="JBFqnCBsd6RMkjVDRZzb", + model_id="eleven_multilingual_v2", +) +``` + +The first phase watermarks complete-file responses from Text-to-Speech, +Speech-to-Speech / Voice Changer, Sound Effects, Text-to-Dialogue, Audio +Isolation, and Music compose. ElevenLabs represents many complete-file calls as +an `Iterator[bytes]`; the wrapper keeps that interface but emits the encoded +file after the native iterator is complete. Calls named `.stream(...)` are +returned untouched. JSON, metadata, timestamp, job ID, and status responses are +also returned untouched and do not require watermark configuration. + +Runtime configuration is available without rebuilding the wrapper: + +```python +client.watermark.config(enabled=False) +client.watermark.config( + enabled=True, + token_id="token-uuid", + strength=1.0, + smooth=True, +) +``` + +For a new ElevenLabs complete-audio method added after this SDK release, opt in +explicitly with `client.watermark.register_audio_method("resource.method")`. +The explicit registry prevents unrelated binary downloads from being mistaken +for audio. + +Run the credentialed latency benchmark with the same TTS request for native and +wrapped calls: + +```bash +python -m benchmarks.elevenlabs_latency --iterations 20 --warmups 2 +``` + +It reports average, p50, and p95 latency for native ElevenLabs, wrapped +ElevenLabs, standalone watermark encode, and the paired added request latency. + `client.audio.encode()` is the default OneFile integration: it sends one audio file to `POST /audio/watermark/encode` and returns one encoded audio file. Omit `interval` to leave the option unset, or pass `0.0` explicitly for continuous diff --git a/benchmarks/elevenlabs_api_smoke.py b/benchmarks/elevenlabs_api_smoke.py new file mode 100644 index 0000000..a0be2c3 --- /dev/null +++ b/benchmarks/elevenlabs_api_smoke.py @@ -0,0 +1,296 @@ +"""Run one live request for each supported ElevenLabs complete-audio endpoint. + +This is a credentialed, quota-consuming smoke test, not a latency benchmark. +It loads ``.env`` from the repository root and prints no credentials. + +Run: + python -m benchmarks.elevenlabs_api_smoke +""" + +import argparse +import json +import os +import tempfile +import time +import uuid +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, Optional, Set + +from dotenv import load_dotenv +from elevenlabs.client import ElevenLabs + +from ofspectrum import Ofspectrum + +VOICE_ID = "JBFqnCBsd6RMkjVDRZzb" +OUTPUT_FORMAT = "mp3_44100_128" +LONG_TEXT = ( + "This live integration check uses a sufficiently long audio sample for reliable " + "watermark encoding. It verifies that ElevenLabs can generate a complete audio " + "file, that the transparent OfSpectrum wrapper intercepts the response, and that " + "the encoded audio returns through the same client method without changing the " + "call structure expected by an existing application." +) + + +class SmokeSkip(Exception): + pass + + +def collect_audio(response: Any) -> bytes: + if isinstance(response, bytes): + return response + return b"".join(bytes(chunk) for chunk in response) + + +def create_test_video(path: Path, seconds: int = 12) -> None: + """Create a tiny deterministic MP4 used only by video-to-music.""" + import av + import numpy as np + + with av.open(str(path), mode="w") as container: + stream = container.add_stream("mpeg4", rate=1) + stream.width = 320 + stream.height = 180 + stream.pix_fmt = "yuv420p" + for index in range(seconds): + pixels = np.zeros((180, 320, 3), dtype=np.uint8) + pixels[:, :, 0] = min(255, index * 15) + pixels[:, :, 1] = 48 + frame = av.VideoFrame.from_ndarray(pixels, format="rgb24") + for packet in stream.encode(frame): + container.mux(packet) + for packet in stream.encode(): + container.mux(packet) + + +def run_case(name: str, operation: Callable[[], Any]) -> Dict[str, Any]: + started = time.perf_counter() + try: + audio = collect_audio(operation()) + if not audio: + raise RuntimeError("endpoint returned empty audio") + return { + "endpoint": name, + "status": "PASS", + "elapsed_ms": round((time.perf_counter() - started) * 1000.0, 2), + "audio_bytes": len(audio), + } + except SmokeSkip as exc: + return { + "endpoint": name, + "status": "SKIP", + "elapsed_ms": round((time.perf_counter() - started) * 1000.0, 2), + "reason": str(exc), + } + except Exception as exc: + return { + "endpoint": name, + "status": "FAIL", + "elapsed_ms": round((time.perf_counter() - started) * 1000.0, 2), + "error_type": type(exc).__name__, + "reason": str(exc), + } + + +def first_history_id(native: ElevenLabs, expected_text: str) -> str: + response = native.history.list(page_size=10, sort_direction="desc") + history = getattr(response, "history", None) or [] + for item in history: + if getattr(item, "text", None) != expected_text: + continue + item_id = getattr(item, "history_item_id", None) + if item_id: + return item_id + raise SmokeSkip("the controlled TTS smoke item was not found in recent history") + + +def first_voice_sample(native: ElevenLabs) -> tuple: + voice = native.voices.get(VOICE_ID) + samples = getattr(voice, "samples", None) or [] + for sample in samples: + sample_id = getattr(sample, "sample_id", None) + if sample_id: + return VOICE_ID, sample_id + raise SmokeSkip("the selected voice has no downloadable sample") + + +def first_conversation_id(native: ElevenLabs) -> str: + response = native.conversational_ai.conversations.list(page_size=10) + conversations = getattr(response, "conversations", None) or [] + if not conversations: + raise SmokeSkip("the account has no existing ElevenLabs conversation") + for conversation in conversations: + conversation_id = getattr(conversation, "conversation_id", None) + if conversation_id: + return conversation_id + raise SmokeSkip("existing conversations have no usable ID") + + +def require_credentials() -> None: + required = ("ELEVENLABS_API_KEY", "OFSPECTRUM_API_KEY", "OFSPECTRUM_TOKEN_ID") + missing = [name for name in required if not os.environ.get(name)] + if missing: + raise SystemExit("Missing required environment variable(s): " + ", ".join(missing)) + + +def skip_account_audio(kind: str) -> Any: + raise SmokeSkip( + kind + + " requires --include-account-audio because existing private audio would be sent to OfSpectrum" + ) + + +def run( + *, + include_account_audio: bool = False, + only: Optional[Iterable[str]] = None, +) -> Dict[str, Any]: + load_dotenv(Path(__file__).resolve().parents[1] / ".env") + require_credentials() + + native = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"]) + wrapped = Ofspectrum( + native, + api_key=os.environ["OFSPECTRUM_API_KEY"], + token_id=os.environ["OFSPECTRUM_TOKEN_ID"], + ) + test_text = LONG_TEXT + " The unique smoke marker is " + uuid.uuid4().hex[:8] + "." + source_path = Path(__file__).resolve().parents[1] / "examples/audio/sample-speech-like-12s.wav" + source_audio = (source_path.name, source_path.read_bytes(), "audio/wav") + selected: Optional[Set[str]] = set(only) if only else None + results = [] + + with tempfile.TemporaryDirectory(prefix="ofspectrum-elevenlabs-smoke-") as temp_dir: + video_path = Path(temp_dir) / "input.mp4" + create_test_video(video_path) + source_video = (video_path.name, video_path.read_bytes(), "video/mp4") + + cases = [ + ( + "text_to_speech.convert", + lambda: wrapped.text_to_speech.convert( + voice_id=VOICE_ID, + text=test_text, + model_id="eleven_multilingual_v2", + output_format=OUTPUT_FORMAT, + ), + ), + ( + "speech_to_speech.convert", + lambda: wrapped.speech_to_speech.convert( + voice_id=VOICE_ID, + audio=source_audio, + model_id="eleven_multilingual_sts_v2", + output_format=OUTPUT_FORMAT, + ), + ), + ( + "text_to_sound_effects.convert", + lambda: wrapped.text_to_sound_effects.convert( + text="A steady ocean shoreline with rolling waves and gentle wind", + duration_seconds=12.0, + output_format=OUTPUT_FORMAT, + ), + ), + ( + "text_to_dialogue.convert", + lambda: wrapped.text_to_dialogue.convert( + inputs=[ + {"voice_id": VOICE_ID, "text": LONG_TEXT}, + { + "voice_id": VOICE_ID, + "text": "The second speaker confirms that the complete dialogue file is ready.", + }, + ], + output_format=OUTPUT_FORMAT, + ), + ), + ( + "audio_isolation.convert", + lambda: wrapped.audio_isolation.convert(audio=source_audio), + ), + ( + "music.compose", + lambda: wrapped.music.compose( + prompt="A calm instrumental ambient track with soft piano and warm pads", + music_length_ms=12000, + force_instrumental=True, + output_format=OUTPUT_FORMAT, + ), + ), + ( + "music.video_to_music", + lambda: wrapped.music.video_to_music( + videos=[source_video], + description="A calm instrumental background track", + output_format=OUTPUT_FORMAT, + ), + ), + ( + "history.get_audio", + lambda: wrapped.history.get_audio(first_history_id(native, test_text)), + ), + ( + "voices.samples.audio.get", + ( + lambda: wrapped.voices.samples.audio.get(*first_voice_sample(native)) + if include_account_audio + else skip_account_audio("voice sample audio") + ), + ), + ( + "conversational_ai.conversations.audio.get", + ( + lambda: wrapped.conversational_ai.conversations.audio.get( + first_conversation_id(native) + ) + if include_account_audio + else skip_account_audio("conversation audio") + ), + ), + ] + + try: + for name, operation in cases: + if selected is not None and name not in selected: + continue + result = run_case(name, operation) + results.append(result) + print(json.dumps(result, ensure_ascii=False), flush=True) + finally: + wrapped.close() + + counts = { + status: sum(result["status"] == status for result in results) + for status in ("PASS", "FAIL", "SKIP") + } + return {"summary": counts, "results": results} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--include-account-audio", + action="store_true", + help=( + "read an existing voice sample and conversation recording from the " + "ElevenLabs account and send each to OfSpectrum for watermark encoding" + ), + ) + parser.add_argument( + "--only", + action="append", + help="run only the named endpoint; repeat this option for multiple endpoints", + ) + return parser.parse_args() + + +if __name__ == "__main__": + arguments = parse_args() + report = run( + include_account_audio=arguments.include_account_audio, + only=arguments.only, + ) + print(json.dumps(report["summary"], indent=2, sort_keys=True)) + if report["summary"]["FAIL"]: + raise SystemExit(1) diff --git a/benchmarks/elevenlabs_duration_latency.py b/benchmarks/elevenlabs_duration_latency.py new file mode 100644 index 0000000..b61de95 --- /dev/null +++ b/benchmarks/elevenlabs_duration_latency.py @@ -0,0 +1,262 @@ +"""Measure watermark latency at 2s, 5s, 10s, and 30s audio tiers. + +Each tier runs three consecutive paired measurements. One ElevenLabs TTS +response is measured as encode-disabled, then those exact audio bytes are sent +through ``WatermarkController.encode_bytes``. The encode-enabled complete +latency is the TTS latency plus the encode latency, avoiding cross-generation +variance in the comparison. + +The script consumes real ElevenLabs and OfSpectrum quota. It checkpoints every +result to one timestamped JSON file under ``benchmarks/results``. + +Run: + python -m benchmarks.elevenlabs_duration_latency +""" + +import argparse +import json +import os +import statistics +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from dotenv import load_dotenv +from elevenlabs.client import ElevenLabs + +from ofspectrum import Ofspectrum +from ofspectrum.media import probe_audio + +VOICE_ID = "JBFqnCBsd6RMkjVDRZzb" +MODEL_ID = "eleven_multilingual_v2" +OUTPUT_FORMAT = "mp3_44100_128" +ITERATIONS = 3 + +TIER_TEXTS = { + 2: "Testing audio watermark latency now.", + 5: "This short sample measures watermark latency for a five second spoken audio clip.", + 10: ( + "This ten second sample measures the additional latency introduced by audio " + "watermark encoding while keeping the voice, model, and output format unchanged." + ), + 30: ( + "This thirty second benchmark sample measures watermark latency on a longer piece " + "of generated speech. Every run uses the same voice, model, output format, and " + "spoken content. First, the script records how long ElevenLabs takes to return the " + "complete audio file. It then sends those exact audio bytes to OfSpectrum for " + "watermark encoding. Using the same generated file for both measurements prevents " + "normal variation between separate speech generations from being counted as " + "watermark overhead, producing a cleaner and more useful comparison." + ), +} + + +def collect_audio(response: Any) -> bytes: + if isinstance(response, bytes): + return response + return b"".join(bytes(chunk) for chunk in response) + + +def percentile(values: Iterable[float], probability: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("at least one value is required") + position = (len(ordered) - 1) * probability + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def latency_summary(values: List[float]) -> Optional[Dict[str, float]]: + if not values: + return None + return { + "average_ms": round(statistics.fmean(values), 2), + "p50_ms": round(percentile(values, 0.50), 2), + "p95_ms": round(percentile(values, 0.95), 2), + "min_ms": round(min(values), 2), + "max_ms": round(max(values), 2), + } + + +def error_record(exc: Exception, elapsed_ms: float) -> Dict[str, Any]: + return { + "status": "FAIL", + "elapsed_until_failure_ms": round(elapsed_ms, 2), + "error_type": type(exc).__name__, + "code": getattr(exc, "code", None), + "status_code": getattr(exc, "status_code", None), + "reason": str(exc), + } + + +def summarize_tier(runs: List[Dict[str, Any]]) -> Dict[str, Any]: + completed = [run for run in runs if run["encode_enabled"]["status"] == "PASS"] + disabled = [run["encode_disabled"]["latency_ms"] for run in runs] + enabled = [run["encode_enabled"]["complete_latency_ms"] for run in completed] + additions = [run["added_latency_ms"] for run in completed] + durations = [run["encode_disabled"]["actual_audio_seconds"] for run in runs] + return { + "attempted_pairs": len(runs), + "successful_pairs": len(completed), + "failed_pairs": len(runs) - len(completed), + "actual_audio_seconds": { + "average": round(statistics.fmean(durations), 3), + "min": round(min(durations), 3), + "max": round(max(durations), 3), + }, + "encode_disabled": latency_summary(disabled), + "encode_enabled": latency_summary(enabled), + "added_by_encode": latency_summary(additions), + } + + +def write_report(path: Path, report: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def require_credentials() -> None: + names = ("ELEVENLABS_API_KEY", "OFSPECTRUM_API_KEY", "OFSPECTRUM_TOKEN_ID") + missing = [name for name in names if not os.environ.get(name)] + if missing: + raise SystemExit("Missing required environment variable(s): " + ", ".join(missing)) + + +def default_output_path() -> Path: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return Path(__file__).resolve().parent / "results" / ( + "elevenlabs_duration_latency_" + timestamp + ".json" + ) + + +def run(output_path: Path) -> Dict[str, Any]: + load_dotenv(Path(__file__).resolve().parents[1] / ".env") + require_credentials() + + native = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"]) + wrapped = Ofspectrum( + native, + api_key=os.environ["OFSPECTRUM_API_KEY"], + token_id=os.environ["OFSPECTRUM_TOKEN_ID"], + ) + report = { + "created_at": datetime.now(timezone.utc).isoformat(), + "methodology": { + "iterations_per_tier": ITERATIONS, + "order": "TTS encode-disabled, then encode the exact same bytes", + "encode_disabled_ms": "complete ElevenLabs TTS latency", + "encode_enabled_ms": "TTS latency + OfSpectrum encode latency", + "added_latency_ms": "OfSpectrum encode latency for the paired audio", + "retries": 0, + }, + "request": { + "voice_id": VOICE_ID, + "model_id": MODEL_ID, + "output_format": OUTPUT_FORMAT, + }, + "tiers": [], + } # type: Dict[str, Any] + write_report(output_path, report) + + try: + for target_seconds, text in TIER_TEXTS.items(): + tier = { + "target_seconds": target_seconds, + "text_word_count": len(text.split()), + "runs": [], + } # type: Dict[str, Any] + report["tiers"].append(tier) + + for iteration in range(1, ITERATIONS + 1): + request = { + "voice_id": VOICE_ID, + "text": text, + "model_id": MODEL_ID, + "output_format": OUTPUT_FORMAT, + "seed": 42, + } + tts_started = time.perf_counter() + try: + native_audio = collect_audio(native.text_to_speech.convert(**request)) + tts_ms = (time.perf_counter() - tts_started) * 1000.0 + actual_seconds = probe_audio(native_audio).duration_seconds + run_result = { + "iteration": iteration, + "encode_disabled": { + "status": "PASS", + "latency_ms": round(tts_ms, 2), + "actual_audio_seconds": round(actual_seconds, 3), + "audio_bytes": len(native_audio), + }, + } # type: Dict[str, Any] + except Exception as exc: + tts_ms = (time.perf_counter() - tts_started) * 1000.0 + run_result = { + "iteration": iteration, + "encode_disabled": error_record(exc, tts_ms), + "encode_enabled": { + "status": "SKIP", + "reason": "encode-disabled TTS request failed", + }, + "added_latency_ms": None, + } + tier["runs"].append(run_result) + write_report(output_path, report) + print(json.dumps({"target_seconds": target_seconds, **run_result}), flush=True) + continue + + encode_started = time.perf_counter() + try: + encoded_audio = wrapped.watermark.encode_bytes( + native_audio, + filename="elevenlabs-output.mp3", + ) + encode_ms = (time.perf_counter() - encode_started) * 1000.0 + run_result["encode_enabled"] = { + "status": "PASS", + "complete_latency_ms": round(tts_ms + encode_ms, 2), + "encode_latency_ms": round(encode_ms, 2), + "audio_bytes": len(encoded_audio), + } + run_result["added_latency_ms"] = round(encode_ms, 2) + run_result["increase_percent_vs_disabled"] = round( + encode_ms / tts_ms * 100.0, 2 + ) + except Exception as exc: + encode_ms = (time.perf_counter() - encode_started) * 1000.0 + run_result["encode_enabled"] = error_record(exc, encode_ms) + run_result["added_latency_ms"] = None + run_result["increase_percent_vs_disabled"] = None + + tier["runs"].append(run_result) + write_report(output_path, report) + print(json.dumps({"target_seconds": target_seconds, **run_result}), flush=True) + + successful_tts = [ + item for item in tier["runs"] if item["encode_disabled"]["status"] == "PASS" + ] + if successful_tts: + tier["summary"] = summarize_tier(successful_tts) + write_report(output_path, report) + finally: + wrapped.close() + + report["completed_at"] = datetime.now(timezone.utc).isoformat() + report["output_file"] = str(output_path) + write_report(output_path, report) + return report + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=None) + return parser.parse_args() + + +if __name__ == "__main__": + arguments = parse_args() + destination = arguments.output or default_output_path() + result = run(destination) + print(json.dumps({"output_file": result["output_file"]}, indent=2)) diff --git a/benchmarks/elevenlabs_latency.py b/benchmarks/elevenlabs_latency.py new file mode 100644 index 0000000..9975cb8 --- /dev/null +++ b/benchmarks/elevenlabs_latency.py @@ -0,0 +1,171 @@ +"""Benchmark native ElevenLabs TTS against OfSpectrum-wrapped TTS. + +Required environment variables: + ELEVENLABS_API_KEY + OFSPECTRUM_API_KEY + OFSPECTRUM_TOKEN_ID + +Example: + python -m benchmarks.elevenlabs_latency --iterations 20 --warmups 2 +""" + +import argparse +import json +import math +import os +import statistics +import time +from typing import Any, Dict, Iterable, List, Tuple + +from ofspectrum import Ofspectrum + + +def collect_audio(response: Any) -> bytes: + if isinstance(response, bytes): + return response + return b"".join(bytes(chunk) for chunk in response) + + +def percentile(values: Iterable[float], probability: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("at least one value is required") + position = (len(ordered) - 1) * probability + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def summary(values: List[float]) -> Dict[str, float]: + return { + "average_ms": statistics.fmean(values) * 1000.0, + "p50_ms": percentile(values, 0.50) * 1000.0, + "p95_ms": percentile(values, 0.95) * 1000.0, + } + + +def timed_tts(client: Any, request: Dict[str, Any]) -> Tuple[float, bytes]: + started = time.perf_counter() + audio = collect_audio(client.text_to_speech.convert(**request)) + return time.perf_counter() - started, audio + + +def output_filename(output_format: str) -> str: + codec = output_format.split("_", 1)[0].lower() + extension = "mp3" if codec == "auto" else codec + return "elevenlabs-output." + extension + + +def run(args: argparse.Namespace) -> Dict[str, Any]: + try: + from dotenv import load_dotenv + from elevenlabs.client import ElevenLabs + except ImportError as exc: + raise SystemExit( + 'The benchmark requires the optional dependency: pip install "ofspectrum[elevenlabs]"' + ) from exc + + load_dotenv() + + elevenlabs_key = os.environ.get("ELEVENLABS_API_KEY") + ofspectrum_key = os.environ.get("OFSPECTRUM_API_KEY") + token_id = os.environ.get("OFSPECTRUM_TOKEN_ID") + missing = [ + name + for name, value in ( + ("ELEVENLABS_API_KEY", elevenlabs_key), + ("OFSPECTRUM_API_KEY", ofspectrum_key), + ("OFSPECTRUM_TOKEN_ID", token_id), + ) + if not value + ] + if missing: + raise SystemExit("Missing required environment variable(s): " + ", ".join(missing)) + + native = ElevenLabs(api_key=elevenlabs_key) + wrapped = Ofspectrum( + native, + api_key=ofspectrum_key, + token_id=token_id, + ) + request = { + "text": args.text, + "voice_id": args.voice_id, + "model_id": args.model_id, + "output_format": args.output_format, + } + + native_times = [] # type: List[float] + wrapped_times = [] # type: List[float] + encode_times = [] # type: List[float] + additions = [] # type: List[float] + try: + for _ in range(args.warmups): + _, audio = timed_tts(native, request) + timed_tts(wrapped, request) + wrapped.watermark.encode_bytes( + audio, filename=output_filename(args.output_format) + ) + + for index in range(args.iterations): + # Alternate order to reduce connection warming and service drift + # from systematically favoring either client path. + if index % 2: + wrapped_elapsed, _ = timed_tts(wrapped, request) + native_elapsed, native_audio = timed_tts(native, request) + else: + native_elapsed, native_audio = timed_tts(native, request) + wrapped_elapsed, _ = timed_tts(wrapped, request) + + encode_started = time.perf_counter() + wrapped.watermark.encode_bytes( + native_audio, + filename=output_filename(args.output_format), + ) + encode_elapsed = time.perf_counter() - encode_started + + native_times.append(native_elapsed) + wrapped_times.append(wrapped_elapsed) + encode_times.append(encode_elapsed) + additions.append(wrapped_elapsed - native_elapsed) + finally: + wrapped.close() + + return { + "iterations": args.iterations, + "warmups": args.warmups, + "request": request, + "native_elevenlabs": summary(native_times), + "elevenlabs_plus_watermark": summary(wrapped_times), + "watermark_encode_only": summary(encode_times), + "added_to_complete_request": summary(additions), + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--iterations", type=int, default=20) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--voice-id", default="JBFqnCBsd6RMkjVDRZzb") + parser.add_argument("--model-id", default="eleven_multilingual_v2") + parser.add_argument("--output-format", default="mp3_44100_128") + parser.add_argument( + "--text", + default=( + "Audio watermark latency needs a sample long enough for reliable encoding. " + "This benchmark uses the same ElevenLabs request for both the native client " + "and the OfSpectrum wrapper. The resulting measurements compare generation " + "time, watermark encoding time, and the total additional request latency " + "without changing the voice, model, output format, or spoken content." + ), + ) + args = parser.parse_args() + if args.iterations < 1 or args.warmups < 0: + parser.error("iterations must be positive and warmups cannot be negative") + return args + + +if __name__ == "__main__": + print(json.dumps(run(parse_args()), indent=2, sort_keys=True)) diff --git a/benchmarks/elevenlabs_services_duration_latency.py b/benchmarks/elevenlabs_services_duration_latency.py new file mode 100644 index 0000000..1e9551a --- /dev/null +++ b/benchmarks/elevenlabs_services_duration_latency.py @@ -0,0 +1,315 @@ +"""Run duration-tier latency tests across ElevenLabs complete-audio services. + +Services: Speech-to-Speech, Sound Effects, Text-to-Dialogue, Audio Isolation, +Music Compose, and Video-to-Music. Each service runs 2s, 5s, 10s, and 30s +tiers three consecutive times. Every native response is then watermarked using +the exact same bytes, producing a paired encode-disabled / encode-enabled +comparison without cross-generation variance. + +This script consumes real API quota and checkpoints all results to one JSON +file under ``benchmarks/results``. +""" + +import argparse +import json +import os +import tempfile +import time +import wave +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +from dotenv import load_dotenv +from elevenlabs.client import ElevenLabs + +from ofspectrum import Ofspectrum +from ofspectrum.media import probe_audio + +from .elevenlabs_api_smoke import create_test_video +from .elevenlabs_duration_latency import ( + ITERATIONS, + MODEL_ID, + OUTPUT_FORMAT, + TIER_TEXTS, + VOICE_ID, + collect_audio, + error_record, + summarize_tier, + write_report, +) + +SERVICES = ( + "speech_to_speech", + "sound_effects", + "text_to_dialogue", + "audio_isolation", + "music_compose", + "video_to_music", +) + + +def create_tier_wav(source: Path, destination: Path, seconds: int) -> None: + """Trim or repeat the repository speech sample to an exact WAV duration.""" + with wave.open(str(source), "rb") as input_file: + parameters = input_file.getparams() + source_frames = input_file.readframes(input_file.getnframes()) + frame_bytes = parameters.sampwidth * parameters.nchannels + required_bytes = int(parameters.framerate * seconds) * frame_bytes + repeats = required_bytes // len(source_frames) + 1 + output_frames = (source_frames * repeats)[:required_bytes] + with wave.open(str(destination), "wb") as output_file: + output_file.setparams(parameters) + output_file.writeframes(output_frames) + + +def native_operation( + native: ElevenLabs, + service: str, + target_seconds: int, + text: str, + wav_input: Tuple[str, bytes, str], + video_input: Tuple[str, bytes, str], +) -> Any: + if service == "speech_to_speech": + return native.speech_to_speech.convert( + voice_id=VOICE_ID, + audio=wav_input, + model_id="eleven_multilingual_sts_v2", + output_format=OUTPUT_FORMAT, + enable_logging=False, + ) + if service == "sound_effects": + return native.text_to_sound_effects.convert( + text="A steady ocean shoreline with rolling waves and gentle wind", + duration_seconds=float(target_seconds), + output_format=OUTPUT_FORMAT, + ) + if service == "text_to_dialogue": + return native.text_to_dialogue.convert( + inputs=[{"voice_id": VOICE_ID, "text": text}], + output_format=OUTPUT_FORMAT, + enable_logging=False, + ) + if service == "audio_isolation": + return native.audio_isolation.convert(audio=wav_input) + if service == "music_compose": + return native.music.compose( + prompt="A calm instrumental ambient track with soft piano and warm pads", + music_length_ms=target_seconds * 1000, + force_instrumental=True, + output_format=OUTPUT_FORMAT, + ) + if service == "video_to_music": + return native.music.video_to_music( + videos=[video_input], + description="A calm instrumental background track", + output_format=OUTPUT_FORMAT, + ) + raise ValueError("Unsupported service: " + service) + + +def default_output_path() -> Path: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return Path(__file__).resolve().parent / "results" / ( + "elevenlabs_services_duration_latency_" + timestamp + ".json" + ) + + +def require_credentials() -> None: + names = ("ELEVENLABS_API_KEY", "OFSPECTRUM_API_KEY", "OFSPECTRUM_TOKEN_ID") + missing = [name for name in names if not os.environ.get(name)] + if missing: + raise SystemExit("Missing required environment variable(s): " + ", ".join(missing)) + + +def selected_services(requested: Optional[Iterable[str]]) -> Set[str]: + selected = set(requested or SERVICES) + unsupported = selected - set(SERVICES) + if unsupported: + raise ValueError("Unsupported service(s): " + ", ".join(sorted(unsupported))) + return selected + + +def empty_summary(runs: List[Dict[str, Any]]) -> Dict[str, Any]: + return { + "attempted_pairs": len(runs), + "successful_pairs": 0, + "failed_pairs": len(runs), + "actual_audio_seconds": None, + "encode_disabled": None, + "encode_enabled": None, + "added_by_encode": None, + } + + +def run(output_path: Path, requested_services: Optional[Iterable[str]] = None) -> Dict[str, Any]: + load_dotenv(Path(__file__).resolve().parents[1] / ".env") + require_credentials() + selected = selected_services(requested_services) + + native = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"]) + wrapped = Ofspectrum( + native, + api_key=os.environ["OFSPECTRUM_API_KEY"], + token_id=os.environ["OFSPECTRUM_TOKEN_ID"], + ) + report = { + "created_at": datetime.now(timezone.utc).isoformat(), + "methodology": { + "iterations_per_tier": ITERATIONS, + "tiers_seconds": list(TIER_TEXTS), + "order": "native service response, then encode the exact same bytes", + "encode_disabled_ms": "complete native ElevenLabs service latency", + "encode_enabled_ms": "native latency + OfSpectrum encode latency", + "added_latency_ms": "OfSpectrum encode latency for the paired audio", + "retries": 0, + }, + "request": { + "voice_id": VOICE_ID, + "tts_model_id": MODEL_ID, + "output_format": OUTPUT_FORMAT, + }, + "services": [], + } # type: Dict[str, Any] + write_report(output_path, report) + + source_path = Path(__file__).resolve().parents[1] / "examples/audio/sample-speech-like-12s.wav" + try: + with tempfile.TemporaryDirectory(prefix="ofspectrum-duration-services-") as temp_dir: + temp_path = Path(temp_dir) + inputs = {} + for target_seconds in TIER_TEXTS: + wav_path = temp_path / ("speech-" + str(target_seconds) + "s.wav") + video_path = temp_path / ("video-" + str(target_seconds) + "s.mp4") + create_tier_wav(source_path, wav_path, target_seconds) + create_test_video(video_path, seconds=target_seconds) + inputs[target_seconds] = { + "wav": (wav_path.name, wav_path.read_bytes(), "audio/wav"), + "video": (video_path.name, video_path.read_bytes(), "video/mp4"), + } + + for service in SERVICES: + if service not in selected: + continue + service_result = {"service": service, "tiers": []} + report["services"].append(service_result) + + for target_seconds, text in TIER_TEXTS.items(): + tier = {"target_seconds": target_seconds, "runs": []} + service_result["tiers"].append(tier) + + for iteration in range(1, ITERATIONS + 1): + native_started = time.perf_counter() + try: + response = native_operation( + native, + service, + target_seconds, + text, + inputs[target_seconds]["wav"], + inputs[target_seconds]["video"], + ) + native_audio = collect_audio(response) + native_ms = (time.perf_counter() - native_started) * 1000.0 + actual_seconds = probe_audio(native_audio).duration_seconds + run_result = { + "iteration": iteration, + "encode_disabled": { + "status": "PASS", + "latency_ms": round(native_ms, 2), + "actual_audio_seconds": round(actual_seconds, 3), + "audio_bytes": len(native_audio), + }, + } # type: Dict[str, Any] + except Exception as exc: + native_ms = (time.perf_counter() - native_started) * 1000.0 + run_result = { + "iteration": iteration, + "encode_disabled": error_record(exc, native_ms), + "encode_enabled": { + "status": "SKIP", + "reason": "native service request failed", + }, + "added_latency_ms": None, + "increase_percent_vs_disabled": None, + } + tier["runs"].append(run_result) + write_report(output_path, report) + print( + json.dumps( + {"service": service, "target_seconds": target_seconds, **run_result} + ), + flush=True, + ) + continue + + encode_started = time.perf_counter() + try: + encoded_audio = wrapped.watermark.encode_bytes( + native_audio, + filename="elevenlabs-output.mp3", + ) + encode_ms = (time.perf_counter() - encode_started) * 1000.0 + run_result["encode_enabled"] = { + "status": "PASS", + "complete_latency_ms": round(native_ms + encode_ms, 2), + "encode_latency_ms": round(encode_ms, 2), + "audio_bytes": len(encoded_audio), + } + run_result["added_latency_ms"] = round(encode_ms, 2) + run_result["increase_percent_vs_disabled"] = round( + encode_ms / native_ms * 100.0, 2 + ) + except Exception as exc: + encode_ms = (time.perf_counter() - encode_started) * 1000.0 + run_result["encode_enabled"] = error_record(exc, encode_ms) + run_result["added_latency_ms"] = None + run_result["increase_percent_vs_disabled"] = None + + tier["runs"].append(run_result) + write_report(output_path, report) + print( + json.dumps( + {"service": service, "target_seconds": target_seconds, **run_result} + ), + flush=True, + ) + + native_successes = [ + item + for item in tier["runs"] + if item["encode_disabled"]["status"] == "PASS" + ] + tier["summary"] = ( + summarize_tier(native_successes) + if native_successes + else empty_summary(tier["runs"]) + ) + write_report(output_path, report) + finally: + wrapped.close() + + report["completed_at"] = datetime.now(timezone.utc).isoformat() + report["output_file"] = str(output_path) + write_report(output_path, report) + return report + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=None) + parser.add_argument( + "--service", + action="append", + choices=SERVICES, + help="run only this service; repeat for multiple services", + ) + return parser.parse_args() + + +if __name__ == "__main__": + arguments = parse_args() + destination = arguments.output or default_output_path() + result = run(destination, requested_services=arguments.service) + print(json.dumps({"output_file": result["output_file"]}, indent=2)) diff --git a/benchmarks/results/elevenlabs_duration_latency_20260831T192145Z.json b/benchmarks/results/elevenlabs_duration_latency_20260831T192145Z.json new file mode 100644 index 0000000..62a76d3 --- /dev/null +++ b/benchmarks/results/elevenlabs_duration_latency_20260831T192145Z.json @@ -0,0 +1,366 @@ +{ + "completed_at": "2026-08-31T19:22:35.847963+00:00", + "created_at": "2026-08-31T19:21:45.515315+00:00", + "methodology": { + "added_latency_ms": "OfSpectrum encode latency for the paired audio", + "encode_disabled_ms": "complete ElevenLabs TTS latency", + "encode_enabled_ms": "TTS latency + OfSpectrum encode latency", + "iterations_per_tier": 3, + "order": "TTS encode-disabled, then encode the exact same bytes", + "retries": 0 + }, + "output_file": "/Users/dingshenggang/Desktop/python-sdk/benchmarks/results/elevenlabs_duration_latency_20260831T192145Z.json", + "request": { + "model_id": "eleven_multilingual_v2", + "output_format": "mp3_44100_128", + "voice_id": "JBFqnCBsd6RMkjVDRZzb" + }, + "tiers": [ + { + "runs": [ + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.554, + "audio_bytes": 41839, + "latency_ms": 1326.72, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 305.02, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 1 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.554, + "audio_bytes": 41839, + "latency_ms": 957.41, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 139.44, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.554, + "audio_bytes": 41839, + "latency_ms": 962.27, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 139.93, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 2.554, + "max": 2.554, + "min": 2.554 + }, + "added_by_encode": null, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1082.13, + "max_ms": 1326.72, + "min_ms": 957.41, + "p50_ms": 962.27, + "p95_ms": 1290.28 + }, + "encode_enabled": null, + "failed_pairs": 3, + "successful_pairs": 0 + }, + "target_seconds": 2, + "text_word_count": 5 + }, + { + "runs": [ + { + "added_latency_ms": 1898.7, + "encode_disabled": { + "actual_audio_seconds": 5.108, + "audio_bytes": 82799, + "latency_ms": 1171.71, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 82799, + "complete_latency_ms": 3070.41, + "encode_latency_ms": 1898.7, + "status": "PASS" + }, + "increase_percent_vs_disabled": 162.05, + "iteration": 1 + }, + { + "added_latency_ms": 1890.74, + "encode_disabled": { + "actual_audio_seconds": 5.108, + "audio_bytes": 82799, + "latency_ms": 1114.52, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 82799, + "complete_latency_ms": 3005.25, + "encode_latency_ms": 1890.74, + "status": "PASS" + }, + "increase_percent_vs_disabled": 169.65, + "iteration": 2 + }, + { + "added_latency_ms": 1915.88, + "encode_disabled": { + "actual_audio_seconds": 5.108, + "audio_bytes": 82799, + "latency_ms": 1276.35, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 82799, + "complete_latency_ms": 3192.23, + "encode_latency_ms": 1915.88, + "status": "PASS" + }, + "increase_percent_vs_disabled": 150.11, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 5.108, + "max": 5.108, + "min": 5.108 + }, + "added_by_encode": { + "average_ms": 1901.77, + "max_ms": 1915.88, + "min_ms": 1890.74, + "p50_ms": 1898.7, + "p95_ms": 1914.16 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1187.53, + "max_ms": 1276.35, + "min_ms": 1114.52, + "p50_ms": 1171.71, + "p95_ms": 1265.89 + }, + "encode_enabled": { + "average_ms": 3089.3, + "max_ms": 3192.23, + "min_ms": 3005.25, + "p50_ms": 3070.41, + "p95_ms": 3180.05 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 5, + "text_word_count": 13 + }, + { + "runs": [ + { + "added_latency_ms": 2087.47, + "encode_disabled": { + "actual_audio_seconds": 10.077, + "audio_bytes": 162212, + "latency_ms": 1590.87, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 3678.34, + "encode_latency_ms": 2087.47, + "status": "PASS" + }, + "increase_percent_vs_disabled": 131.22, + "iteration": 1 + }, + { + "added_latency_ms": 2034.49, + "encode_disabled": { + "actual_audio_seconds": 10.077, + "audio_bytes": 162212, + "latency_ms": 1618.58, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 3653.07, + "encode_latency_ms": 2034.49, + "status": "PASS" + }, + "increase_percent_vs_disabled": 125.7, + "iteration": 2 + }, + { + "added_latency_ms": 2126.83, + "encode_disabled": { + "actual_audio_seconds": 10.077, + "audio_bytes": 162212, + "latency_ms": 1552.59, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 3679.42, + "encode_latency_ms": 2126.83, + "status": "PASS" + }, + "increase_percent_vs_disabled": 136.99, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 10.077, + "max": 10.077, + "min": 10.077 + }, + "added_by_encode": { + "average_ms": 2082.93, + "max_ms": 2126.83, + "min_ms": 2034.49, + "p50_ms": 2087.47, + "p95_ms": 2122.89 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1587.35, + "max_ms": 1618.58, + "min_ms": 1552.59, + "p50_ms": 1590.87, + "p95_ms": 1615.81 + }, + "encode_enabled": { + "average_ms": 3670.28, + "max_ms": 3679.42, + "min_ms": 3653.07, + "p50_ms": 3678.34, + "p95_ms": 3679.31 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 10, + "text_word_count": 22 + }, + { + "runs": [ + { + "added_latency_ms": 3052.61, + "encode_disabled": { + "actual_audio_seconds": 34.04, + "audio_bytes": 545898, + "latency_ms": 5411.01, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 545898, + "complete_latency_ms": 8463.62, + "encode_latency_ms": 3052.61, + "status": "PASS" + }, + "increase_percent_vs_disabled": 56.41, + "iteration": 1 + }, + { + "added_latency_ms": 3154.98, + "encode_disabled": { + "actual_audio_seconds": 34.04, + "audio_bytes": 545898, + "latency_ms": 5327.26, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 545898, + "complete_latency_ms": 8482.24, + "encode_latency_ms": 3154.98, + "status": "PASS" + }, + "increase_percent_vs_disabled": 59.22, + "iteration": 2 + }, + { + "added_latency_ms": 3140.4, + "encode_disabled": { + "actual_audio_seconds": 34.04, + "audio_bytes": 545898, + "latency_ms": 5412.46, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 545898, + "complete_latency_ms": 8552.85, + "encode_latency_ms": 3140.4, + "status": "PASS" + }, + "increase_percent_vs_disabled": 58.02, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 34.04, + "max": 34.04, + "min": 34.04 + }, + "added_by_encode": { + "average_ms": 3116.0, + "max_ms": 3154.98, + "min_ms": 3052.61, + "p50_ms": 3140.4, + "p95_ms": 3153.52 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 5383.58, + "max_ms": 5412.46, + "min_ms": 5327.26, + "p50_ms": 5411.01, + "p95_ms": 5412.31 + }, + "encode_enabled": { + "average_ms": 8499.57, + "max_ms": 8552.85, + "min_ms": 8463.62, + "p50_ms": 8482.24, + "p95_ms": 8545.79 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 30, + "text_word_count": 81 + } + ] +} diff --git a/benchmarks/results/elevenlabs_services_duration_latency_20260831T192903Z.json b/benchmarks/results/elevenlabs_services_duration_latency_20260831T192903Z.json new file mode 100644 index 0000000..fef65d9 --- /dev/null +++ b/benchmarks/results/elevenlabs_services_duration_latency_20260831T192903Z.json @@ -0,0 +1,2078 @@ +{ + "completed_at": "2026-08-31T19:36:51.685891+00:00", + "created_at": "2026-08-31T19:29:03.059889+00:00", + "methodology": { + "added_latency_ms": "OfSpectrum encode latency for the paired audio", + "encode_disabled_ms": "complete native ElevenLabs service latency", + "encode_enabled_ms": "native latency + OfSpectrum encode latency", + "iterations_per_tier": 3, + "order": "native service response, then encode the exact same bytes", + "retries": 0, + "tiers_seconds": [ + 2, + 5, + 10, + 30 + ] + }, + "output_file": "/Users/dingshenggang/Desktop/python-sdk/benchmarks/results/elevenlabs_services_duration_latency_20260831T192903Z.json", + "request": { + "output_format": "mp3_44100_128", + "tts_model_id": "eleven_multilingual_v2", + "voice_id": "JBFqnCBsd6RMkjVDRZzb" + }, + "services": [ + { + "service": "speech_to_speech", + "tiers": [ + { + "runs": [ + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.043, + "audio_bytes": 33898, + "latency_ms": 1043.22, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 288.6, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 1 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.043, + "audio_bytes": 33898, + "latency_ms": 1299.5, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 137.27, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.043, + "audio_bytes": 33898, + "latency_ms": 1048.0, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 128.33, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 2.043, + "max": 2.043, + "min": 2.043 + }, + "added_by_encode": null, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1130.24, + "max_ms": 1299.5, + "min_ms": 1043.22, + "p50_ms": 1048.0, + "p95_ms": 1274.35 + }, + "encode_enabled": null, + "failed_pairs": 3, + "successful_pairs": 0 + }, + "target_seconds": 2 + }, + { + "runs": [ + { + "added_latency_ms": 1881.22, + "encode_disabled": { + "actual_audio_seconds": 5.016, + "audio_bytes": 81128, + "latency_ms": 1244.92, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81546, + "complete_latency_ms": 3126.15, + "encode_latency_ms": 1881.22, + "status": "PASS" + }, + "increase_percent_vs_disabled": 151.11, + "iteration": 1 + }, + { + "added_latency_ms": 1823.79, + "encode_disabled": { + "actual_audio_seconds": 5.016, + "audio_bytes": 81128, + "latency_ms": 1226.28, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81546, + "complete_latency_ms": 3050.07, + "encode_latency_ms": 1823.79, + "status": "PASS" + }, + "increase_percent_vs_disabled": 148.73, + "iteration": 2 + }, + { + "added_latency_ms": 1879.11, + "encode_disabled": { + "actual_audio_seconds": 5.016, + "audio_bytes": 81128, + "latency_ms": 1206.08, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81546, + "complete_latency_ms": 3085.2, + "encode_latency_ms": 1879.11, + "status": "PASS" + }, + "increase_percent_vs_disabled": 155.8, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 5.016, + "max": 5.016, + "min": 5.016 + }, + "added_by_encode": { + "average_ms": 1861.37, + "max_ms": 1881.22, + "min_ms": 1823.79, + "p50_ms": 1879.11, + "p95_ms": 1881.01 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1225.76, + "max_ms": 1244.92, + "min_ms": 1206.08, + "p50_ms": 1226.28, + "p95_ms": 1243.06 + }, + "encode_enabled": { + "average_ms": 3087.14, + "max_ms": 3126.15, + "min_ms": 3050.07, + "p50_ms": 3085.2, + "p95_ms": 3122.05 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 5 + }, + { + "runs": [ + { + "added_latency_ms": 1977.96, + "encode_disabled": { + "actual_audio_seconds": 10.031, + "audio_bytes": 161376, + "latency_ms": 1660.52, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 161794, + "complete_latency_ms": 3638.49, + "encode_latency_ms": 1977.96, + "status": "PASS" + }, + "increase_percent_vs_disabled": 119.12, + "iteration": 1 + }, + { + "added_latency_ms": 2001.68, + "encode_disabled": { + "actual_audio_seconds": 10.031, + "audio_bytes": 161376, + "latency_ms": 1571.71, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 161794, + "complete_latency_ms": 3573.39, + "encode_latency_ms": 2001.68, + "status": "PASS" + }, + "increase_percent_vs_disabled": 127.36, + "iteration": 2 + }, + { + "added_latency_ms": 2033.01, + "encode_disabled": { + "actual_audio_seconds": 10.031, + "audio_bytes": 161376, + "latency_ms": 1547.38, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 161794, + "complete_latency_ms": 3580.39, + "encode_latency_ms": 2033.01, + "status": "PASS" + }, + "increase_percent_vs_disabled": 131.38, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 10.031, + "max": 10.031, + "min": 10.031 + }, + "added_by_encode": { + "average_ms": 2004.22, + "max_ms": 2033.01, + "min_ms": 1977.96, + "p50_ms": 2001.68, + "p95_ms": 2029.88 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1593.2, + "max_ms": 1660.52, + "min_ms": 1547.38, + "p50_ms": 1571.71, + "p95_ms": 1651.64 + }, + "encode_enabled": { + "average_ms": 3597.42, + "max_ms": 3638.49, + "min_ms": 3573.39, + "p50_ms": 3580.39, + "p95_ms": 3632.68 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 10 + }, + { + "runs": [ + { + "added_latency_ms": 2821.57, + "encode_disabled": { + "actual_audio_seconds": 30.0, + "audio_bytes": 481115, + "latency_ms": 4537.81, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 7359.38, + "encode_latency_ms": 2821.57, + "status": "PASS" + }, + "increase_percent_vs_disabled": 62.18, + "iteration": 1 + }, + { + "added_latency_ms": 2741.37, + "encode_disabled": { + "actual_audio_seconds": 30.0, + "audio_bytes": 481115, + "latency_ms": 4436.89, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 7178.26, + "encode_latency_ms": 2741.37, + "status": "PASS" + }, + "increase_percent_vs_disabled": 61.79, + "iteration": 2 + }, + { + "added_latency_ms": 2931.28, + "encode_disabled": { + "actual_audio_seconds": 30.0, + "audio_bytes": 481115, + "latency_ms": 4633.33, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 7564.6, + "encode_latency_ms": 2931.28, + "status": "PASS" + }, + "increase_percent_vs_disabled": 63.27, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 30.0, + "max": 30.0, + "min": 30.0 + }, + "added_by_encode": { + "average_ms": 2831.41, + "max_ms": 2931.28, + "min_ms": 2741.37, + "p50_ms": 2821.57, + "p95_ms": 2920.31 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 4536.01, + "max_ms": 4633.33, + "min_ms": 4436.89, + "p50_ms": 4537.81, + "p95_ms": 4623.78 + }, + "encode_enabled": { + "average_ms": 7367.41, + "max_ms": 7564.6, + "min_ms": 7178.26, + "p50_ms": 7359.38, + "p95_ms": 7544.08 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 30 + } + ] + }, + { + "service": "sound_effects", + "tiers": [ + { + "runs": [ + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.0, + "audio_bytes": 33062, + "latency_ms": 1892.34, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 126.2, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 1 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.0, + "audio_bytes": 33062, + "latency_ms": 2103.61, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 132.1, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.0, + "audio_bytes": 33062, + "latency_ms": 1827.97, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 125.43, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 2.0, + "max": 2.0, + "min": 2.0 + }, + "added_by_encode": null, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1941.31, + "max_ms": 2103.61, + "min_ms": 1827.97, + "p50_ms": 1892.34, + "p95_ms": 2082.48 + }, + "encode_enabled": null, + "failed_pairs": 3, + "successful_pairs": 0 + }, + "target_seconds": 2 + }, + { + "runs": [ + { + "added_latency_ms": 2027.82, + "encode_disabled": { + "actual_audio_seconds": 5.0, + "audio_bytes": 81128, + "latency_ms": 5581.3, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81128, + "complete_latency_ms": 7609.13, + "encode_latency_ms": 2027.82, + "status": "PASS" + }, + "increase_percent_vs_disabled": 36.33, + "iteration": 1 + }, + { + "added_latency_ms": 1889.95, + "encode_disabled": { + "actual_audio_seconds": 5.0, + "audio_bytes": 81128, + "latency_ms": 1697.07, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81128, + "complete_latency_ms": 3587.02, + "encode_latency_ms": 1889.95, + "status": "PASS" + }, + "increase_percent_vs_disabled": 111.37, + "iteration": 2 + }, + { + "added_latency_ms": 1819.04, + "encode_disabled": { + "actual_audio_seconds": 5.0, + "audio_bytes": 81128, + "latency_ms": 1656.99, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81128, + "complete_latency_ms": 3476.04, + "encode_latency_ms": 1819.04, + "status": "PASS" + }, + "increase_percent_vs_disabled": 109.78, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 5.0, + "max": 5.0, + "min": 5.0 + }, + "added_by_encode": { + "average_ms": 1912.27, + "max_ms": 2027.82, + "min_ms": 1819.04, + "p50_ms": 1889.95, + "p95_ms": 2014.03 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 2978.45, + "max_ms": 5581.3, + "min_ms": 1656.99, + "p50_ms": 1697.07, + "p95_ms": 5192.88 + }, + "encode_enabled": { + "average_ms": 4890.73, + "max_ms": 7609.13, + "min_ms": 3476.04, + "p50_ms": 3587.02, + "p95_ms": 7206.92 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 5 + }, + { + "runs": [ + { + "added_latency_ms": 2100.68, + "encode_disabled": { + "actual_audio_seconds": 10.0, + "audio_bytes": 160958, + "latency_ms": 2924.64, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 160958, + "complete_latency_ms": 5025.32, + "encode_latency_ms": 2100.68, + "status": "PASS" + }, + "increase_percent_vs_disabled": 71.83, + "iteration": 1 + }, + { + "added_latency_ms": 2019.88, + "encode_disabled": { + "actual_audio_seconds": 10.0, + "audio_bytes": 160958, + "latency_ms": 2986.2, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 160958, + "complete_latency_ms": 5006.08, + "encode_latency_ms": 2019.88, + "status": "PASS" + }, + "increase_percent_vs_disabled": 67.64, + "iteration": 2 + }, + { + "added_latency_ms": 2035.05, + "encode_disabled": { + "actual_audio_seconds": 10.0, + "audio_bytes": 160958, + "latency_ms": 3107.87, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 160958, + "complete_latency_ms": 5142.92, + "encode_latency_ms": 2035.05, + "status": "PASS" + }, + "increase_percent_vs_disabled": 65.48, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 10.0, + "max": 10.0, + "min": 10.0 + }, + "added_by_encode": { + "average_ms": 2051.87, + "max_ms": 2100.68, + "min_ms": 2019.88, + "p50_ms": 2035.05, + "p95_ms": 2094.12 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 3006.24, + "max_ms": 3107.87, + "min_ms": 2924.64, + "p50_ms": 2986.2, + "p95_ms": 3095.7 + }, + "encode_enabled": { + "average_ms": 5058.11, + "max_ms": 5142.92, + "min_ms": 5006.08, + "p50_ms": 5025.32, + "p95_ms": 5131.16 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 10 + }, + { + "runs": [ + { + "added_latency_ms": 3129.52, + "encode_disabled": { + "actual_audio_seconds": 30.0, + "audio_bytes": 481115, + "latency_ms": 4118.12, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 7247.64, + "encode_latency_ms": 3129.52, + "status": "PASS" + }, + "increase_percent_vs_disabled": 75.99, + "iteration": 1 + }, + { + "added_latency_ms": 3033.33, + "encode_disabled": { + "actual_audio_seconds": 30.0, + "audio_bytes": 481115, + "latency_ms": 4428.53, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 7461.86, + "encode_latency_ms": 3033.33, + "status": "PASS" + }, + "increase_percent_vs_disabled": 68.5, + "iteration": 2 + }, + { + "added_latency_ms": 3021.38, + "encode_disabled": { + "actual_audio_seconds": 30.0, + "audio_bytes": 481115, + "latency_ms": 4087.63, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 7109.01, + "encode_latency_ms": 3021.38, + "status": "PASS" + }, + "increase_percent_vs_disabled": 73.92, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 30.0, + "max": 30.0, + "min": 30.0 + }, + "added_by_encode": { + "average_ms": 3061.41, + "max_ms": 3129.52, + "min_ms": 3021.38, + "p50_ms": 3033.33, + "p95_ms": 3119.9 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 4211.43, + "max_ms": 4428.53, + "min_ms": 4087.63, + "p50_ms": 4118.12, + "p95_ms": 4397.49 + }, + "encode_enabled": { + "average_ms": 7272.84, + "max_ms": 7461.86, + "min_ms": 7109.01, + "p50_ms": 7247.64, + "p95_ms": 7440.44 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 30 + } + ] + }, + { + "service": "text_to_dialogue", + "tiers": [ + { + "runs": [ + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.64, + "audio_bytes": 43511, + "latency_ms": 1131.24, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 127.18, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 1 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 2.72, + "audio_bytes": 44765, + "latency_ms": 1273.65, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 129.53, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 3.04, + "audio_bytes": 49781, + "latency_ms": 1205.57, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 192.33, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 2.8, + "max": 3.04, + "min": 2.64 + }, + "added_by_encode": null, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1203.49, + "max_ms": 1273.65, + "min_ms": 1131.24, + "p50_ms": 1205.57, + "p95_ms": 1266.84 + }, + "encode_enabled": null, + "failed_pairs": 3, + "successful_pairs": 0 + }, + "target_seconds": 2 + }, + { + "runs": [ + { + "added_latency_ms": 1858.78, + "encode_disabled": { + "actual_audio_seconds": 4.96, + "audio_bytes": 80292, + "latency_ms": 1824.2, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 80292, + "complete_latency_ms": 3682.97, + "encode_latency_ms": 1858.78, + "status": "PASS" + }, + "increase_percent_vs_disabled": 101.9, + "iteration": 1 + }, + { + "added_latency_ms": 1758.39, + "encode_disabled": { + "actual_audio_seconds": 5.44, + "audio_bytes": 88233, + "latency_ms": 2173.76, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 88233, + "complete_latency_ms": 3932.16, + "encode_latency_ms": 1758.39, + "status": "PASS" + }, + "increase_percent_vs_disabled": 80.89, + "iteration": 2 + }, + { + "added_latency_ms": 1775.37, + "encode_disabled": { + "actual_audio_seconds": 5.36, + "audio_bytes": 86979, + "latency_ms": 2206.35, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 86979, + "complete_latency_ms": 3981.73, + "encode_latency_ms": 1775.37, + "status": "PASS" + }, + "increase_percent_vs_disabled": 80.47, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 5.253, + "max": 5.44, + "min": 4.96 + }, + "added_by_encode": { + "average_ms": 1797.51, + "max_ms": 1858.78, + "min_ms": 1758.39, + "p50_ms": 1775.37, + "p95_ms": 1850.44 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 2068.1, + "max_ms": 2206.35, + "min_ms": 1824.2, + "p50_ms": 2173.76, + "p95_ms": 2203.09 + }, + "encode_enabled": { + "average_ms": 3865.62, + "max_ms": 3981.73, + "min_ms": 3682.97, + "p50_ms": 3932.16, + "p95_ms": 3976.77 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 5 + }, + { + "runs": [ + { + "added_latency_ms": 1976.1, + "encode_disabled": { + "actual_audio_seconds": 10.64, + "audio_bytes": 171407, + "latency_ms": 3665.53, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 171407, + "complete_latency_ms": 5641.63, + "encode_latency_ms": 1976.1, + "status": "PASS" + }, + "increase_percent_vs_disabled": 53.91, + "iteration": 1 + }, + { + "added_latency_ms": 2025.01, + "encode_disabled": { + "actual_audio_seconds": 10.4, + "audio_bytes": 167645, + "latency_ms": 3216.89, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 167645, + "complete_latency_ms": 5241.9, + "encode_latency_ms": 2025.01, + "status": "PASS" + }, + "increase_percent_vs_disabled": 62.95, + "iteration": 2 + }, + { + "added_latency_ms": 2033.1, + "encode_disabled": { + "actual_audio_seconds": 10.64, + "audio_bytes": 171407, + "latency_ms": 3616.04, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 171407, + "complete_latency_ms": 5649.14, + "encode_latency_ms": 2033.1, + "status": "PASS" + }, + "increase_percent_vs_disabled": 56.22, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 10.56, + "max": 10.64, + "min": 10.4 + }, + "added_by_encode": { + "average_ms": 2011.4, + "max_ms": 2033.1, + "min_ms": 1976.1, + "p50_ms": 2025.01, + "p95_ms": 2032.29 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 3499.49, + "max_ms": 3665.53, + "min_ms": 3216.89, + "p50_ms": 3616.04, + "p95_ms": 3660.58 + }, + "encode_enabled": { + "average_ms": 5510.89, + "max_ms": 5649.14, + "min_ms": 5241.9, + "p50_ms": 5641.63, + "p95_ms": 5648.39 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 10 + }, + { + "runs": [ + { + "added_latency_ms": 3237.93, + "encode_disabled": { + "actual_audio_seconds": 38.56, + "audio_bytes": 618205, + "latency_ms": 12642.65, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 618205, + "complete_latency_ms": 15880.58, + "encode_latency_ms": 3237.93, + "status": "PASS" + }, + "increase_percent_vs_disabled": 25.61, + "iteration": 1 + }, + { + "added_latency_ms": 3382.24, + "encode_disabled": { + "actual_audio_seconds": 37.84, + "audio_bytes": 606502, + "latency_ms": 12667.67, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 606502, + "complete_latency_ms": 16049.92, + "encode_latency_ms": 3382.24, + "status": "PASS" + }, + "increase_percent_vs_disabled": 26.7, + "iteration": 2 + }, + { + "added_latency_ms": 3303.96, + "encode_disabled": { + "actual_audio_seconds": 36.64, + "audio_bytes": 587276, + "latency_ms": 12887.81, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 587276, + "complete_latency_ms": 16191.77, + "encode_latency_ms": 3303.96, + "status": "PASS" + }, + "increase_percent_vs_disabled": 25.64, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 37.68, + "max": 38.56, + "min": 36.64 + }, + "added_by_encode": { + "average_ms": 3308.04, + "max_ms": 3382.24, + "min_ms": 3237.93, + "p50_ms": 3303.96, + "p95_ms": 3374.41 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 12732.71, + "max_ms": 12887.81, + "min_ms": 12642.65, + "p50_ms": 12667.67, + "p95_ms": 12865.8 + }, + "encode_enabled": { + "average_ms": 16040.76, + "max_ms": 16191.77, + "min_ms": 15880.58, + "p50_ms": 16049.92, + "p95_ms": 16177.59 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 30 + } + ] + }, + { + "service": "audio_isolation", + "tiers": [ + { + "runs": [ + { + "added_latency_ms": null, + "encode_disabled": { + "code": null, + "elapsed_until_failure_ms": 171.21, + "error_type": "ApiError", + "reason": "headers: {'date': 'Mon, 31 Aug 2026 19:32:07 GMT', 'server': 'uvicorn', 'content-length': '247', 'content-type': 'application/json', 'vary': 'Accept-Language', 'access-control-allow-origin': '*', 'access-control-allow-headers': '*', 'access-control-allow-methods': 'POST, PATCH, OPTIONS, DELETE, GET, PUT', 'access-control-max-age': '600', 'strict-transport-security': 'max-age=1800;', 'x-trace-id': 'fdc7f96f648e7bdddcd15176a4a75c78', 'x-region': 'us-central1', 'via': '1.1 google', 'alt-svc': 'h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000'}, status_code: 400, body: {'detail': {'type': 'validation_error', 'code': 'audio_too_short', 'message': 'Audio duration is 2.0 seconds, which is below the minimum of 4.6 seconds.', 'status': 'invalid_audio_duration', 'request_id': 'fdc7f96f648e7bdddcd15176a4a75c78', 'param': 'audio'}}", + "status": "FAIL", + "status_code": 400 + }, + "encode_enabled": { + "reason": "native service request failed", + "status": "SKIP" + }, + "increase_percent_vs_disabled": null, + "iteration": 1 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "code": null, + "elapsed_until_failure_ms": 112.67, + "error_type": "ApiError", + "reason": "headers: {'date': 'Mon, 31 Aug 2026 19:32:07 GMT', 'server': 'uvicorn', 'content-length': '247', 'content-type': 'application/json', 'vary': 'Accept-Language', 'access-control-allow-origin': '*', 'access-control-allow-headers': '*', 'access-control-allow-methods': 'POST, PATCH, OPTIONS, DELETE, GET, PUT', 'access-control-max-age': '600', 'strict-transport-security': 'max-age=1800;', 'x-trace-id': '5a3b1863b8c1cd63dcd15176a4a75d5a', 'x-region': 'us-central1', 'via': '1.1 google', 'alt-svc': 'h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000'}, status_code: 400, body: {'detail': {'type': 'validation_error', 'code': 'audio_too_short', 'message': 'Audio duration is 2.0 seconds, which is below the minimum of 4.6 seconds.', 'status': 'invalid_audio_duration', 'request_id': '5a3b1863b8c1cd63dcd15176a4a75d5a', 'param': 'audio'}}", + "status": "FAIL", + "status_code": 400 + }, + "encode_enabled": { + "reason": "native service request failed", + "status": "SKIP" + }, + "increase_percent_vs_disabled": null, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "code": null, + "elapsed_until_failure_ms": 212.79, + "error_type": "ApiError", + "reason": "headers: {'date': 'Mon, 31 Aug 2026 19:32:07 GMT', 'server': 'uvicorn', 'content-length': '247', 'content-type': 'application/json', 'vary': 'Accept-Language', 'access-control-allow-origin': '*', 'access-control-allow-headers': '*', 'access-control-allow-methods': 'POST, PATCH, OPTIONS, DELETE, GET, PUT', 'access-control-max-age': '600', 'strict-transport-security': 'max-age=1800;', 'x-trace-id': 'b4097f91963b91b0dcd15176a4a75c55', 'x-region': 'us-central1', 'via': '1.1 google', 'alt-svc': 'h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000'}, status_code: 400, body: {'detail': {'type': 'validation_error', 'code': 'audio_too_short', 'message': 'Audio duration is 2.0 seconds, which is below the minimum of 4.6 seconds.', 'status': 'invalid_audio_duration', 'request_id': 'b4097f91963b91b0dcd15176a4a75c55', 'param': 'audio'}}", + "status": "FAIL", + "status_code": 400 + }, + "encode_enabled": { + "reason": "native service request failed", + "status": "SKIP" + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": null, + "added_by_encode": null, + "attempted_pairs": 3, + "encode_disabled": null, + "encode_enabled": null, + "failed_pairs": 3, + "successful_pairs": 0 + }, + "target_seconds": 2 + }, + { + "runs": [ + { + "added_latency_ms": 2086.8, + "encode_disabled": { + "actual_audio_seconds": 4.992, + "audio_bytes": 202754, + "latency_ms": 852.21, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 202754, + "complete_latency_ms": 2939.02, + "encode_latency_ms": 2086.8, + "status": "PASS" + }, + "increase_percent_vs_disabled": 244.87, + "iteration": 1 + }, + { + "added_latency_ms": 2155.57, + "encode_disabled": { + "actual_audio_seconds": 4.992, + "audio_bytes": 202754, + "latency_ms": 797.08, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 202754, + "complete_latency_ms": 2952.65, + "encode_latency_ms": 2155.57, + "status": "PASS" + }, + "increase_percent_vs_disabled": 270.43, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 4.992, + "audio_bytes": 202754, + "latency_ms": 779.68, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 1469.94, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio input is not supported by Audio Encode V2", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 4.992, + "max": 4.992, + "min": 4.992 + }, + "added_by_encode": { + "average_ms": 2121.19, + "max_ms": 2155.57, + "min_ms": 2086.8, + "p50_ms": 2121.19, + "p95_ms": 2152.13 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 809.66, + "max_ms": 852.21, + "min_ms": 779.68, + "p50_ms": 797.08, + "p95_ms": 846.7 + }, + "encode_enabled": { + "average_ms": 2945.84, + "max_ms": 2952.65, + "min_ms": 2939.02, + "p50_ms": 2945.84, + "p95_ms": 2951.97 + }, + "failed_pairs": 1, + "successful_pairs": 2 + }, + "target_seconds": 5 + }, + { + "runs": [ + { + "added_latency_ms": 2753.21, + "encode_disabled": { + "actual_audio_seconds": 9.985, + "audio_bytes": 402329, + "latency_ms": 1129.25, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 402329, + "complete_latency_ms": 3882.47, + "encode_latency_ms": 2753.21, + "status": "PASS" + }, + "increase_percent_vs_disabled": 243.81, + "iteration": 1 + }, + { + "added_latency_ms": 2622.44, + "encode_disabled": { + "actual_audio_seconds": 9.985, + "audio_bytes": 402329, + "latency_ms": 1036.95, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 402329, + "complete_latency_ms": 3659.39, + "encode_latency_ms": 2622.44, + "status": "PASS" + }, + "increase_percent_vs_disabled": 252.9, + "iteration": 2 + }, + { + "added_latency_ms": 2772.87, + "encode_disabled": { + "actual_audio_seconds": 9.985, + "audio_bytes": 402329, + "latency_ms": 861.74, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 402329, + "complete_latency_ms": 3634.61, + "encode_latency_ms": 2772.87, + "status": "PASS" + }, + "increase_percent_vs_disabled": 321.78, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 9.985, + "max": 9.985, + "min": 9.985 + }, + "added_by_encode": { + "average_ms": 2716.17, + "max_ms": 2772.87, + "min_ms": 2622.44, + "p50_ms": 2753.21, + "p95_ms": 2770.9 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1009.31, + "max_ms": 1129.25, + "min_ms": 861.74, + "p50_ms": 1036.95, + "p95_ms": 1120.02 + }, + "encode_enabled": { + "average_ms": 3725.49, + "max_ms": 3882.47, + "min_ms": 3634.61, + "p50_ms": 3659.39, + "p95_ms": 3860.16 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 10 + }, + { + "runs": [ + { + "added_latency_ms": 5069.14, + "encode_disabled": { + "actual_audio_seconds": 29.977, + "audio_bytes": 1201676, + "latency_ms": 1631.36, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 1201676, + "complete_latency_ms": 6700.5, + "encode_latency_ms": 5069.14, + "status": "PASS" + }, + "increase_percent_vs_disabled": 310.73, + "iteration": 1 + }, + { + "added_latency_ms": 5159.2, + "encode_disabled": { + "actual_audio_seconds": 29.977, + "audio_bytes": 1201676, + "latency_ms": 1954.54, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 1201676, + "complete_latency_ms": 7113.74, + "encode_latency_ms": 5159.2, + "status": "PASS" + }, + "increase_percent_vs_disabled": 263.96, + "iteration": 2 + }, + { + "added_latency_ms": 5033.19, + "encode_disabled": { + "actual_audio_seconds": 29.977, + "audio_bytes": 1201676, + "latency_ms": 1674.22, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 1201676, + "complete_latency_ms": 6707.41, + "encode_latency_ms": 5033.19, + "status": "PASS" + }, + "increase_percent_vs_disabled": 300.63, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 29.977, + "max": 29.977, + "min": 29.977 + }, + "added_by_encode": { + "average_ms": 5087.18, + "max_ms": 5159.2, + "min_ms": 5033.19, + "p50_ms": 5069.14, + "p95_ms": 5150.19 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 1753.37, + "max_ms": 1954.54, + "min_ms": 1631.36, + "p50_ms": 1674.22, + "p95_ms": 1926.51 + }, + "encode_enabled": { + "average_ms": 6840.55, + "max_ms": 7113.74, + "min_ms": 6700.5, + "p50_ms": 6707.41, + "p95_ms": 7073.11 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 30 + } + ] + }, + { + "service": "music_compose", + "tiers": [ + { + "runs": [ + { + "added_latency_ms": null, + "encode_disabled": { + "code": null, + "elapsed_until_failure_ms": 180.96, + "error_type": "UnprocessableEntityError", + "reason": "headers: {'date': 'Mon, 31 Aug 2026 19:32:47 GMT', 'server': 'uvicorn', 'content-length': '160', 'content-type': 'application/json', 'vary': 'Accept-Language', 'access-control-allow-origin': '*', 'access-control-allow-headers': '*', 'access-control-allow-methods': 'POST, PATCH, OPTIONS, DELETE, GET, PUT', 'access-control-max-age': '600', 'strict-transport-security': 'max-age=1800;', 'x-trace-id': '3674efbdf8c0bf41754006ef5ad00195', 'x-region': 'us-central1', 'via': '1.1 google', 'alt-svc': 'h3=\":443\"; ma=2592000'}, status_code: 422, body: {'detail': [{'type': 'greater_than_equal', 'loc': ['body', 'music_length_ms'], 'msg': 'Input should be greater than or equal to 3000', 'input': 2000, 'ctx': {'ge': 3000}}]}", + "status": "FAIL", + "status_code": 422 + }, + "encode_enabled": { + "reason": "native service request failed", + "status": "SKIP" + }, + "increase_percent_vs_disabled": null, + "iteration": 1 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "code": null, + "elapsed_until_failure_ms": 65.58, + "error_type": "UnprocessableEntityError", + "reason": "headers: {'date': 'Mon, 31 Aug 2026 19:32:47 GMT', 'server': 'uvicorn', 'content-length': '160', 'content-type': 'application/json', 'vary': 'Accept-Language', 'access-control-allow-origin': '*', 'access-control-allow-headers': '*', 'access-control-allow-methods': 'POST, PATCH, OPTIONS, DELETE, GET, PUT', 'access-control-max-age': '600', 'strict-transport-security': 'max-age=1800;', 'x-trace-id': '16cb0a8ec8b2f6d0754006ef5ad00696', 'x-region': 'us-central1', 'via': '1.1 google', 'alt-svc': 'h3=\":443\"; ma=2592000'}, status_code: 422, body: {'detail': [{'type': 'greater_than_equal', 'loc': ['body', 'music_length_ms'], 'msg': 'Input should be greater than or equal to 3000', 'input': 2000, 'ctx': {'ge': 3000}}]}", + "status": "FAIL", + "status_code": 422 + }, + "encode_enabled": { + "reason": "native service request failed", + "status": "SKIP" + }, + "increase_percent_vs_disabled": null, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "code": null, + "elapsed_until_failure_ms": 66.85, + "error_type": "UnprocessableEntityError", + "reason": "headers: {'date': 'Mon, 31 Aug 2026 19:32:47 GMT', 'server': 'uvicorn', 'content-length': '160', 'content-type': 'application/json', 'vary': 'Accept-Language', 'access-control-allow-origin': '*', 'access-control-allow-headers': '*', 'access-control-allow-methods': 'POST, PATCH, OPTIONS, DELETE, GET, PUT', 'access-control-max-age': '600', 'strict-transport-security': 'max-age=1800;', 'x-trace-id': 'd1dbe997b156995b754006ef5ad00751', 'x-region': 'us-central1', 'via': '1.1 google', 'alt-svc': 'h3=\":443\"; ma=2592000'}, status_code: 422, body: {'detail': [{'type': 'greater_than_equal', 'loc': ['body', 'music_length_ms'], 'msg': 'Input should be greater than or equal to 3000', 'input': 2000, 'ctx': {'ge': 3000}}]}", + "status": "FAIL", + "status_code": 422 + }, + "encode_enabled": { + "reason": "native service request failed", + "status": "SKIP" + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": null, + "added_by_encode": null, + "attempted_pairs": 3, + "encode_disabled": null, + "encode_enabled": null, + "failed_pairs": 3, + "successful_pairs": 0 + }, + "target_seconds": 2 + }, + { + "runs": [ + { + "added_latency_ms": 1834.36, + "encode_disabled": { + "actual_audio_seconds": 5.042, + "audio_bytes": 80711, + "latency_ms": 4319.6, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81964, + "complete_latency_ms": 6153.96, + "encode_latency_ms": 1834.36, + "status": "PASS" + }, + "increase_percent_vs_disabled": 42.47, + "iteration": 1 + }, + { + "added_latency_ms": 1828.56, + "encode_disabled": { + "actual_audio_seconds": 6.087, + "audio_bytes": 97429, + "latency_ms": 4236.66, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 98682, + "complete_latency_ms": 6065.22, + "encode_latency_ms": 1828.56, + "status": "PASS" + }, + "increase_percent_vs_disabled": 43.16, + "iteration": 2 + }, + { + "added_latency_ms": 1851.03, + "encode_disabled": { + "actual_audio_seconds": 6.087, + "audio_bytes": 97429, + "latency_ms": 4648.25, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 98682, + "complete_latency_ms": 6499.28, + "encode_latency_ms": 1851.03, + "status": "PASS" + }, + "increase_percent_vs_disabled": 39.82, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 5.739, + "max": 6.087, + "min": 5.042 + }, + "added_by_encode": { + "average_ms": 1837.98, + "max_ms": 1851.03, + "min_ms": 1828.56, + "p50_ms": 1834.36, + "p95_ms": 1849.36 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 4401.5, + "max_ms": 4648.25, + "min_ms": 4236.66, + "p50_ms": 4319.6, + "p95_ms": 4615.39 + }, + "encode_enabled": { + "average_ms": 6239.49, + "max_ms": 6499.28, + "min_ms": 6065.22, + "p50_ms": 6153.96, + "p95_ms": 6464.75 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 5 + }, + { + "runs": [ + { + "added_latency_ms": 2051.38, + "encode_disabled": { + "actual_audio_seconds": 10.057, + "audio_bytes": 160959, + "latency_ms": 4531.49, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 6582.88, + "encode_latency_ms": 2051.38, + "status": "PASS" + }, + "increase_percent_vs_disabled": 45.27, + "iteration": 1 + }, + { + "added_latency_ms": 2321.59, + "encode_disabled": { + "actual_audio_seconds": 10.031, + "audio_bytes": 160541, + "latency_ms": 7051.35, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 161794, + "complete_latency_ms": 9372.95, + "encode_latency_ms": 2321.59, + "status": "PASS" + }, + "increase_percent_vs_disabled": 32.92, + "iteration": 2 + }, + { + "added_latency_ms": 2180.95, + "encode_disabled": { + "actual_audio_seconds": 10.057, + "audio_bytes": 160959, + "latency_ms": 5365.41, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 7546.36, + "encode_latency_ms": 2180.95, + "status": "PASS" + }, + "increase_percent_vs_disabled": 40.65, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 10.048, + "max": 10.057, + "min": 10.031 + }, + "added_by_encode": { + "average_ms": 2184.64, + "max_ms": 2321.59, + "min_ms": 2051.38, + "p50_ms": 2180.95, + "p95_ms": 2307.53 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 5649.42, + "max_ms": 7051.35, + "min_ms": 4531.49, + "p50_ms": 5365.41, + "p95_ms": 6882.76 + }, + "encode_enabled": { + "average_ms": 7834.06, + "max_ms": 9372.95, + "min_ms": 6582.88, + "p50_ms": 7546.36, + "p95_ms": 9190.29 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 10 + }, + { + "runs": [ + { + "added_latency_ms": 3312.25, + "encode_disabled": { + "actual_audio_seconds": 29.989, + "audio_bytes": 479862, + "latency_ms": 6949.21, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 10261.46, + "encode_latency_ms": 3312.25, + "status": "PASS" + }, + "increase_percent_vs_disabled": 47.66, + "iteration": 1 + }, + { + "added_latency_ms": 3132.18, + "encode_disabled": { + "actual_audio_seconds": 29.989, + "audio_bytes": 479862, + "latency_ms": 8923.83, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 12056.01, + "encode_latency_ms": 3132.18, + "status": "PASS" + }, + "increase_percent_vs_disabled": 35.1, + "iteration": 2 + }, + { + "added_latency_ms": 3275.32, + "encode_disabled": { + "actual_audio_seconds": 29.989, + "audio_bytes": 479862, + "latency_ms": 6464.48, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481115, + "complete_latency_ms": 9739.8, + "encode_latency_ms": 3275.32, + "status": "PASS" + }, + "increase_percent_vs_disabled": 50.67, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 29.989, + "max": 29.989, + "min": 29.989 + }, + "added_by_encode": { + "average_ms": 3239.92, + "max_ms": 3312.25, + "min_ms": 3132.18, + "p50_ms": 3275.32, + "p95_ms": 3308.56 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 7445.84, + "max_ms": 8923.83, + "min_ms": 6464.48, + "p50_ms": 6949.21, + "p95_ms": 8726.37 + }, + "encode_enabled": { + "average_ms": 10685.76, + "max_ms": 12056.01, + "min_ms": 9739.8, + "p50_ms": 10261.46, + "p95_ms": 11876.56 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 30 + } + ] + }, + { + "service": "video_to_music", + "tiers": [ + { + "runs": [ + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 3.056, + "audio_bytes": 48946, + "latency_ms": 9822.6, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 207.09, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 1 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 3.056, + "audio_bytes": 48946, + "latency_ms": 9712.56, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 246.66, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 2 + }, + { + "added_latency_ms": null, + "encode_disabled": { + "actual_audio_seconds": 3.056, + "audio_bytes": 48946, + "latency_ms": 7350.83, + "status": "PASS" + }, + "encode_enabled": { + "code": "PROC_4002", + "elapsed_until_failure_ms": 301.23, + "error_type": "ValidationError", + "reason": "[PROC_4002] Audio is too short for this token. Please use a longer audio clip.", + "status": "FAIL", + "status_code": 400 + }, + "increase_percent_vs_disabled": null, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 3.056, + "max": 3.056, + "min": 3.056 + }, + "added_by_encode": null, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 8962.0, + "max_ms": 9822.6, + "min_ms": 7350.83, + "p50_ms": 9712.56, + "p95_ms": 9811.6 + }, + "encode_enabled": null, + "failed_pairs": 3, + "successful_pairs": 0 + }, + "target_seconds": 2 + }, + { + "runs": [ + { + "added_latency_ms": 2143.82, + "encode_disabled": { + "actual_audio_seconds": 6.087, + "audio_bytes": 97429, + "latency_ms": 11117.92, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 98682, + "complete_latency_ms": 13261.75, + "encode_latency_ms": 2143.82, + "status": "PASS" + }, + "increase_percent_vs_disabled": 19.28, + "iteration": 1 + }, + { + "added_latency_ms": 2116.24, + "encode_disabled": { + "actual_audio_seconds": 9.091, + "audio_bytes": 145494, + "latency_ms": 12247.34, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 146747, + "complete_latency_ms": 14363.57, + "encode_latency_ms": 2116.24, + "status": "PASS" + }, + "increase_percent_vs_disabled": 17.28, + "iteration": 2 + }, + { + "added_latency_ms": 2083.3, + "encode_disabled": { + "actual_audio_seconds": 5.042, + "audio_bytes": 80711, + "latency_ms": 8690.5, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 81964, + "complete_latency_ms": 10773.79, + "encode_latency_ms": 2083.3, + "status": "PASS" + }, + "increase_percent_vs_disabled": 23.97, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 6.74, + "max": 9.091, + "min": 5.042 + }, + "added_by_encode": { + "average_ms": 2114.45, + "max_ms": 2143.82, + "min_ms": 2083.3, + "p50_ms": 2116.24, + "p95_ms": 2141.06 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 10685.25, + "max_ms": 12247.34, + "min_ms": 8690.5, + "p50_ms": 11117.92, + "p95_ms": 12134.4 + }, + "encode_enabled": { + "average_ms": 12799.7, + "max_ms": 14363.57, + "min_ms": 10773.79, + "p50_ms": 13261.75, + "p95_ms": 14253.39 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 5 + }, + { + "runs": [ + { + "added_latency_ms": 2175.56, + "encode_disabled": { + "actual_audio_seconds": 10.057, + "audio_bytes": 160959, + "latency_ms": 12943.7, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 15119.26, + "encode_latency_ms": 2175.56, + "status": "PASS" + }, + "increase_percent_vs_disabled": 16.81, + "iteration": 1 + }, + { + "added_latency_ms": 2244.05, + "encode_disabled": { + "actual_audio_seconds": 10.057, + "audio_bytes": 160959, + "latency_ms": 16027.42, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 18271.47, + "encode_latency_ms": 2244.05, + "status": "PASS" + }, + "increase_percent_vs_disabled": 14.0, + "iteration": 2 + }, + { + "added_latency_ms": 2217.17, + "encode_disabled": { + "actual_audio_seconds": 10.057, + "audio_bytes": 160959, + "latency_ms": 14479.25, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 162212, + "complete_latency_ms": 16696.42, + "encode_latency_ms": 2217.17, + "status": "PASS" + }, + "increase_percent_vs_disabled": 15.31, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 10.057, + "max": 10.057, + "min": 10.057 + }, + "added_by_encode": { + "average_ms": 2212.26, + "max_ms": 2244.05, + "min_ms": 2175.56, + "p50_ms": 2217.17, + "p95_ms": 2241.36 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 14483.46, + "max_ms": 16027.42, + "min_ms": 12943.7, + "p50_ms": 14479.25, + "p95_ms": 15872.6 + }, + "encode_enabled": { + "average_ms": 16695.72, + "max_ms": 18271.47, + "min_ms": 15119.26, + "p50_ms": 16696.42, + "p95_ms": 18113.97 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 10 + }, + { + "runs": [ + { + "added_latency_ms": 4200.98, + "encode_disabled": { + "actual_audio_seconds": 48.065, + "audio_bytes": 769089, + "latency_ms": 10537.01, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 770342, + "complete_latency_ms": 14737.98, + "encode_latency_ms": 4200.98, + "status": "PASS" + }, + "increase_percent_vs_disabled": 39.87, + "iteration": 1 + }, + { + "added_latency_ms": 3290.45, + "encode_disabled": { + "actual_audio_seconds": 30.041, + "audio_bytes": 480698, + "latency_ms": 13977.48, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481951, + "complete_latency_ms": 17267.94, + "encode_latency_ms": 3290.45, + "status": "PASS" + }, + "increase_percent_vs_disabled": 23.54, + "iteration": 2 + }, + { + "added_latency_ms": 3388.1, + "encode_disabled": { + "actual_audio_seconds": 30.041, + "audio_bytes": 480698, + "latency_ms": 17118.43, + "status": "PASS" + }, + "encode_enabled": { + "audio_bytes": 481951, + "complete_latency_ms": 20506.53, + "encode_latency_ms": 3388.1, + "status": "PASS" + }, + "increase_percent_vs_disabled": 19.79, + "iteration": 3 + } + ], + "summary": { + "actual_audio_seconds": { + "average": 36.049, + "max": 48.065, + "min": 30.041 + }, + "added_by_encode": { + "average_ms": 3626.51, + "max_ms": 4200.98, + "min_ms": 3290.45, + "p50_ms": 3388.1, + "p95_ms": 4119.69 + }, + "attempted_pairs": 3, + "encode_disabled": { + "average_ms": 13877.64, + "max_ms": 17118.43, + "min_ms": 10537.01, + "p50_ms": 13977.48, + "p95_ms": 16804.33 + }, + "encode_enabled": { + "average_ms": 17504.15, + "max_ms": 20506.53, + "min_ms": 14737.98, + "p50_ms": 17267.94, + "p95_ms": 20182.67 + }, + "failed_pairs": 0, + "successful_pairs": 3 + }, + "target_seconds": 30 + } + ] + } + ] +} diff --git a/benchmarks/results/elevenlabs_services_duration_latency_20260831T192903Z.md b/benchmarks/results/elevenlabs_services_duration_latency_20260831T192903Z.md new file mode 100644 index 0000000..0eddff2 --- /dev/null +++ b/benchmarks/results/elevenlabs_services_duration_latency_20260831T192903Z.md @@ -0,0 +1,52 @@ +# ElevenLabs services duration latency benchmark + +- Run completed: 2026-08-31 +- Duration targets: 2s, 5s, 10s, 30s +- Consecutive runs per target: 3 +- Measurement order: call the native ElevenLabs service once, then encode the exact returned audio bytes +- `encode on` = native service latency + OfSpectrum encode latency +- Detailed per-run data: `elevenlabs_services_duration_latency_20260831T192903Z.json` + +All values below are three-run averages in milliseconds. `Added` is the OfSpectrum encode latency. Percentages compare the average added latency with the average native-service latency. Actual output duration can differ from the target for generative services. + +| Service | Target | Actual output | Encode off | Encode on | Added | Increase | Successful pairs | +|---|---:|---:|---:|---:|---:|---:|---:| +| Speech-to-Speech | 2s | 2.043s | 1,130.24 | — | — | — | 0/3 | +| Speech-to-Speech | 5s | 5.016s | 1,225.76 | 3,087.14 | 1,861.37 | 151.9% | 3/3 | +| Speech-to-Speech | 10s | 10.031s | 1,593.20 | 3,597.42 | 2,004.22 | 125.8% | 3/3 | +| Speech-to-Speech | 30s | 30.000s | 4,536.01 | 7,367.41 | 2,831.41 | 62.4% | 3/3 | +| Sound Effects | 2s | 2.000s | 1,941.31 | — | — | — | 0/3 | +| Sound Effects | 5s | 5.000s | 2,978.45 | 4,890.73 | 1,912.27 | 64.2% | 3/3 | +| Sound Effects | 10s | 10.000s | 3,006.24 | 5,058.11 | 2,051.87 | 68.3% | 3/3 | +| Sound Effects | 30s | 30.000s | 4,211.43 | 7,272.84 | 3,061.41 | 72.7% | 3/3 | +| Text-to-Dialogue | 2s | 2.800s | 1,203.49 | — | — | — | 0/3 | +| Text-to-Dialogue | 5s | 5.253s | 2,068.10 | 3,865.62 | 1,797.51 | 86.9% | 3/3 | +| Text-to-Dialogue | 10s | 10.560s | 3,499.49 | 5,510.89 | 2,011.40 | 57.5% | 3/3 | +| Text-to-Dialogue | 30s | 37.680s | 12,732.71 | 16,040.76 | 3,308.04 | 26.0% | 3/3 | +| Audio Isolation | 2s | — | — | — | — | — | 0/3 | +| Audio Isolation | 5s | 4.992s | 809.66 | 2,945.84 | 2,121.19 | 262.0% | 2/3 | +| Audio Isolation | 10s | 9.985s | 1,009.31 | 3,725.49 | 2,716.17 | 269.1% | 3/3 | +| Audio Isolation | 30s | 29.977s | 1,753.37 | 6,840.55 | 5,087.18 | 290.1% | 3/3 | +| Music Compose | 2s | — | — | — | — | — | 0/3 | +| Music Compose | 5s | 5.739s | 4,401.50 | 6,239.49 | 1,837.98 | 41.8% | 3/3 | +| Music Compose | 10s | 10.048s | 5,649.42 | 7,834.06 | 2,184.64 | 38.7% | 3/3 | +| Music Compose | 30s | 29.989s | 7,445.84 | 10,685.76 | 3,239.92 | 43.5% | 3/3 | +| Video-to-Music | 2s | 3.056s | 8,962.00 | — | — | — | 0/3 | +| Video-to-Music | 5s | 6.740s | 10,685.25 | 12,799.70 | 2,114.45 | 19.8% | 3/3 | +| Video-to-Music | 10s | 10.057s | 14,483.46 | 16,695.72 | 2,212.26 | 15.3% | 3/3 | +| Video-to-Music | 30s | 36.049s | 13,877.64 | 17,504.15 | 3,626.51 | 26.1% | 3/3 | + +## Failures and service constraints + +- Speech-to-Speech, Sound Effects, Text-to-Dialogue, and Video-to-Music: all three short-tier encode attempts returned `PROC_4002`, indicating that the resulting audio was too short for this watermark token. +- Audio Isolation 2s: ElevenLabs rejected all three native requests because this endpoint requires at least 4.6 seconds of input audio, so encode was not attempted. +- Music Compose 2s: ElevenLabs rejected all three native requests because `music_length_ms` must be at least 3,000 ms, so encode was not attempted. +- Audio Isolation 5s: two pairs succeeded; one encode attempt returned `PROC_4002: Audio input is not supported by Audio Encode V2`. Its summary is based on the two successful encode pairs, while the native average includes all three native calls. +- No automatic retries were used, so the report preserves service behavior from the original consecutive attempts. + +## Interpretation + +- For successful 5s and 10s generated MP3 results, watermark encoding generally added about 1.8–2.7 seconds. +- For successful 30s MP3 results, it generally added about 2.8–3.6 seconds. +- Audio Isolation returned WAV data and showed a higher encode cost: about 2.1 seconds at 5s, 2.7 seconds at 10s, and 5.1 seconds at 30s. +- Relative percentage depends heavily on how fast the native endpoint is. Audio Isolation has low native latency, so encode dominates the percentage; Video-to-Music has high native latency, so the same few seconds appear as a smaller percentage. diff --git a/ofspectrum/__init__.py b/ofspectrum/__init__.py index 66f4f18..af37402 100644 --- a/ofspectrum/__init__.py +++ b/ofspectrum/__init__.py @@ -33,6 +33,7 @@ __author__ = "OfSpectrum" from .client import AsyncOfSpectrum, OfSpectrum +from .elevenlabs import Ofspectrum, WatermarkController from .exceptions import ( AuthenticationError, ConflictError, @@ -45,6 +46,7 @@ ServiceUnavailableError, TimeoutError, ValidationError, + WatermarkConfigurationError, WatermarkExistsError, ) from .models import ( @@ -79,8 +81,10 @@ __all__ = [ # Client "OfSpectrum", + "Ofspectrum", "AsyncOfSpectrum", "StreamEncodePool", + "WatermarkController", # Exceptions "OfSpectrumError", "AuthenticationError", @@ -89,6 +93,7 @@ "ResourceNotFoundError", "ValidationError", "WatermarkExistsError", + "WatermarkConfigurationError", "TimeoutError", "ServiceUnavailableError", "NetworkError", diff --git a/ofspectrum/elevenlabs.py b/ofspectrum/elevenlabs.py new file mode 100644 index 0000000..2529747 --- /dev/null +++ b/ofspectrum/elevenlabs.py @@ -0,0 +1,448 @@ +"""Transparent ElevenLabs client wrapper with OfSpectrum watermarking. + +This module intentionally has no import-time dependency on ``elevenlabs``. +The wrapper uses duck typing so it can follow the generated ElevenLabs client +surface across compatible SDK releases. +""" + +import functools +import io +import os +import threading +from collections.abc import Iterator as IteratorABC +from typing import Any, Dict, Iterable, Iterator, Optional, Set, Tuple + +from .client import OfSpectrum +from .exceptions import OfSpectrumError, WatermarkConfigurationError + +_UNSET = object() + + +class _NamedAudio(io.BytesIO): + def __init__(self, content: bytes, name: str): + super().__init__(content) + self.name = name + + +# These methods return one complete audio file even though the generated +# ElevenLabs Python SDK commonly exposes that file as Iterator[bytes]. Methods +# that return JSON/timestamps/detailed result objects are deliberately absent. +_DEFAULT_COMPLETE_AUDIO_METHODS = { + "audio_isolation.convert", + "conversational_ai.conversations.audio.get", + "history.get_audio", + "music.compose", + "music.video_to_music", + "sound_generation.text_to_sound_effects", + "speech_to_speech.convert", + "text_to_dialogue.convert", + "text_to_sound_effects.convert", + "text_to_speech.convert", + "voices.samples.audio.get", +} + +_STREAM_METHOD_NAMES = { + "convert_as_stream", + "stream", + "stream_audio", +} + +_ENCODE_OPTION_NAMES = { + "check_watermark", + "interval", + "smooth", + "strength", + "timeout", + "verify_and_reencode", +} + + +def _audio_filename(call_kwargs: Dict[str, Any]) -> str: + output_format = call_kwargs.get("output_format", "mp3_44100_128") + output_format = getattr(output_format, "value", output_format) + value = str(output_format or "mp3_44100_128").lower() + codec = value.split("_", 1)[0] + extension = { + "alaw": "alaw", + "auto": "mp3", + "mp3": "mp3", + "opus": "opus", + "pcm": "pcm", + "ulaw": "ulaw", + "wav": "wav", + }.get(codec, "mp3") + return "elevenlabs-output." + extension + + +def _all_byte_chunks(value: Any) -> bool: + return isinstance(value, (list, tuple)) and all( + isinstance(chunk, (bytes, bytearray, memoryview)) for chunk in value + ) + + +def _is_stream_method_name(name: str) -> bool: + return ( + name in _STREAM_METHOD_NAMES + or name.startswith("stream_") + or name.endswith("_stream") + or "_stream_" in name + ) + + +class WatermarkController: + """Configuration and encode operations for an :class:`Ofspectrum` wrapper. + + Configuration is read as a snapshot for every encode, making calls safe to + use concurrently. Reconfiguring while a call is in flight affects only + subsequent calls. + """ + + def __init__( + self, + *, + client: Optional[Any] = None, + api_key: Optional[str] = None, + token_id: Optional[str] = None, + enabled: bool = True, + encode_options: Optional[Dict[str, Any]] = None, + audio_methods: Optional[Iterable[str]] = None, + ) -> None: + if not isinstance(enabled, bool): + raise ValueError("enabled must be a boolean") + if api_key is not None and ( + not isinstance(api_key, str) or not api_key.strip() + ): + raise ValueError("api_key must be a non-empty string or None") + if token_id is not None and ( + not isinstance(token_id, str) or not token_id.strip() + ): + raise ValueError("token_id must be a non-empty string or None") + self._lock = threading.RLock() + self._client = client + self._owns_client = client is None + self._api_key = api_key.strip() if api_key else os.getenv("OFSPECTRUM_API_KEY") + self._token_id = token_id.strip() if token_id else os.getenv("OFSPECTRUM_TOKEN_ID") + self._enabled = enabled + self._encode_options: Dict[str, Any] = {} + method_source = ( + _DEFAULT_COMPLETE_AUDIO_METHODS if audio_methods is None else audio_methods + ) + self._audio_methods: Set[str] = { + self._normalize_method_path(path) for path in method_source + } + if encode_options: + self._set_encode_options(encode_options) + + @property + def enabled(self) -> bool: + with self._lock: + return self._enabled + + @property + def token_id(self) -> Optional[str]: + with self._lock: + return self._token_id + + @property + def audio_methods(self) -> Tuple[str, ...]: + with self._lock: + return tuple(sorted(self._audio_methods)) + + def config( + self, + *, + enabled: Any = _UNSET, + token_id: Any = _UNSET, + api_key: Any = _UNSET, + client: Any = _UNSET, + **encode_options: Any, + ) -> "WatermarkController": + """Update watermark settings and return this controller. + + Supported encode options currently mirror the safe subset of + ``OfSpectrum.audio.encode``: ``strength``, ``smooth``, ``interval``, + ``timeout``, ``check_watermark``, and ``verify_and_reencode``. + Persistence and response-format options are fixed so the wrapper always + receives encoded bytes and does not create storage as a side effect. + """ + with self._lock: + if enabled is not _UNSET: + if not isinstance(enabled, bool): + raise ValueError("enabled must be a boolean") + self._enabled = enabled + if token_id is not _UNSET: + if token_id is not None and ( + not isinstance(token_id, str) or not token_id.strip() + ): + raise ValueError("token_id must be a non-empty string or None") + self._token_id = token_id.strip() if token_id else None + if api_key is not _UNSET: + if api_key is not None and ( + not isinstance(api_key, str) or not api_key.strip() + ): + raise ValueError("api_key must be a non-empty string or None") + self._replace_owned_client(None) + self._api_key = api_key.strip() if api_key else None + self._owns_client = True + if client is not _UNSET: + self._replace_owned_client(client) + self._owns_client = client is None + self._set_encode_options(encode_options) + return self + + def register_audio_method(self, path: str) -> "WatermarkController": + """Register another dotted ElevenLabs method returning a complete file.""" + normalized = self._normalize_method_path(path) + with self._lock: + self._audio_methods.add(normalized) + return self + + def unregister_audio_method(self, path: str) -> "WatermarkController": + normalized = self._normalize_method_path(path) + with self._lock: + self._audio_methods.discard(normalized) + return self + + def handles(self, path: Tuple[str, ...]) -> bool: + if not path or _is_stream_method_name(path[-1]) or "with_raw_response" in path: + return False + dotted = ".".join(path) + with self._lock: + return self._enabled and dotted in self._audio_methods + + def transform(self, response: Any, call_kwargs: Dict[str, Any]) -> Any: + """Watermark a supported complete-audio response, preserving its shape.""" + filename = _audio_filename(call_kwargs) + if isinstance(response, bytes): + return self.encode_bytes(response, filename=filename) + if isinstance(response, bytearray): + return bytearray(self.encode_bytes(bytes(response), filename=filename)) + if isinstance(response, memoryview): + return memoryview(self.encode_bytes(response.tobytes(), filename=filename)) + if _all_byte_chunks(response): + encoded = self.encode_bytes( + b"".join(bytes(chunk) for chunk in response), filename=filename + ) + return type(response)([encoded]) + if isinstance(response, IteratorABC): + return self._transform_iterator(response, filename) + # JSON, metadata models, job IDs, status objects, and unknown response + # types retain their exact identity. + return response + + def encode_bytes(self, audio: bytes, *, filename: str = "elevenlabs-output.mp3") -> bytes: + """Encode already-complete audio bytes using the current configuration.""" + if not isinstance(audio, bytes): + raise TypeError("audio must be bytes") + if not audio: + return audio + client, token_id, options = self._encode_snapshot() + source = _NamedAudio(audio, filename) + result = client.audio.encode( + source, + token_id, + save_file=False, + keep_original=False, + response_format="stream", + original_filename=filename, + **options, + ) + encoded = getattr(result, "audio_bytes", None) + if not isinstance(encoded, bytes) or not encoded: + raise OfSpectrumError( + message="Watermark encode did not return audio bytes", + code="InvalidWatermarkEncodeResponse", + ) + return encoded + + def close(self) -> None: + with self._lock: + self._replace_owned_client(None) + + def _transform_iterator(self, response: Iterator, filename: str) -> Iterator[bytes]: + def generate() -> Iterator[bytes]: + chunks = [] + try: + for chunk in response: + if not isinstance(chunk, (bytes, bytearray, memoryview)): + raise TypeError( + "ElevenLabs complete-audio response yielded a non-bytes chunk" + ) + chunks.append(bytes(chunk)) + finally: + close = getattr(response, "close", None) + if callable(close): + close() + audio = b"".join(chunks) + if audio: + yield self.encode_bytes(audio, filename=filename) + + return generate() + + def _encode_snapshot(self) -> Tuple[Any, str, Dict[str, Any]]: + with self._lock: + if not self._enabled: + raise WatermarkConfigurationError("Watermarking is disabled") + if not self._token_id: + raise WatermarkConfigurationError( + "A watermark token_id is required; set OFSPECTRUM_TOKEN_ID " + "or call client.watermark.config(token_id=...)" + ) + if self._client is None: + if not self._api_key: + raise WatermarkConfigurationError( + "An OfSpectrum API key is required; set OFSPECTRUM_API_KEY " + "or call client.watermark.config(api_key=...)" + ) + self._client = OfSpectrum(api_key=self._api_key) + self._owns_client = True + return self._client, self._token_id, dict(self._encode_options) + + def _replace_owned_client(self, replacement: Optional[Any]) -> None: + previous = self._client + if previous is not None and self._owns_client and previous is not replacement: + close = getattr(previous, "close", None) + if callable(close): + close() + self._client = replacement + + def _set_encode_options(self, options: Dict[str, Any]) -> None: + unsupported = set(options) - _ENCODE_OPTION_NAMES + if unsupported: + names = ", ".join(sorted(unsupported)) + raise TypeError("Unsupported watermark encode option(s): " + names) + self._encode_options.update(options) + + @staticmethod + def _normalize_method_path(path: str) -> str: + if not isinstance(path, str) or not path.strip(". "): + raise ValueError("path must be a non-empty dotted method path") + normalized = path.strip(". ") + if _is_stream_method_name(normalized.split(".")[-1]): + raise ValueError("streaming methods cannot be registered for watermarking") + return normalized + + +class _ElevenLabsProxy: + def __init__( + self, + target: Any, + controller: WatermarkController, + path: Tuple[str, ...] = (), + ) -> None: + object.__setattr__(self, "_target", target) + object.__setattr__(self, "_controller", controller) + object.__setattr__(self, "_path", path) + object.__setattr__(self, "_children", {}) + + def __getattr__(self, name: str) -> Any: + target = object.__getattribute__(self, "_target") + value = getattr(target, name) + path = object.__getattribute__(self, "_path") + (name,) + controller = object.__getattribute__(self, "_controller") + + if callable(value): + @functools.wraps(value) + def call(*args: Any, **kwargs: Any) -> Any: + response = value(*args, **kwargs) + if controller.handles(path): + return controller.transform(response, kwargs) + return response + + return call + + if self._is_resource(value): + children = object.__getattribute__(self, "_children") + cached = children.get(name) + if cached is None or object.__getattribute__(cached, "_target") is not value: + cached = _ElevenLabsProxy(value, controller, path) + children[name] = cached + return cached + return value + + def __dir__(self) -> Any: + return sorted(set(object.__dir__(self)) | set(dir(self._target))) + + @staticmethod + def _is_resource(value: Any) -> bool: + if value is None or isinstance( + value, (str, bytes, bytearray, memoryview, bool, int, float, list, tuple, dict, set) + ): + return False + module = type(value).__module__ + if module == "elevenlabs" or module.startswith("elevenlabs."): + return True + try: + return any( + not name.startswith("_") and callable(getattr(value, name, None)) + for name in dir(value) + ) + except Exception: + return False + + +class Ofspectrum(_ElevenLabsProxy): + """Wrap a synchronous ElevenLabs client and watermark complete audio calls. + + The original client remains accessible through ``wrapped_client``. The + wrapper is intentionally named ``Ofspectrum`` to support the integration + syntax without changing the existing ``OfSpectrum`` API client. + """ + + def __init__( + self, + elevenlabs_client: Any, + *, + watermark_client: Optional[Any] = None, + api_key: Optional[str] = None, + token_id: Optional[str] = None, + enabled: bool = True, + audio_methods: Optional[Iterable[str]] = None, + **encode_options: Any, + ) -> None: + if elevenlabs_client is None: + raise ValueError("elevenlabs_client is required") + controller = WatermarkController( + client=watermark_client, + api_key=api_key, + token_id=token_id, + enabled=enabled, + encode_options=encode_options, + audio_methods=audio_methods, + ) + super().__init__(elevenlabs_client, controller) + object.__setattr__(self, "watermark", controller) + + @property + def wrapped_client(self) -> Any: + return object.__getattribute__(self, "_target") + + def close(self) -> None: + try: + close = getattr(self.wrapped_client, "close", None) + if callable(close): + close() + finally: + self.watermark.close() + + def __enter__(self) -> "Ofspectrum": + enter = getattr(self.wrapped_client, "__enter__", None) + if callable(enter): + enter() + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: + try: + exit_method = getattr(self.wrapped_client, "__exit__", None) + if callable(exit_method): + return exit_method(exc_type, exc_value, traceback) + return None + finally: + self.watermark.close() + + +__all__ = [ + "Ofspectrum", + "WatermarkConfigurationError", + "WatermarkController", +] diff --git a/ofspectrum/exceptions.py b/ofspectrum/exceptions.py index 8f61222..0c4d016 100644 --- a/ofspectrum/exceptions.py +++ b/ofspectrum/exceptions.py @@ -117,6 +117,14 @@ def __init__( self.field = field +class WatermarkConfigurationError(OfSpectrumError): + """Raised when an intercepted audio call has incomplete configuration.""" + + def __init__(self, message: str = "Watermark configuration is incomplete", **kwargs): + kwargs.setdefault("code", "WatermarkConfigurationError") + super().__init__(message, **kwargs) + + class WatermarkExistsError(OfSpectrumError): """Raised when trying to encode a watermark on already watermarked audio""" diff --git a/pyproject.toml b/pyproject.toml index d576c9c..011caac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,10 @@ dependencies = [ ] [project.optional-dependencies] +elevenlabs = [ + "elevenlabs>=2.0.0,<3.0.0", + "python-dotenv>=1.0.0", +] dev = [ "pytest>=7.0.0", "pytest-asyncio>=0.21.0", diff --git a/tests/test_elevenlabs_benchmark.py b/tests/test_elevenlabs_benchmark.py new file mode 100644 index 0000000..94fff7d --- /dev/null +++ b/tests/test_elevenlabs_benchmark.py @@ -0,0 +1,30 @@ +import pytest + +from benchmarks.elevenlabs_latency import output_filename, percentile, summary + + +def test_percentile_uses_linear_interpolation(): + assert percentile([1.0, 2.0, 3.0], 0.50) == 2.0 + assert percentile([1.0, 2.0, 3.0], 0.95) == pytest.approx(2.9) + + +def test_summary_reports_average_p50_and_p95_in_milliseconds(): + result = summary([0.1, 0.2, 0.3]) + + assert result == { + "average_ms": pytest.approx(200.0), + "p50_ms": pytest.approx(200.0), + "p95_ms": pytest.approx(290.0), + } + + +@pytest.mark.parametrize( + ("output_format", "expected"), + [ + ("mp3_44100_128", "elevenlabs-output.mp3"), + ("wav_44100", "elevenlabs-output.wav"), + ("auto", "elevenlabs-output.mp3"), + ], +) +def test_output_filename(output_format, expected): + assert output_filename(output_format) == expected diff --git a/tests/test_elevenlabs_wrapper.py b/tests/test_elevenlabs_wrapper.py new file mode 100644 index 0000000..fcb9b39 --- /dev/null +++ b/tests/test_elevenlabs_wrapper.py @@ -0,0 +1,282 @@ +from types import SimpleNamespace + +import pytest + +from ofspectrum import Ofspectrum, WatermarkConfigurationError + + +class _FakeWatermarkAudio: + def __init__(self): + self.calls = [] + + def encode(self, audio, token_id, **kwargs): + source = audio.read() + self.calls.append((source, token_id, kwargs, audio.name)) + return SimpleNamespace(audio_bytes=b"watermarked:" + source) + + +class _FakeWatermarkClient: + def __init__(self): + self.audio = _FakeWatermarkAudio() + self.closed = False + + def close(self): + self.closed = True + + +class _TextToSpeech: + def __init__(self): + self.calls = [] + self.stream_result = iter([b"live-1", b"live-2"]) + + def convert(self, *args, **kwargs): + self.calls.append((args, kwargs)) + return iter([b"original-", b"audio"]) + + def stream(self, *args, **kwargs): + self.calls.append((args, kwargs)) + return self.stream_result + + def convert_with_timestamps(self, **kwargs): + return {"audio_base_64": "not-direct-audio", "alignment": []} + + +class _Models: + def __init__(self): + self.result = [{"id": "eleven-v3"}] + + def list(self): + return self.result + + +class _FakeElevenLabs: + def __init__(self): + self.text_to_speech = _TextToSpeech() + self.models = _Models() + self.closed = False + + def close(self): + self.closed = True + + +def _wrapper(**kwargs): + elevenlabs = _FakeElevenLabs() + watermark = _FakeWatermarkClient() + return Ofspectrum( + elevenlabs, + watermark_client=watermark, + token_id="token-1", + **kwargs, + ), elevenlabs, watermark + + +def test_convert_preserves_call_and_returns_watermarked_iterator(): + client, elevenlabs, watermark = _wrapper() + + result = client.text_to_speech.convert( + "positional", text="hello", output_format="wav_44100" + ) + + assert elevenlabs.text_to_speech.calls == [ + (("positional",), {"text": "hello", "output_format": "wav_44100"}) + ] + assert watermark.audio.calls == [] + assert list(result) == [b"watermarked:original-audio"] + source, token_id, options, name = watermark.audio.calls[0] + assert source == b"original-audio" + assert token_id == "token-1" + assert name == "elevenlabs-output.wav" + assert options["save_file"] is False + assert options["keep_original"] is False + assert options["response_format"] == "stream" + assert options["original_filename"] == "elevenlabs-output.wav" + + +def test_stream_is_returned_without_consuming_or_replacing_it(): + client, elevenlabs, watermark = _wrapper() + + result = client.text_to_speech.stream(text="hello") + + assert result is elevenlabs.text_to_speech.stream_result + assert list(result) == [b"live-1", b"live-2"] + assert watermark.audio.calls == [] + + +def test_json_and_metadata_results_keep_exact_identity(): + client, elevenlabs, watermark = _wrapper() + + models = client.models.list() + timestamped = client.text_to_speech.convert_with_timestamps(text="hello") + + assert models is elevenlabs.models.result + assert timestamped == {"audio_base_64": "not-direct-audio", "alignment": []} + assert watermark.audio.calls == [] + + +@pytest.mark.parametrize( + "path", + [ + "audio_isolation.convert", + "conversational_ai.conversations.audio.get", + "history.get_audio", + "music.compose", + "music.video_to_music", + "sound_generation.text_to_sound_effects", + "speech_to_speech.convert", + "text_to_dialogue.convert", + "text_to_sound_effects.convert", + "voices.samples.audio.get", + ], +) +def test_first_phase_complete_audio_endpoints_are_intercepted(path): + class Resource: + def convert(self, **kwargs): + return b"audio" + + def compose(self, **kwargs): + return b"audio" + + def text_to_sound_effects(self, **kwargs): + return b"audio" + + def get(self, **kwargs): + return b"audio" + + def get_audio(self, **kwargs): + return b"audio" + + def video_to_music(self, **kwargs): + return b"audio" + + elevenlabs = SimpleNamespace() + current = elevenlabs + parts = path.split(".") + for part in parts[:-1]: + resource = Resource() + setattr(current, part, resource) + current = resource + watermark = _FakeWatermarkClient() + client = Ofspectrum( + elevenlabs, watermark_client=watermark, token_id="token-1" + ) + + callable_object = client + for part in parts: + callable_object = getattr(callable_object, part) + + assert callable_object() == b"watermarked:audio" + assert len(watermark.audio.calls) == 1 + + +def test_unknown_bytes_method_is_not_guessed_to_be_audio(): + class Files: + def download(self): + return b"zip-or-other-binary" + + elevenlabs = SimpleNamespace(files=Files()) + watermark = _FakeWatermarkClient() + client = Ofspectrum( + elevenlabs, watermark_client=watermark, token_id="token-1" + ) + + assert client.files.download() == b"zip-or-other-binary" + assert watermark.audio.calls == [] + + +def test_empty_audio_method_registry_disables_default_interception(): + elevenlabs = _FakeElevenLabs() + watermark = _FakeWatermarkClient() + client = Ofspectrum( + elevenlabs, + watermark_client=watermark, + token_id="token-1", + audio_methods=[], + ) + + assert list(client.text_to_speech.convert()) == [b"original-", b"audio"] + assert watermark.audio.calls == [] + + +def test_custom_complete_audio_method_can_be_registered(): + class FutureAudio: + def render(self): + return [b"future-", b"audio"] + + watermark = _FakeWatermarkClient() + client = Ofspectrum( + SimpleNamespace(future_audio=FutureAudio()), + watermark_client=watermark, + token_id="token-1", + ) + client.watermark.register_audio_method("future_audio.render") + + assert client.future_audio.render() == [b"watermarked:future-audio"] + + +def test_disable_returns_native_result_without_requiring_configuration(): + elevenlabs = _FakeElevenLabs() + client = Ofspectrum(elevenlabs, enabled=False) + + result = client.text_to_speech.convert(text="hello") + + assert list(result) == [b"original-", b"audio"] + + +def test_missing_token_fails_only_when_intercepted_audio_is_consumed(): + elevenlabs = _FakeElevenLabs() + client = Ofspectrum(elevenlabs, watermark_client=_FakeWatermarkClient()) + + assert client.models.list() is elevenlabs.models.result + result = client.text_to_speech.convert(text="hello") + with pytest.raises(WatermarkConfigurationError, match="token_id"): + list(result) + + +def test_elevenlabs_exceptions_are_not_wrapped(): + expected = RuntimeError("elevenlabs failure") + + class Broken: + def convert(self): + raise expected + + client = Ofspectrum( + SimpleNamespace(text_to_speech=Broken()), + watermark_client=_FakeWatermarkClient(), + token_id="token-1", + ) + + with pytest.raises(RuntimeError) as raised: + client.text_to_speech.convert() + assert raised.value is expected + + +def test_config_updates_options_and_can_toggle_watermarking(): + client, _, watermark = _wrapper() + client.watermark.config(enabled=False, strength=1.25) + assert list(client.text_to_speech.convert()) == [b"original-", b"audio"] + + client.watermark.config(enabled=True, token_id="token-2") + assert list(client.text_to_speech.convert()) == [b"watermarked:original-audio"] + _, token_id, options, _ = watermark.audio.calls[0] + assert token_id == "token-2" + assert options["strength"] == 1.25 + + +def test_close_closes_elevenlabs_but_not_injected_watermark_client(): + client, elevenlabs, watermark = _wrapper() + + client.close() + + assert elevenlabs.closed is True + assert watermark.closed is False + + +def test_config_rejects_stream_registration_and_output_side_effect_options(): + client, _, _ = _wrapper() + + with pytest.raises(ValueError, match="streaming"): + client.watermark.register_audio_method("future_audio.stream") + with pytest.raises(ValueError, match="streaming"): + client.watermark.register_audio_method("future_audio.compose_detailed_stream") + with pytest.raises(TypeError, match="save_file"): + client.watermark.config(save_file=True)