From 340038de54e790c936abab12e1af8b3329f23d38 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 27 Jul 2025 21:04:34 +0000 Subject: [PATCH 1/3] Add TTS and ASR speech functionality from cursor/fixing-code-issues-5a38 branch --- .../llm4s/samples/speech/SpeechExample.scala | 204 ++++++++++++++++++ .../speech/VoiceAssistantExample.scala | 203 +++++++++++++++++ .../scala/org/llm4s/speech/ASRClient.scala | 21 ++ src/main/scala/org/llm4s/speech/Speech.scala | 58 +++++ .../org/llm4s/speech/SpeechConnect.scala | 119 ++++++++++ .../scala/org/llm4s/speech/TTSClient.scala | 21 ++ .../speech/config/SpeechProviderConfig.scala | 110 ++++++++++ .../model/ASRTranscriptionOptions.scala | 38 ++++ .../org/llm4s/speech/model/SpeechError.scala | 33 +++ .../speech/model/TTSSynthesisOptions.scala | 22 ++ .../speech/provider/AzureSpeechClient.scala | 127 +++++++++++ .../speech/provider/ElevenLabsClient.scala | 76 +++++++ .../speech/provider/GoogleSpeechClient.scala | 182 ++++++++++++++++ .../speech/provider/OpenAISpeechClient.scala | 143 ++++++++++++ .../speech/provider/SpeechProvider.scala | 10 + .../scala/org/llm4s/speech/SpeechTest.scala | 95 ++++++++ 16 files changed, 1462 insertions(+) create mode 100644 samples/src/main/scala/org/llm4s/samples/speech/SpeechExample.scala create mode 100644 samples/src/main/scala/org/llm4s/samples/speech/VoiceAssistantExample.scala create mode 100644 src/main/scala/org/llm4s/speech/ASRClient.scala create mode 100644 src/main/scala/org/llm4s/speech/Speech.scala create mode 100644 src/main/scala/org/llm4s/speech/SpeechConnect.scala create mode 100644 src/main/scala/org/llm4s/speech/TTSClient.scala create mode 100644 src/main/scala/org/llm4s/speech/config/SpeechProviderConfig.scala create mode 100644 src/main/scala/org/llm4s/speech/model/ASRTranscriptionOptions.scala create mode 100644 src/main/scala/org/llm4s/speech/model/SpeechError.scala create mode 100644 src/main/scala/org/llm4s/speech/model/TTSSynthesisOptions.scala create mode 100644 src/main/scala/org/llm4s/speech/provider/AzureSpeechClient.scala create mode 100644 src/main/scala/org/llm4s/speech/provider/ElevenLabsClient.scala create mode 100644 src/main/scala/org/llm4s/speech/provider/GoogleSpeechClient.scala create mode 100644 src/main/scala/org/llm4s/speech/provider/OpenAISpeechClient.scala create mode 100644 src/main/scala/org/llm4s/speech/provider/SpeechProvider.scala create mode 100644 src/test/scala/org/llm4s/speech/SpeechTest.scala diff --git a/samples/src/main/scala/org/llm4s/samples/speech/SpeechExample.scala b/samples/src/main/scala/org/llm4s/samples/speech/SpeechExample.scala new file mode 100644 index 000000000..bc000387a --- /dev/null +++ b/samples/src/main/scala/org/llm4s/samples/speech/SpeechExample.scala @@ -0,0 +1,204 @@ +package org.llm4s.samples.speech + +import org.llm4s.speech._ +import org.llm4s.speech.config.{AzureSpeechConfig, ElevenLabsConfig, GoogleSpeechConfig, OpenAISpeechConfig} +import org.llm4s.speech.model._ +import org.llm4s.speech.provider._ +import org.slf4j.LoggerFactory + +import java.io.{File, FileOutputStream} +import java.nio.file.{Files, Paths} + +/** + * Example demonstrating TTS and ASR functionality with different providers + */ +object SpeechExample { + private val logger = LoggerFactory.getLogger(getClass) + + def main(args: Array[String]): Unit = { + logger.info("Starting Speech TTS/ASR Example") + + // Example 1: OpenAI TTS + openAITTSExample() + + // Example 2: OpenAI ASR (if you have an audio file) + // openAIASRExample() + + // Example 3: Azure Speech TTS + azureTTSSExample() + + // Example 4: Google Speech TTS + googleTTSSExample() + + // Example 5: ElevenLabs TTS + elevenLabsTTSExample() + + logger.info("Speech TTS/ASR Example completed") + } + + def openAITTSExample(): Unit = { + logger.info("=== OpenAI TTS Example ===") + + try { + val config = OpenAISpeechConfig.fromEnv("tts-1") + val ttsClient = Speech.ttsClient(SpeechProvider.OpenAI, config) + + val text = "Hello, this is a test of the text-to-speech functionality using OpenAI's TTS API." + val options = TTSSynthesisOptions( + voice = "alloy", + model = "tts-1", + responseFormat = "mp3", + speed = 1.0 + ) + + ttsClient.synthesize(text, options) match { + case Right(audioResponse) => + logger.info(s"Successfully synthesized audio: ${audioResponse.audioData.length} bytes") + // Save the audio to a file + saveAudioToFile(audioResponse.audioData, "openai_tts_output.mp3") + + case Left(error) => + logger.error(s"OpenAI TTS failed: ${error.message}") + } + } catch { + case e: Exception => + logger.error(s"OpenAI TTS setup failed: ${e.getMessage}") + } + } + + def openAIASRExample(): Unit = { + logger.info("=== OpenAI ASR Example ===") + + try { + val config = OpenAISpeechConfig.fromEnv("whisper-1") + val asrClient = Speech.asrClient(SpeechProvider.OpenAI, config) + + // Read audio file (you would need to provide an actual audio file) + val audioFile = new File("sample_audio.mp3") + if (audioFile.exists()) { + val audioData = Files.readAllBytes(audioFile.toPath) + + val options = ASRTranscriptionOptions( + model = "whisper-1", + language = Some("en"), + responseFormat = "json" + ) + + asrClient.transcribe(audioData, options) match { + case Right(transcription) => + logger.info(s"Transcription: ${transcription.text}") + logger.info(s"Language: ${transcription.language}") + logger.info(s"Segments: ${transcription.segments.length}") + + case Left(error) => + logger.error(s"OpenAI ASR failed: ${error.message}") + } + } else { + logger.warn("Sample audio file not found, skipping ASR example") + } + } catch { + case e: Exception => + logger.error(s"OpenAI ASR setup failed: ${e.getMessage}") + } + } + + def azureTTSSExample(): Unit = { + logger.info("=== Azure Speech TTS Example ===") + + try { + val config = AzureSpeechConfig.fromEnv("en-US-JennyNeural") + val ttsClient = Speech.ttsClient(SpeechProvider.Azure, config) + + val text = "Hello, this is a test of Azure's text-to-speech service." + val options = TTSSynthesisOptions( + voice = "en-US-JennyNeural", + model = "en-US-JennyNeural", + responseFormat = "mp3", + speed = 1.0 + ) + + ttsClient.synthesize(text, options) match { + case Right(audioResponse) => + logger.info(s"Successfully synthesized audio: ${audioResponse.audioData.length} bytes") + saveAudioToFile(audioResponse.audioData, "azure_tts_output.mp3") + + case Left(error) => + logger.error(s"Azure TTS failed: ${error.message}") + } + } catch { + case e: Exception => + logger.error(s"Azure TTS setup failed: ${e.getMessage}") + } + } + + def googleTTSSExample(): Unit = { + logger.info("=== Google Speech TTS Example ===") + + try { + val config = GoogleSpeechConfig.fromEnv("latest") + val ttsClient = Speech.ttsClient(SpeechProvider.Google, config) + + val text = "Hello, this is a test of Google's text-to-speech service." + val options = TTSSynthesisOptions( + voice = "en-US-Standard-A", + model = "latest", + responseFormat = "mp3", + speed = 1.0 + ) + + ttsClient.synthesize(text, options) match { + case Right(audioResponse) => + logger.info(s"Successfully synthesized audio: ${audioResponse.audioData.length} bytes") + saveAudioToFile(audioResponse.audioData, "google_tts_output.mp3") + + case Left(error) => + logger.error(s"Google TTS failed: ${error.message}") + } + } catch { + case e: Exception => + logger.error(s"Google TTS setup failed: ${e.getMessage}") + } + } + + def elevenLabsTTSExample(): Unit = { + logger.info("=== ElevenLabs TTS Example ===") + + try { + val config = ElevenLabsConfig.fromEnv("eleven_monolingual_v1") + val ttsClient = Speech.ttsClient(SpeechProvider.ElevenLabs, config) + + val text = "Hello, this is a test of ElevenLabs text-to-speech service." + val options = TTSSynthesisOptions( + voice = "21m00Tcm4TlvDq8ikWAM", // Example voice ID + model = "eleven_monolingual_v1", + responseFormat = "mp3", + speed = 1.0 + ) + + ttsClient.synthesize(text, options) match { + case Right(audioResponse) => + logger.info(s"Successfully synthesized audio: ${audioResponse.audioData.length} bytes") + saveAudioToFile(audioResponse.audioData, "elevenlabs_tts_output.mp3") + + case Left(error) => + logger.error(s"ElevenLabs TTS failed: ${error.message}") + } + } catch { + case e: Exception => + logger.error(s"ElevenLabs TTS setup failed: ${e.getMessage}") + } + } + + private def saveAudioToFile(audioData: Array[Byte], filename: String): Unit = { + try { + val file = new File(filename) + val fos = new FileOutputStream(file) + fos.write(audioData) + fos.close() + logger.info(s"Audio saved to: ${file.getAbsolutePath}") + } catch { + case e: Exception => + logger.error(s"Failed to save audio file: ${e.getMessage}") + } + } +} \ No newline at end of file diff --git a/samples/src/main/scala/org/llm4s/samples/speech/VoiceAssistantExample.scala b/samples/src/main/scala/org/llm4s/samples/speech/VoiceAssistantExample.scala new file mode 100644 index 000000000..a4ae9fd6b --- /dev/null +++ b/samples/src/main/scala/org/llm4s/samples/speech/VoiceAssistantExample.scala @@ -0,0 +1,203 @@ +package org.llm4s.samples.speech + +import org.llm4s.llmconnect.LLM +import org.llm4s.llmconnect.config.OpenAIConfig +import org.llm4s.llmconnect.model._ +import org.llm4s.llmconnect.provider.LLMProvider +import org.llm4s.speech._ +import org.llm4s.speech.config.OpenAISpeechConfig +import org.llm4s.speech.model._ +import org.llm4s.speech.provider.SpeechProvider +import org.slf4j.LoggerFactory + +import java.io.{File, FileOutputStream} +import java.nio.file.{Files, Paths} +import scala.io.StdIn + +/** + * Voice Assistant Example - Demonstrates integration of TTS, ASR, and LLM + * + * This example shows how to create a voice assistant that: + * 1. Listens to user speech (ASR) + * 2. Processes the speech with an LLM + * 3. Converts the LLM response to speech (TTS) + * 4. Plays the response + */ +object VoiceAssistantExample { + private val logger = LoggerFactory.getLogger(getClass) + + def main(args: Array[String]): Unit = { + logger.info("Starting Voice Assistant Example") + + // Initialize speech clients + val speechConfig = OpenAISpeechConfig.fromEnv("tts-1") + val ttsClient = Speech.ttsClient(SpeechProvider.OpenAI, speechConfig) + val asrClient = Speech.asrClient(SpeechProvider.OpenAI, speechConfig) + + // Initialize LLM client + val llmConfig = OpenAIConfig.fromEnv("gpt-4o") + val llmClient = LLM.client(LLMProvider.OpenAI, llmConfig) + + // System prompt for the voice assistant + val systemPrompt = SystemMessage( + """You are a helpful voice assistant. Keep your responses concise and natural for speech. + |Respond as if you're having a conversation with someone speaking to you. + |Keep responses under 2-3 sentences for better voice interaction.""".stripMargin + ) + + logger.info("Voice Assistant initialized. Type 'quit' to exit.") + + // Main conversation loop + var conversation = Conversation(Seq(systemPrompt)) + var running = true + + while (running) { + print("You: ") + val userInput = StdIn.readLine() + + if (userInput == null || userInput.toLowerCase == "quit") { + running = false + } else { + // Add user message to conversation + val userMessage = UserMessage(userInput) + conversation = conversation.addMessage(userMessage) + + // Get LLM response + val llmResponse = llmClient.complete(conversation, CompletionOptions()) + + llmResponse match { + case Right(completion) => + val assistantMessage = completion.message + conversation = conversation.addMessage(assistantMessage) + + val responseText = assistantMessage.content + logger.info(s"Assistant: $responseText") + + // Convert response to speech + val ttsOptions = TTSSynthesisOptions( + voice = "alloy", + model = "tts-1", + responseFormat = "mp3", + speed = 1.0 + ) + + ttsClient.synthesize(responseText, ttsOptions) match { + case Right(audioResponse) => + logger.info(s"Generated speech: ${audioResponse.audioData.length} bytes") + saveAudioToFile(audioResponse.audioData, "voice_assistant_response.mp3") + logger.info("Audio saved to voice_assistant_response.mp3") + + case Left(error) => + logger.error(s"TTS failed: ${error.message}") + } + + case Left(error) => + logger.error(s"LLM failed: ${error.message}") + } + } + } + + logger.info("Voice Assistant stopped") + } + + /** + * Simulate voice input by reading from an audio file + */ + def simulateVoiceInput(audioFilePath: String): Option[String] = { + try { + val audioFile = new File(audioFilePath) + if (audioFile.exists()) { + val audioData = Files.readAllBytes(audioFile.toPath) + + val asrClient = Speech.asrClient() + val options = ASRTranscriptionOptions( + model = "whisper-1", + language = Some("en"), + responseFormat = "json" + ) + + asrClient.transcribe(audioData, options) match { + case Right(transcription) => + logger.info(s"Transcribed: ${transcription.text}") + Some(transcription.text) + + case Left(error) => + logger.error(s"ASR failed: ${error.message}") + None + } + } else { + logger.warn(s"Audio file not found: $audioFilePath") + None + } + } catch { + case e: Exception => + logger.error(s"Failed to process audio file: ${e.getMessage}") + None + } + } + + /** + * Process audio file and get LLM response + */ + def processAudioFile(audioFilePath: String): Unit = { + logger.info(s"Processing audio file: $audioFilePath") + + // Step 1: Transcribe audio to text + val transcribedText = simulateVoiceInput(audioFilePath) + + transcribedText.foreach { text => + // Step 2: Get LLM response + val llmConfig = OpenAIConfig.fromEnv("gpt-4o") + val llmClient = LLM.client(LLMProvider.OpenAI, llmConfig) + + val systemPrompt = SystemMessage( + "You are a helpful voice assistant. Keep your responses concise and natural for speech." + ) + val userMessage = UserMessage(text) + val conversation = Conversation(Seq(systemPrompt, userMessage)) + + val llmResponse = llmClient.complete(conversation, CompletionOptions()) + + llmResponse match { + case Right(completion) => + val responseText = completion.message.content + logger.info(s"LLM Response: $responseText") + + // Step 3: Convert response to speech + val ttsClient = Speech.ttsClient() + val ttsOptions = TTSSynthesisOptions( + voice = "alloy", + model = "tts-1", + responseFormat = "mp3", + speed = 1.0 + ) + + ttsClient.synthesize(responseText, ttsOptions) match { + case Right(audioResponse) => + logger.info(s"Generated speech: ${audioResponse.audioData.length} bytes") + saveAudioToFile(audioResponse.audioData, "voice_response.mp3") + logger.info("Audio saved to voice_response.mp3") + + case Left(error) => + logger.error(s"TTS failed: ${error.message}") + } + + case Left(error) => + logger.error(s"LLM failed: ${error.message}") + } + } + } + + private def saveAudioToFile(audioData: Array[Byte], filename: String): Unit = { + try { + val file = new File(filename) + val fos = new FileOutputStream(file) + fos.write(audioData) + fos.close() + logger.info(s"Audio saved to: ${file.getAbsolutePath}") + } catch { + case e: Exception => + logger.error(s"Failed to save audio file: ${e.getMessage}") + } + } +} \ No newline at end of file diff --git a/src/main/scala/org/llm4s/speech/ASRClient.scala b/src/main/scala/org/llm4s/speech/ASRClient.scala new file mode 100644 index 000000000..82775c34b --- /dev/null +++ b/src/main/scala/org/llm4s/speech/ASRClient.scala @@ -0,0 +1,21 @@ +package org.llm4s.speech + +import org.llm4s.speech.model.{ ASRTranscriptionOptions, SpeechError, TranscriptionResponse } + +/** + * Client interface for automatic speech recognition + */ +trait ASRClient { + + /** + * Transcribe audio to text + * + * @param audioData The audio data to transcribe + * @param options Configuration options for transcription + * @return Either a SpeechError or TranscriptionResponse + */ + def transcribe( + audioData: Array[Byte], + options: ASRTranscriptionOptions = ASRTranscriptionOptions() + ): Either[SpeechError, TranscriptionResponse] +} diff --git a/src/main/scala/org/llm4s/speech/Speech.scala b/src/main/scala/org/llm4s/speech/Speech.scala new file mode 100644 index 000000000..0bdafadcc --- /dev/null +++ b/src/main/scala/org/llm4s/speech/Speech.scala @@ -0,0 +1,58 @@ +package org.llm4s.speech + +import org.llm4s.speech.config.SpeechProviderConfig +import org.llm4s.speech.model._ +import org.llm4s.speech.provider.SpeechProvider + +object Speech { + + /** Factory method for getting a TTS client with the right configuration */ + def ttsClient( + provider: SpeechProvider, + config: SpeechProviderConfig + ): TTSClient = SpeechConnect.getTTSClient(provider, config) + + /** Factory method for getting an ASR client with the right configuration */ + def asrClient( + provider: SpeechProvider, + config: SpeechProviderConfig + ): ASRClient = SpeechConnect.getASRClient(provider, config) + + /** Convenience method for quick text-to-speech conversion */ + def synthesize( + text: String, + provider: SpeechProvider, + config: SpeechProviderConfig, + options: TTSSynthesisOptions = TTSSynthesisOptions() + ): Either[SpeechError, AudioResponse] = + ttsClient(provider, config).synthesize(text, options) + + /** Convenience method for quick speech-to-text conversion */ + def transcribe( + audioData: Array[Byte], + provider: SpeechProvider, + config: SpeechProviderConfig, + options: ASRTranscriptionOptions = ASRTranscriptionOptions() + ): Either[SpeechError, TranscriptionResponse] = + asrClient(provider, config).transcribe(audioData, options) + + /** Get a TTS client based on environment variables */ + def ttsClient(): TTSClient = SpeechConnect.getTTSClient() + + /** Get an ASR client based on environment variables */ + def asrClient(): ASRClient = SpeechConnect.getASRClient() + + /** Convenience method for quick text-to-speech using environment variables */ + def synthesizeWithEnv( + text: String, + options: TTSSynthesisOptions = TTSSynthesisOptions() + ): Either[SpeechError, AudioResponse] = + ttsClient().synthesize(text, options) + + /** Convenience method for quick speech-to-text using environment variables */ + def transcribeWithEnv( + audioData: Array[Byte], + options: ASRTranscriptionOptions = ASRTranscriptionOptions() + ): Either[SpeechError, TranscriptionResponse] = + asrClient().transcribe(audioData, options) +} diff --git a/src/main/scala/org/llm4s/speech/SpeechConnect.scala b/src/main/scala/org/llm4s/speech/SpeechConnect.scala new file mode 100644 index 000000000..77ff2c894 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/SpeechConnect.scala @@ -0,0 +1,119 @@ +package org.llm4s.speech + +import org.llm4s.speech.config.{ + AzureSpeechConfig, + ElevenLabsConfig, + GoogleSpeechConfig, + OpenAISpeechConfig, + SpeechProviderConfig +} +import org.llm4s.speech.provider.{ + AzureSpeechClient, + ElevenLabsClient, + GoogleSpeechClient, + OpenAISpeechClient, + SpeechProvider +} + +object SpeechConnect { + private def readEnv(key: String): Option[String] = + sys.env.get(key) + + /** + * Get a TTS client based on environment variables + */ + def getTTSClient(): TTSClient = { + val SPEECH_MODEL_ENV_KEY = "SPEECH_MODEL" + val model = readEnv(SPEECH_MODEL_ENV_KEY).getOrElse( + throw new IllegalArgumentException( + s"Please set the `$SPEECH_MODEL_ENV_KEY` environment variable to specify the default speech model" + ) + ) + + if (model.startsWith("openai/")) { + val modelName = model.replace("openai/", "") + val config = OpenAISpeechConfig.fromEnv(modelName) + new OpenAISpeechClient(config) + } else if (model.startsWith("azure/")) { + val modelName = model.replace("azure/", "") + val config = AzureSpeechConfig.fromEnv(modelName) + new AzureSpeechClient(config) + } else if (model.startsWith("google/")) { + val modelName = model.replace("google/", "") + val config = GoogleSpeechConfig.fromEnv(modelName) + new GoogleSpeechClient(config) + } else if (model.startsWith("elevenlabs/")) { + val modelName = model.replace("elevenlabs/", "") + val config = ElevenLabsConfig.fromEnv(modelName) + new ElevenLabsClient(config) + } else { + throw new IllegalArgumentException( + s"Model $model is not supported. Supported formats are: 'openai/model-name', 'azure/model-name', 'google/model-name', or 'elevenlabs/model-name'." + ) + } + } + + /** + * Get an ASR client based on environment variables + */ + def getASRClient(): ASRClient = { + val SPEECH_MODEL_ENV_KEY = "SPEECH_MODEL" + val model = readEnv(SPEECH_MODEL_ENV_KEY).getOrElse( + throw new IllegalArgumentException( + s"Please set the `$SPEECH_MODEL_ENV_KEY` environment variable to specify the default speech model" + ) + ) + + if (model.startsWith("openai/")) { + val modelName = model.replace("openai/", "") + val config = OpenAISpeechConfig.fromEnv(modelName) + new OpenAISpeechClient(config) + } else if (model.startsWith("azure/")) { + val modelName = model.replace("azure/", "") + val config = AzureSpeechConfig.fromEnv(modelName) + new AzureSpeechClient(config) + } else if (model.startsWith("google/")) { + val modelName = model.replace("google/", "") + val config = GoogleSpeechConfig.fromEnv(modelName) + new GoogleSpeechClient(config) + } else { + throw new IllegalArgumentException( + s"Model $model is not supported for ASR. Supported formats are: 'openai/model-name', 'azure/model-name', or 'google/model-name'." + ) + } + } + + /** + * Get a TTS client with explicit provider and configuration + */ + def getTTSClient(provider: SpeechProvider, config: SpeechProviderConfig): TTSClient = + provider match { + case SpeechProvider.OpenAI => + new OpenAISpeechClient(config.asInstanceOf[OpenAISpeechConfig]) + case SpeechProvider.Azure => + new AzureSpeechClient(config.asInstanceOf[AzureSpeechConfig]) + case SpeechProvider.Google => + new GoogleSpeechClient(config.asInstanceOf[GoogleSpeechConfig]) + case SpeechProvider.ElevenLabs => + new ElevenLabsClient(config.asInstanceOf[ElevenLabsConfig]) + case SpeechProvider.Amazon => + throw new UnsupportedOperationException("Amazon Speech not yet implemented") + } + + /** + * Get an ASR client with explicit provider and configuration + */ + def getASRClient(provider: SpeechProvider, config: SpeechProviderConfig): ASRClient = + provider match { + case SpeechProvider.OpenAI => + new OpenAISpeechClient(config.asInstanceOf[OpenAISpeechConfig]) + case SpeechProvider.Azure => + new AzureSpeechClient(config.asInstanceOf[AzureSpeechConfig]) + case SpeechProvider.Google => + new GoogleSpeechClient(config.asInstanceOf[GoogleSpeechConfig]) + case SpeechProvider.ElevenLabs => + throw new UnsupportedOperationException("ElevenLabs does not support ASR") + case SpeechProvider.Amazon => + throw new UnsupportedOperationException("Amazon Speech not yet implemented") + } +} diff --git a/src/main/scala/org/llm4s/speech/TTSClient.scala b/src/main/scala/org/llm4s/speech/TTSClient.scala new file mode 100644 index 000000000..b6be24157 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/TTSClient.scala @@ -0,0 +1,21 @@ +package org.llm4s.speech + +import org.llm4s.speech.model.{ AudioResponse, SpeechError, TTSSynthesisOptions } + +/** + * Client interface for text-to-speech synthesis + */ +trait TTSClient { + + /** + * Synthesize text to speech audio + * + * @param text The text to synthesize + * @param options Configuration options for synthesis + * @return Either a SpeechError or AudioResponse + */ + def synthesize( + text: String, + options: TTSSynthesisOptions = TTSSynthesisOptions() + ): Either[SpeechError, AudioResponse] +} diff --git a/src/main/scala/org/llm4s/speech/config/SpeechProviderConfig.scala b/src/main/scala/org/llm4s/speech/config/SpeechProviderConfig.scala new file mode 100644 index 000000000..be1c77821 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/config/SpeechProviderConfig.scala @@ -0,0 +1,110 @@ +package org.llm4s.speech.config + +sealed trait SpeechProviderConfig { + def model: String +} + +object SpeechProviderConfig { + def readEnv(key: String): Option[String] = + sys.env.get(key) +} + +case class OpenAISpeechConfig( + apiKey: String, + model: String = "tts-1", + baseUrl: String = "https://api.openai.com/v1" +) extends SpeechProviderConfig + +object OpenAISpeechConfig { + + /** + * Create an OpenAISpeechConfig from environment variables + */ + def fromEnv(modelName: String): OpenAISpeechConfig = { + val readEnv = SpeechProviderConfig.readEnv _ + + OpenAISpeechConfig( + apiKey = readEnv("OPENAI_API_KEY").getOrElse( + throw new IllegalArgumentException("OPENAI_API_KEY not set, required when using openai/ model.") + ), + model = modelName, + baseUrl = readEnv("OPENAI_BASE_URL").getOrElse("https://api.openai.com/v1") + ) + } +} + +case class AzureSpeechConfig( + apiKey: String, + region: String, + model: String = "en-US-JennyNeural", + baseUrl: String = "https://%s.tts.speech.microsoft.com/cognitiveservices/v1" +) extends SpeechProviderConfig + +object AzureSpeechConfig { + + /** + * Create an AzureSpeechConfig from environment variables + */ + def fromEnv(modelName: String): AzureSpeechConfig = { + val readEnv = SpeechProviderConfig.readEnv _ + + AzureSpeechConfig( + apiKey = readEnv("AZURE_SPEECH_API_KEY").getOrElse( + throw new IllegalArgumentException("AZURE_SPEECH_API_KEY not set, required when using azure/ model.") + ), + region = readEnv("AZURE_SPEECH_REGION").getOrElse( + throw new IllegalArgumentException("AZURE_SPEECH_REGION not set, required when using azure/ model.") + ), + model = modelName, + baseUrl = readEnv("AZURE_SPEECH_BASE_URL").getOrElse("https://%s.tts.speech.microsoft.com/cognitiveservices/v1") + ) + } +} + +case class GoogleSpeechConfig( + apiKey: String, + model: String = "latest", + baseUrl: String = "https://texttospeech.googleapis.com/v1" +) extends SpeechProviderConfig + +object GoogleSpeechConfig { + + /** + * Create a GoogleSpeechConfig from environment variables + */ + def fromEnv(modelName: String): GoogleSpeechConfig = { + val readEnv = SpeechProviderConfig.readEnv _ + + GoogleSpeechConfig( + apiKey = readEnv("GOOGLE_SPEECH_API_KEY").getOrElse( + throw new IllegalArgumentException("GOOGLE_SPEECH_API_KEY not set, required when using google/ model.") + ), + model = modelName, + baseUrl = readEnv("GOOGLE_SPEECH_BASE_URL").getOrElse("https://texttospeech.googleapis.com/v1") + ) + } +} + +case class ElevenLabsConfig( + apiKey: String, + model: String = "eleven_monolingual_v1", + baseUrl: String = "https://api.elevenlabs.io/v1" +) extends SpeechProviderConfig + +object ElevenLabsConfig { + + /** + * Create an ElevenLabsConfig from environment variables + */ + def fromEnv(modelName: String): ElevenLabsConfig = { + val readEnv = SpeechProviderConfig.readEnv _ + + ElevenLabsConfig( + apiKey = readEnv("ELEVENLABS_API_KEY").getOrElse( + throw new IllegalArgumentException("ELEVENLABS_API_KEY not set, required when using elevenlabs/ model.") + ), + model = modelName, + baseUrl = readEnv("ELEVENLABS_BASE_URL").getOrElse("https://api.elevenlabs.io/v1") + ) + } +} diff --git a/src/main/scala/org/llm4s/speech/model/ASRTranscriptionOptions.scala b/src/main/scala/org/llm4s/speech/model/ASRTranscriptionOptions.scala new file mode 100644 index 000000000..b66297adc --- /dev/null +++ b/src/main/scala/org/llm4s/speech/model/ASRTranscriptionOptions.scala @@ -0,0 +1,38 @@ +package org.llm4s.speech.model + +/** + * Options for automatic speech recognition + */ +case class ASRTranscriptionOptions( + model: String = "whisper-1", + language: Option[String] = None, + prompt: Option[String] = None, + responseFormat: String = "json", + temperature: Double = 0.0, + timestampGranularities: Seq[String] = Seq("word", "segment") +) + +/** + * Response from automatic speech recognition + */ +case class TranscriptionResponse( + text: String, + language: Option[String] = None, + duration: Option[Double] = None, + segments: Seq[TranscriptionSegment] = Seq.empty +) + +/** + * Represents a segment of transcribed audio + */ +case class TranscriptionSegment( + id: Int, + start: Double, + end: Double, + text: String, + tokens: Seq[Int] = Seq.empty, + temperature: Option[Double] = None, + avgLogprob: Option[Double] = None, + compressionRatio: Option[Double] = None, + noSpeechProb: Option[Double] = None +) diff --git a/src/main/scala/org/llm4s/speech/model/SpeechError.scala b/src/main/scala/org/llm4s/speech/model/SpeechError.scala new file mode 100644 index 000000000..e8481b842 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/model/SpeechError.scala @@ -0,0 +1,33 @@ +package org.llm4s.speech.model + +import scala.util.control.NoStackTrace + +/** + * Represents errors that can occur during speech processing operations. + */ +sealed trait SpeechError extends Exception with NoStackTrace { + def message: String + override def getMessage: String = message +} + +/** + * Authentication error - invalid API key or credentials + */ +case class SpeechAuthenticationError(message: String) extends SpeechError + +/** + * Rate limit error - too many requests + */ +case class SpeechRateLimitError(message: String) extends SpeechError + +/** + * Validation error - invalid input parameters + */ +case class SpeechValidationError(message: String) extends SpeechError + +/** + * Unknown or unexpected error + */ +case class SpeechUnknownError(cause: Throwable) extends SpeechError { + def message: String = s"Unknown speech error: ${cause.getMessage}" +} diff --git a/src/main/scala/org/llm4s/speech/model/TTSSynthesisOptions.scala b/src/main/scala/org/llm4s/speech/model/TTSSynthesisOptions.scala new file mode 100644 index 000000000..d95dca2b0 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/model/TTSSynthesisOptions.scala @@ -0,0 +1,22 @@ +package org.llm4s.speech.model + +/** + * Options for text-to-speech synthesis + */ +case class TTSSynthesisOptions( + voice: String = "alloy", + model: String = "tts-1", + responseFormat: String = "mp3", + speed: Double = 1.0, + temperature: Double = 1.0 +) + +/** + * Response from text-to-speech synthesis + */ +case class AudioResponse( + audioData: Array[Byte], + format: String, + duration: Option[Double] = None, + wordCount: Option[Int] = None +) diff --git a/src/main/scala/org/llm4s/speech/provider/AzureSpeechClient.scala b/src/main/scala/org/llm4s/speech/provider/AzureSpeechClient.scala new file mode 100644 index 000000000..e1f26e9da --- /dev/null +++ b/src/main/scala/org/llm4s/speech/provider/AzureSpeechClient.scala @@ -0,0 +1,127 @@ +package org.llm4s.speech.provider + +import requests.Response +import requests.Session +import org.llm4s.speech._ +import org.llm4s.speech.config.AzureSpeechConfig +import org.llm4s.speech.model._ +import ujson._ + +import java.util.Base64 + +class AzureSpeechClient(config: AzureSpeechConfig) extends TTSClient with ASRClient { + + private val session = Session() + + override def synthesize( + text: String, + options: TTSSynthesisOptions + ): Either[SpeechError, AudioResponse] = + try { + val requestBody = Obj( + "text" -> text + ) + + val response = session.post( + s"${config.baseUrl.format(config.region)}/synthesize", + data = requestBody.render(), + headers = Map( + "Ocp-Apim-Subscription-Key" -> config.apiKey, + "Content-Type" -> "application/json", + "X-Microsoft-OutputFormat" -> "audio-16khz-128kbitrate-mono-mp3" + ) + ) + + if (response.statusCode == 200) { + val audioData = response.bytes + Right( + AudioResponse( + audioData = audioData, + format = "mp3" + ) + ) + } else { + handleErrorResponse(response) + } + } catch { + case e: Exception => + Left(SpeechUnknownError(e)) + } + + override def transcribe( + audioData: Array[Byte], + options: ASRTranscriptionOptions + ): Either[SpeechError, TranscriptionResponse] = + try { + // For Azure Speech, we need to use the Speech SDK or REST API + // This is a simplified implementation using REST API + val base64Audio = Base64.getEncoder.encodeToString(audioData) + + val requestBody = Obj( + "audio" -> base64Audio, + "language" -> Str(options.language.getOrElse("en-US")), + "model" -> options.model + ) + + val response = session.post( + s"${config.baseUrl.format(config.region)}/speechtotext/v3.0/transcriptions", + data = requestBody.render(), + headers = Map( + "Ocp-Apim-Subscription-Key" -> config.apiKey, + "Content-Type" -> "application/json" + ) + ) + + if (response.statusCode == 200) { + val responseJson = ujson.read(response.text(), trace = false) + val text = responseJson("DisplayText").str + val language = responseJson.obj.get("Language").map(_.str) + + // Azure doesn't provide detailed segments in the same format as OpenAI + // We'll create a simple segment with the full text + val segments = Seq( + TranscriptionSegment( + id = 0, + start = 0.0, + end = 0.0, // Duration not provided in this simplified response + text = text + ) + ) + + Right( + TranscriptionResponse( + text = text, + language = language, + segments = segments + ) + ) + } else { + handleErrorResponse(response) + } + } catch { + case e: Exception => + Left(SpeechUnknownError(e)) + } + + private def handleErrorResponse(response: Response): Either[SpeechError, Nothing] = { + val errorBody = + try + ujson.read(response.text(), trace = false) + catch { + case _: Exception => Obj("error" -> Obj("message" -> response.text())) + } + + val errorMessage = errorBody.obj + .get("error") + .flatMap(_.obj.get("message")) + .map(_.str) + .getOrElse(s"HTTP ${response.statusCode}: ${response.text()}") + + response.statusCode match { + case 401 => Left(SpeechAuthenticationError(errorMessage)) + case 429 => Left(SpeechRateLimitError(errorMessage)) + case 400 => Left(SpeechValidationError(errorMessage)) + case _ => Left(SpeechUnknownError(new Exception(errorMessage))) + } + } +} diff --git a/src/main/scala/org/llm4s/speech/provider/ElevenLabsClient.scala b/src/main/scala/org/llm4s/speech/provider/ElevenLabsClient.scala new file mode 100644 index 000000000..c9d63f378 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/provider/ElevenLabsClient.scala @@ -0,0 +1,76 @@ +package org.llm4s.speech.provider + +import requests.Response +import requests.Session +import org.llm4s.speech._ +import org.llm4s.speech.config.ElevenLabsConfig +import org.llm4s.speech.model._ +import ujson._ + +class ElevenLabsClient(config: ElevenLabsConfig) extends TTSClient { + + private val session = Session() + + override def synthesize( + text: String, + options: TTSSynthesisOptions + ): Either[SpeechError, AudioResponse] = + try { + val requestBody = Obj( + "text" -> text, + "model_id" -> options.model, + "voice_settings" -> Obj( + "stability" -> 0.5, + "similarity_boost" -> 0.5, + "style" -> 0.0, + "use_speaker_boost" -> true + ) + ) + + val response = session.post( + s"${config.baseUrl}/text-to-speech/${options.voice}", + data = requestBody.render(), + headers = Map( + "xi-api-key" -> config.apiKey, + "Content-Type" -> "application/json" + ) + ) + + if (response.statusCode == 200) { + val audioData = response.bytes + Right( + AudioResponse( + audioData = audioData, + format = "mp3" + ) + ) + } else { + handleErrorResponse(response) + } + } catch { + case e: Exception => + Left(SpeechUnknownError(e)) + } + + private def handleErrorResponse(response: Response): Either[SpeechError, Nothing] = { + val errorBody = + try + ujson.read(response.text(), trace = false) + catch { + case _: Exception => Obj("error" -> Obj("message" -> response.text())) + } + + val errorMessage = errorBody.obj + .get("error") + .flatMap(_.obj.get("message")) + .map(_.str) + .getOrElse(s"HTTP ${response.statusCode}: ${response.text()}") + + response.statusCode match { + case 401 => Left(SpeechAuthenticationError(errorMessage)) + case 429 => Left(SpeechRateLimitError(errorMessage)) + case 400 => Left(SpeechValidationError(errorMessage)) + case _ => Left(SpeechUnknownError(new Exception(errorMessage))) + } + } +} diff --git a/src/main/scala/org/llm4s/speech/provider/GoogleSpeechClient.scala b/src/main/scala/org/llm4s/speech/provider/GoogleSpeechClient.scala new file mode 100644 index 000000000..5201982de --- /dev/null +++ b/src/main/scala/org/llm4s/speech/provider/GoogleSpeechClient.scala @@ -0,0 +1,182 @@ +package org.llm4s.speech.provider + +import requests.Response +import requests.Session +import org.llm4s.speech._ +import org.llm4s.speech.config.GoogleSpeechConfig +import org.llm4s.speech.model._ +import ujson._ + +import java.util.Base64 + +class GoogleSpeechClient(config: GoogleSpeechConfig) extends TTSClient with ASRClient { + + private val session = Session() + + override def synthesize( + text: String, + options: TTSSynthesisOptions + ): Either[SpeechError, AudioResponse] = + try { + val requestBody = Obj( + "input" -> Obj( + "text" -> text + ), + "voice" -> Obj( + "languageCode" -> "en-US", + "name" -> options.voice, + "ssmlGender" -> "NEUTRAL" + ), + "audioConfig" -> Obj( + "audioEncoding" -> "MP3", + "speakingRate" -> options.speed, + "pitch" -> 0.0, + "volumeGainDb" -> 0.0 + ) + ) + + val response = session.post( + s"${config.baseUrl}/text:synthesize", + data = requestBody.render(), + headers = Map( + "Authorization" -> s"Bearer ${config.apiKey}", + "Content-Type" -> "application/json" + ) + ) + + if (response.statusCode == 200) { + val responseJson = ujson.read(response.text(), trace = false) + val audioContent = responseJson("audioContent").str + val audioData = Base64.getDecoder.decode(audioContent) + + Right( + AudioResponse( + audioData = audioData, + format = "mp3" + ) + ) + } else { + handleErrorResponse(response) + } + } catch { + case e: Exception => + Left(SpeechUnknownError(e)) + } + + override def transcribe( + audioData: Array[Byte], + options: ASRTranscriptionOptions + ): Either[SpeechError, TranscriptionResponse] = + try { + val base64Audio = Base64.getEncoder.encodeToString(audioData) + + val requestBody = Obj( + "config" -> Obj( + "encoding" -> "MP3", + "sampleRateHertz" -> 16000, + "languageCode" -> Str(options.language.getOrElse("en-US")), + "model" -> options.model, + "enableWordTimeOffsets" -> true, + "enableAutomaticPunctuation" -> true + ), + "audio" -> Obj( + "content" -> base64Audio + ) + ) + + val response = session.post( + s"${config.baseUrl}/speech:recognize", + data = requestBody.render(), + headers = Map( + "Authorization" -> s"Bearer ${config.apiKey}", + "Content-Type" -> "application/json" + ) + ) + + if (response.statusCode == 200) { + val responseJson = ujson.read(response.text(), trace = false) + val results = responseJson("results").arr + + if (results.nonEmpty) { + val alternatives = results.head("alternatives").arr + if (alternatives.nonEmpty) { + val alternative = alternatives.head + val text = alternative("transcript").str + + // Extract word-level timing information + val words = alternative.obj + .get("words") + .map { wordsJson => + wordsJson.arr.map { word => + val startTime = word("startTime").str.replace("s", "").toDouble + val endTime = word("endTime").str.replace("s", "").toDouble + val wordText = word("word").str + (startTime, endTime, wordText) + }.toSeq + } + .getOrElse(Seq.empty) + + // Create segments based on words + val segments = if (words.nonEmpty) { + Seq( + TranscriptionSegment( + id = 0, + start = words.head._1, + end = words.last._2, + text = text + ) + ) + } else { + Seq( + TranscriptionSegment( + id = 0, + start = 0.0, + end = 0.0, + text = text + ) + ) + } + + Right( + TranscriptionResponse( + text = text, + language = options.language, + segments = segments + ) + ) + } else { + Left(SpeechValidationError("No transcription alternatives found")) + } + } else { + Left(SpeechValidationError("No transcription results found")) + } + } else { + handleErrorResponse(response) + } + } catch { + case e: Exception => + Left(SpeechUnknownError(e)) + } + + private def handleErrorResponse(response: Response): Either[SpeechError, Nothing] = { + val errorBody = + try + ujson.read(response.text(), trace = false) + catch { + case _: Exception => Obj("error" -> Obj("message" -> response.text())) + } + + val errorMessage = errorBody.obj + .get("error") + .flatMap(_.obj.get("message")) + .map(_.str) + .getOrElse(s"HTTP ${response.statusCode}: ${response.text()}") + + response.statusCode match { + case 401 => Left(SpeechAuthenticationError(errorMessage)) + case 429 => Left(SpeechRateLimitError(errorMessage)) + case 400 => Left(SpeechValidationError(errorMessage)) + case _ => Left(SpeechUnknownError(new Exception(errorMessage))) + } + } +} diff --git a/src/main/scala/org/llm4s/speech/provider/OpenAISpeechClient.scala b/src/main/scala/org/llm4s/speech/provider/OpenAISpeechClient.scala new file mode 100644 index 000000000..4afb419f4 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/provider/OpenAISpeechClient.scala @@ -0,0 +1,143 @@ +package org.llm4s.speech.provider + +import requests.Response +import requests.Session +import org.llm4s.speech._ +import org.llm4s.speech.config.OpenAISpeechConfig +import org.llm4s.speech.model._ +import ujson._ + +import java.util.Base64 + +class OpenAISpeechClient(config: OpenAISpeechConfig) extends TTSClient with ASRClient { + + private val session = Session() + + override def synthesize( + text: String, + options: TTSSynthesisOptions + ): Either[SpeechError, AudioResponse] = + try { + val requestBody = Obj( + "model" -> options.model, + "input" -> text, + "voice" -> options.voice, + "response_format" -> options.responseFormat, + "speed" -> options.speed + ) + + val response = session.post( + s"${config.baseUrl}/audio/speech", + data = requestBody.render(), + headers = Map( + "Authorization" -> s"Bearer ${config.apiKey}", + "Content-Type" -> "application/json" + ) + ) + + if (response.statusCode == 200) { + val audioData = response.bytes + Right( + AudioResponse( + audioData = audioData, + format = options.responseFormat + ) + ) + } else { + handleErrorResponse(response) + } + } catch { + case e: Exception => + Left(SpeechUnknownError(e)) + } + + override def transcribe( + audioData: Array[Byte], + options: ASRTranscriptionOptions + ): Either[SpeechError, TranscriptionResponse] = + try { + // Convert audio data to base64 + val base64Audio = Base64.getEncoder.encodeToString(audioData) + + val baseObj = Obj( + "model" -> options.model, + "file" -> base64Audio, + "response_format" -> options.responseFormat, + "temperature" -> options.temperature + ) + + val withLanguage = + options.language.map(lang => baseObj.obj ++ Map("language" -> Str(lang))).getOrElse(baseObj.obj) + val requestBody = + options.prompt.map(prompt => withLanguage ++ Map("prompt" -> Str(prompt))).getOrElse(withLanguage) + + val response = session.post( + s"${config.baseUrl}/audio/transcriptions", + data = requestBody.render(), + headers = Map( + "Authorization" -> s"Bearer ${config.apiKey}", + "Content-Type" -> "application/json" + ) + ) + + if (response.statusCode == 200) { + val responseJson = ujson.read(response.text(), trace = false) + val text = responseJson("text").str + val language = responseJson.obj.get("language").map(_.str) + + val segments = responseJson.obj + .get("segments") + .map { segmentsJson => + segmentsJson.arr.map { segment => + TranscriptionSegment( + id = segment("id").num.toInt, + start = segment("start").num, + end = segment("end").num, + text = segment("text").str, + tokens = segment.obj.get("tokens").map(_.arr.map(_.num.toInt).toSeq).getOrElse(Seq.empty), + temperature = segment.obj.get("temperature").map(_.num), + avgLogprob = segment.obj.get("avg_logprob").map(_.num), + compressionRatio = segment.obj.get("compression_ratio").map(_.num), + noSpeechProb = segment.obj.get("no_speech_prob").map(_.num) + ) + }.toSeq + } + .getOrElse(Seq.empty) + + Right( + TranscriptionResponse( + text = text, + language = language, + segments = segments + ) + ) + } else { + handleErrorResponse(response) + } + } catch { + case e: Exception => + Left(SpeechUnknownError(e)) + } + + private def handleErrorResponse(response: Response): Either[SpeechError, Nothing] = { + val errorBody = + try + ujson.read(response.text(), trace = false) + catch { + case _: Exception => Obj("error" -> Obj("message" -> response.text())) + } + + val errorMessage = errorBody.obj + .get("error") + .flatMap(_.obj.get("message")) + .map(_.str) + .getOrElse(s"HTTP ${response.statusCode}: ${response.text()}") + + response.statusCode match { + case 401 => Left(SpeechAuthenticationError(errorMessage)) + case 429 => Left(SpeechRateLimitError(errorMessage)) + case 400 => Left(SpeechValidationError(errorMessage)) + case _ => Left(SpeechUnknownError(new Exception(errorMessage))) + } + } +} diff --git a/src/main/scala/org/llm4s/speech/provider/SpeechProvider.scala b/src/main/scala/org/llm4s/speech/provider/SpeechProvider.scala new file mode 100644 index 000000000..1d5b64a24 --- /dev/null +++ b/src/main/scala/org/llm4s/speech/provider/SpeechProvider.scala @@ -0,0 +1,10 @@ +package org.llm4s.speech.provider + +sealed trait SpeechProvider +object SpeechProvider { + case object OpenAI extends SpeechProvider + case object Azure extends SpeechProvider + case object Google extends SpeechProvider + case object Amazon extends SpeechProvider + case object ElevenLabs extends SpeechProvider +} diff --git a/src/test/scala/org/llm4s/speech/SpeechTest.scala b/src/test/scala/org/llm4s/speech/SpeechTest.scala new file mode 100644 index 000000000..10f5d1d6f --- /dev/null +++ b/src/test/scala/org/llm4s/speech/SpeechTest.scala @@ -0,0 +1,95 @@ +package org.llm4s.speech + +import org.llm4s.speech.config.OpenAISpeechConfig +import org.llm4s.speech.model._ +import org.llm4s.speech.provider.SpeechProvider +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class SpeechTest extends AnyFlatSpec with Matchers { + + "Speech object" should "provide factory methods" in { + // Test that the Speech object can be instantiated with explicit config + noException should be thrownBy { + val config = OpenAISpeechConfig("test-key", "tts-1") + Speech.ttsClient(SpeechProvider.OpenAI, config) + Speech.asrClient(SpeechProvider.OpenAI, config) + } + } + + "TTSSynthesisOptions" should "have default values" in { + val options = TTSSynthesisOptions() + options.voice shouldBe "alloy" + options.model shouldBe "tts-1" + options.responseFormat shouldBe "mp3" + options.speed shouldBe 1.0 + } + + "ASRTranscriptionOptions" should "have default values" in { + val options = ASRTranscriptionOptions() + options.model shouldBe "whisper-1" + options.language shouldBe None + options.responseFormat shouldBe "json" + options.temperature shouldBe 0.0 + } + + "SpeechError" should "handle different error types" in { + val authError = SpeechAuthenticationError("Invalid API key") + authError.message shouldBe "Invalid API key" + + val rateLimitError = SpeechRateLimitError("Rate limit exceeded") + rateLimitError.message shouldBe "Rate limit exceeded" + + val validationError = SpeechValidationError("Invalid input") + validationError.message shouldBe "Invalid input" + } + + "AudioResponse" should "contain audio data" in { + val audioData = Array[Byte](1, 2, 3, 4) + val response = AudioResponse(audioData, "mp3") + response.audioData shouldBe audioData + response.format shouldBe "mp3" + } + + "TranscriptionResponse" should "contain transcription data" in { + val text = "Hello world" + val response = TranscriptionResponse(text) + response.text shouldBe text + response.language shouldBe None + response.segments shouldBe Seq.empty + } + + "TranscriptionSegment" should "contain segment data" in { + val segment = TranscriptionSegment( + id = 0, + start = 0.0, + end = 1.5, + text = "Hello" + ) + segment.id shouldBe 0 + segment.start shouldBe 0.0 + segment.end shouldBe 1.5 + segment.text shouldBe "Hello" + } + + "SpeechProvider" should "support all providers" in { + SpeechProvider.OpenAI shouldBe SpeechProvider.OpenAI + SpeechProvider.Azure shouldBe SpeechProvider.Azure + SpeechProvider.Google shouldBe SpeechProvider.Google + SpeechProvider.ElevenLabs shouldBe SpeechProvider.ElevenLabs + SpeechProvider.Amazon shouldBe SpeechProvider.Amazon + } + + "OpenAISpeechConfig" should "be created from environment" in { + // This test will fail if environment variables are not set + // but it tests the structure + try { + val config = OpenAISpeechConfig.fromEnv("tts-1") + config.model shouldBe "tts-1" + } catch { + case _: IllegalArgumentException => + // Expected if environment variables are not set + succeed + } + } +} From 304c3bca877151ec7f670cb5ebf39501acee3e82 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 27 Jul 2025 21:19:08 +0000 Subject: [PATCH 2/3] Add TTS/ASR test scripts for ElevenLabs and OpenAI speech services Co-authored-by: mcs23026 --- setup_test.sh | 39 ++++++++++++++++++++++++++++++++++ test_asr.scala | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++ test_tts.scala | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100755 setup_test.sh create mode 100644 test_asr.scala create mode 100644 test_tts.scala diff --git a/setup_test.sh b/setup_test.sh new file mode 100755 index 000000000..24a020340 --- /dev/null +++ b/setup_test.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +echo "๐ŸŽค TTS/ASR Testing Setup" +echo "=========================" + +echo "" +echo "๐Ÿ“‹ Available Free Options:" +echo "1. ElevenLabs (Recommended) - 10,000 chars/month free" +echo "2. OpenAI - $5 credit free (requires card)" +echo "3. Google Cloud - 60 minutes/month free" + +echo "" +echo "๐Ÿš€ Quick Setup for ElevenLabs:" +echo "1. Go to https://elevenlabs.io" +echo "2. Sign up for free account" +echo "3. Go to Profile โ†’ API Key" +echo "4. Copy your API key" + +echo "" +echo "๐Ÿ”‘ Set your API key:" +echo "export ELEVENLABS_API_KEY='your-api-key-here'" +echo "export SPEECH_MODEL='elevenlabs/eleven_monolingual_v1'" + +echo "" +echo "๐Ÿงช Test TTS:" +echo "sbt 'runMain TestTTS'" + +echo "" +echo "๐ŸŽต Test ASR (requires audio file):" +echo "1. Record a short audio clip" +echo "2. Save as 'sample_audio.mp3'" +echo "3. Set OPENAI_API_KEY" +echo "4. Run: sbt 'runMain TestASR'" + +echo "" +echo "๐Ÿ“ Test files created:" +echo "- test_tts.scala (TTS test)" +echo "- test_asr.scala (ASR test)" +echo "- setup_test.sh (this script)" \ No newline at end of file diff --git a/test_asr.scala b/test_asr.scala new file mode 100644 index 000000000..fd8b45c65 --- /dev/null +++ b/test_asr.scala @@ -0,0 +1,57 @@ +import org.llm4s.speech._ +import org.llm4s.speech.config.OpenAISpeechConfig +import org.llm4s.speech.model._ +import org.llm4s.speech.provider._ + +object TestASR { + def main(args: Array[String]): Unit = { + println("Testing ASR functionality with OpenAI...") + + try { + // Create config from environment + val config = OpenAISpeechConfig.fromEnv("whisper-1") + println(s"โœ“ Config created successfully") + + // Create ASR client + val asrClient = Speech.asrClient(SpeechProvider.OpenAI, config) + println(s"โœ“ ASR client created successfully") + + // Test with a sample audio file (you would need to provide one) + val audioFile = new java.io.File("sample_audio.mp3") + + if (audioFile.exists()) { + val audioData = java.nio.file.Files.readAllBytes(audioFile.toPath) + println(s"โœ“ Loaded audio file: ${audioData.length} bytes") + + val options = ASRTranscriptionOptions( + model = "whisper-1", + language = Some("en"), + responseFormat = "json" + ) + + println("Attempting to transcribe audio...") + val result = asrClient.transcribe(audioData, options) + + result match { + case Right(transcription) => + println(s"โœ“ ASR Success!") + println(s"โœ“ Transcribed text: '${transcription.text}'") + println(s"โœ“ Language: ${transcription.language.getOrElse("auto-detected")}") + println(s"โœ“ Segments: ${transcription.segments.length}") + + case Left(error) => + println(s"โœ— ASR Failed: ${error.message}") + } + } else { + println("โœ— No sample audio file found (sample_audio.mp3)") + println("To test ASR, you need to provide an audio file") + println("You can record a short audio clip and save it as 'sample_audio.mp3'") + } + + } catch { + case e: Exception => + println(s"โœ— Setup failed: ${e.getMessage}") + println("Make sure you have set OPENAI_API_KEY environment variable") + } + } +} \ No newline at end of file diff --git a/test_tts.scala b/test_tts.scala new file mode 100644 index 000000000..5234d6984 --- /dev/null +++ b/test_tts.scala @@ -0,0 +1,53 @@ +import org.llm4s.speech._ +import org.llm4s.speech.config.ElevenLabsConfig +import org.llm4s.speech.model._ +import org.llm4s.speech.provider._ + +object TestTTS { + def main(args: Array[String]): Unit = { + println("Testing TTS functionality with ElevenLabs...") + + try { + // Create config from environment + val config = ElevenLabsConfig.fromEnv("eleven_monolingual_v1") + println(s"โœ“ Config created successfully") + + // Create TTS client + val ttsClient = Speech.ttsClient(SpeechProvider.ElevenLabs, config) + println(s"โœ“ TTS client created successfully") + + // Test synthesis + val text = "Hello! This is a test of the text-to-speech functionality." + val options = TTSSynthesisOptions( + voice = "21m00Tcm4TlvDq8ikWAM", // Rachel voice (free) + model = "eleven_monolingual_v1", + responseFormat = "mp3", + speed = 1.0 + ) + + println(s"Attempting to synthesize: '$text'") + val result = ttsClient.synthesize(text, options) + + result match { + case Right(audioResponse) => + println(s"โœ“ TTS Success! Generated ${audioResponse.audioData.length} bytes of audio") + println(s"โœ“ Format: ${audioResponse.format}") + + // Save to file + import java.io.FileOutputStream + val fos = new FileOutputStream("test_output.mp3") + fos.write(audioResponse.audioData) + fos.close() + println(s"โœ“ Audio saved to test_output.mp3") + + case Left(error) => + println(s"โœ— TTS Failed: ${error.message}") + } + + } catch { + case e: Exception => + println(s"โœ— Setup failed: ${e.getMessage}") + println("Make sure you have set ELEVENLABS_API_KEY environment variable") + } + } +} \ No newline at end of file From 12374b7b2cf6041b084d10e69e3505f73052587e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 27 Jul 2025 21:25:41 +0000 Subject: [PATCH 3/3] Remove TTS/ASR test scripts and setup documentation Co-authored-by: mcs23026 --- setup_test.sh | 39 ---------------------------------- test_asr.scala | 57 -------------------------------------------------- test_tts.scala | 53 ---------------------------------------------- 3 files changed, 149 deletions(-) delete mode 100755 setup_test.sh delete mode 100644 test_asr.scala delete mode 100644 test_tts.scala diff --git a/setup_test.sh b/setup_test.sh deleted file mode 100755 index 24a020340..000000000 --- a/setup_test.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -echo "๐ŸŽค TTS/ASR Testing Setup" -echo "=========================" - -echo "" -echo "๐Ÿ“‹ Available Free Options:" -echo "1. ElevenLabs (Recommended) - 10,000 chars/month free" -echo "2. OpenAI - $5 credit free (requires card)" -echo "3. Google Cloud - 60 minutes/month free" - -echo "" -echo "๐Ÿš€ Quick Setup for ElevenLabs:" -echo "1. Go to https://elevenlabs.io" -echo "2. Sign up for free account" -echo "3. Go to Profile โ†’ API Key" -echo "4. Copy your API key" - -echo "" -echo "๐Ÿ”‘ Set your API key:" -echo "export ELEVENLABS_API_KEY='your-api-key-here'" -echo "export SPEECH_MODEL='elevenlabs/eleven_monolingual_v1'" - -echo "" -echo "๐Ÿงช Test TTS:" -echo "sbt 'runMain TestTTS'" - -echo "" -echo "๐ŸŽต Test ASR (requires audio file):" -echo "1. Record a short audio clip" -echo "2. Save as 'sample_audio.mp3'" -echo "3. Set OPENAI_API_KEY" -echo "4. Run: sbt 'runMain TestASR'" - -echo "" -echo "๐Ÿ“ Test files created:" -echo "- test_tts.scala (TTS test)" -echo "- test_asr.scala (ASR test)" -echo "- setup_test.sh (this script)" \ No newline at end of file diff --git a/test_asr.scala b/test_asr.scala deleted file mode 100644 index fd8b45c65..000000000 --- a/test_asr.scala +++ /dev/null @@ -1,57 +0,0 @@ -import org.llm4s.speech._ -import org.llm4s.speech.config.OpenAISpeechConfig -import org.llm4s.speech.model._ -import org.llm4s.speech.provider._ - -object TestASR { - def main(args: Array[String]): Unit = { - println("Testing ASR functionality with OpenAI...") - - try { - // Create config from environment - val config = OpenAISpeechConfig.fromEnv("whisper-1") - println(s"โœ“ Config created successfully") - - // Create ASR client - val asrClient = Speech.asrClient(SpeechProvider.OpenAI, config) - println(s"โœ“ ASR client created successfully") - - // Test with a sample audio file (you would need to provide one) - val audioFile = new java.io.File("sample_audio.mp3") - - if (audioFile.exists()) { - val audioData = java.nio.file.Files.readAllBytes(audioFile.toPath) - println(s"โœ“ Loaded audio file: ${audioData.length} bytes") - - val options = ASRTranscriptionOptions( - model = "whisper-1", - language = Some("en"), - responseFormat = "json" - ) - - println("Attempting to transcribe audio...") - val result = asrClient.transcribe(audioData, options) - - result match { - case Right(transcription) => - println(s"โœ“ ASR Success!") - println(s"โœ“ Transcribed text: '${transcription.text}'") - println(s"โœ“ Language: ${transcription.language.getOrElse("auto-detected")}") - println(s"โœ“ Segments: ${transcription.segments.length}") - - case Left(error) => - println(s"โœ— ASR Failed: ${error.message}") - } - } else { - println("โœ— No sample audio file found (sample_audio.mp3)") - println("To test ASR, you need to provide an audio file") - println("You can record a short audio clip and save it as 'sample_audio.mp3'") - } - - } catch { - case e: Exception => - println(s"โœ— Setup failed: ${e.getMessage}") - println("Make sure you have set OPENAI_API_KEY environment variable") - } - } -} \ No newline at end of file diff --git a/test_tts.scala b/test_tts.scala deleted file mode 100644 index 5234d6984..000000000 --- a/test_tts.scala +++ /dev/null @@ -1,53 +0,0 @@ -import org.llm4s.speech._ -import org.llm4s.speech.config.ElevenLabsConfig -import org.llm4s.speech.model._ -import org.llm4s.speech.provider._ - -object TestTTS { - def main(args: Array[String]): Unit = { - println("Testing TTS functionality with ElevenLabs...") - - try { - // Create config from environment - val config = ElevenLabsConfig.fromEnv("eleven_monolingual_v1") - println(s"โœ“ Config created successfully") - - // Create TTS client - val ttsClient = Speech.ttsClient(SpeechProvider.ElevenLabs, config) - println(s"โœ“ TTS client created successfully") - - // Test synthesis - val text = "Hello! This is a test of the text-to-speech functionality." - val options = TTSSynthesisOptions( - voice = "21m00Tcm4TlvDq8ikWAM", // Rachel voice (free) - model = "eleven_monolingual_v1", - responseFormat = "mp3", - speed = 1.0 - ) - - println(s"Attempting to synthesize: '$text'") - val result = ttsClient.synthesize(text, options) - - result match { - case Right(audioResponse) => - println(s"โœ“ TTS Success! Generated ${audioResponse.audioData.length} bytes of audio") - println(s"โœ“ Format: ${audioResponse.format}") - - // Save to file - import java.io.FileOutputStream - val fos = new FileOutputStream("test_output.mp3") - fos.write(audioResponse.audioData) - fos.close() - println(s"โœ“ Audio saved to test_output.mp3") - - case Left(error) => - println(s"โœ— TTS Failed: ${error.message}") - } - - } catch { - case e: Exception => - println(s"โœ— Setup failed: ${e.getMessage}") - println("Make sure you have set ELEVENLABS_API_KEY environment variable") - } - } -} \ No newline at end of file