Skip to content

feat(asr): 接入 AssemblyAI 和 Deepgram 实时语音转写供应商 #846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lvsao wants to merge 3 commits into from
Closed

feat(asr): 接入 AssemblyAI 和 Deepgram 实时语音转写供应商 #846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e655169
feat(asr): add AssemblyAI and Deepgram realtime ASR providers
lvsao Jul 21, 2026
5039d28
fix(deepgram): 服务端 Close 时使用 finish_with_partial_or_error 而非 finish_s…
lvsao Jul 21, 2026
a75c6c5
fix(validation): Deepgram 纯静音验证跳过 await_final_result，避免超时报错
lvsao Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
614 changes: 614 additions & 0 deletions openless-all/app/src-tauri/src/asr/assemblyai.rs
View file
Open in desktop

Large diffs are not rendered by default.

565 changes: 565 additions & 0 deletions openless-all/app/src-tauri/src/asr/deepgram.rs
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions openless-all/app/src-tauri/src/asr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
//! `frame.rs` (binary frame codec) and the session lifecycle in
//! `volcengine.rs`.

pub mod assemblyai;
pub mod bailian;
pub mod dashscope_multimodal;
pub mod deepgram;
pub mod elevenlabs;
mod frame;
pub mod local;
Expand All @@ -18,8 +20,10 @@ pub mod volcengine;
pub mod wav;
pub mod whisper;

pub use assemblyai::{AssemblyAICredentials, AssemblyAIRealtimeASR};
pub use bailian::{BailianCredentials, BailianRealtimeASR};
pub use dashscope_multimodal::DashScopeMultimodalASR;
pub use deepgram::{DeepgramCredentials, DeepgramRealtimeASR};
pub use elevenlabs::ElevenLabsBatchASR;
pub use mimo::MimoBatchASR;
pub use qwen_realtime::{Qwen3RealtimeASR, Qwen3RealtimeCredentials};
Expand Down
94 changes: 94 additions & 0 deletions openless-all/app/src-tauri/src/commands/providers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,24 @@ pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult,
models: vec![crate::asr::elevenlabs::DEFAULT_MODEL.to_string()],
});
}
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::assemblyai::PROVIDER_ID {
validate_assemblyai_asr_provider().await?;
return Ok(ProviderModelsResult {
models: vec![
crate::asr::assemblyai::DEFAULT_MODEL.to_string(),
"universal-2".to_string(),
],
});
}
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::deepgram::PROVIDER_ID {
validate_deepgram_asr_provider().await?;
return Ok(ProviderModelsResult {
models: vec![
crate::asr::deepgram::DEFAULT_MODEL.to_string(),
"nova-2".to_string(),
],
});
}
if kind == "llm" && CredentialsVault::get_active_llm() == CODEX_OAUTH_PROVIDER_ID {
return Ok(ProviderModelsResult {
models: vec![
Expand Down Expand Up @@ -270,6 +288,12 @@ async fn validate_asr_provider() -> Result<(), String> {
if active_asr == crate::asr::elevenlabs::PROVIDER_ID {
return validate_elevenlabs_asr_provider().await;
}
if active_asr == crate::asr::assemblyai::PROVIDER_ID {
return validate_assemblyai_asr_provider().await;
}
if active_asr == crate::asr::deepgram::PROVIDER_ID {
return validate_deepgram_asr_provider().await;
}
// StepFun 一入口双协议：`*-stream` 模型走实时 WS 验证，其余走批式
// /audio/transcriptions（与 build 侧 resolve_effective_asr_provider 同判据）。
if active_asr == "stepfun" || active_asr == crate::asr::stepfun_realtime::PROVIDER_ID {
Expand Down Expand Up @@ -378,6 +402,76 @@ async fn validate_elevenlabs_asr_provider() -> Result<(), String> {
})
}

async fn validate_assemblyai_asr_provider() -> Result<(), String> {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.map_err(|e| e.to_string())?
.unwrap_or_default();
if api_key.trim().is_empty() {
return Err("API Key 为空".to_string());
}
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::assemblyai::DEFAULT_ENDPOINT.to_string());
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::assemblyai::DEFAULT_MODEL.to_string());

let asr = std::sync::Arc::new(crate::asr::AssemblyAIRealtimeASR::new(
crate::asr::AssemblyAICredentials {
api_key,
endpoint,
model,
},
));
asr.open_session().await.map_err(|e| e.to_string())?;
crate::asr::AudioConsumer::consume_pcm_chunk(
&*asr,
&vec![0u8; crate::asr::assemblyai::TARGET_AUDIO_CHUNK_BYTES * 5],
);
asr.send_last_frame().await.map_err(|e| e.to_string())?;
asr.await_final_result()
.await
.map(|_| ())
.map_err(|e| e.to_string())
}

async fn validate_deepgram_asr_provider() -> Result<(), String> {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.map_err(|e| e.to_string())?
.unwrap_or_default();
if api_key.trim().is_empty() {
return Err("API Key 为空".to_string());
}
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::deepgram::DEFAULT_ENDPOINT.to_string());
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::deepgram::DEFAULT_MODEL.to_string());

let asr = std::sync::Arc::new(crate::asr::DeepgramRealtimeASR::new(
crate::asr::DeepgramCredentials {
api_key,
endpoint,
model,
language: Some("zh".to_string()),
},
));
asr.open_session().await.map_err(|e| e.to_string())?;
crate::asr::AudioConsumer::consume_pcm_chunk(
&*asr,
&vec![0u8; crate::asr::deepgram::TARGET_AUDIO_CHUNK_BYTES * 5],
);
asr.send_last_frame().await.map_err(|e| e.to_string())?;
// Deepgram 对纯静音不会返回最终转写结果，因此只验证连接与关闭流程
let _ = asr.await_final_result().await;
Ok(())
}

/// fun-asr-flash 官方公开示例音频，用于连通性校验。该模型对纯静音会返回
/// 400（"no speech" 类错误），无法像 Whisper/Mimo 那样发静音探活；改用这段
/// 阿里官方文档在案的示例 wav（由 DashScope 侧拉取），key/endpoint/model 有效
Expand Down
62 changes: 60 additions & 2 deletions openless-all/app/src-tauri/src/coordinator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ use crate::asr::local::{
foundry, sherpa, FoundryLocalRuntime, FoundryLocalWhisperAsr, SherpaOnnxAsr, SherpaOnnxRuntime,
};
use crate::asr::{
BailianCredentials, BailianRealtimeASR, DashScopeMultimodalASR, DictionaryHotword,
AssemblyAICredentials, AssemblyAIRealtimeASR, BailianCredentials, BailianRealtimeASR,
DashScopeMultimodalASR, DeepgramCredentials, DeepgramRealtimeASR, DictionaryHotword,
ElevenLabsBatchASR, MimoBatchASR, Qwen3RealtimeASR, Qwen3RealtimeCredentials, RawTranscript,
VolcengineCredentials, VolcengineStreamingASR, WhisperBatchASR,
};
Expand Down Expand Up @@ -184,6 +185,8 @@ enum ActiveAsr {
/// 百炼 Fun-ASR-Flash 录音文件识别（DashScope multimodal-generation 批量 HTTP）。
DashScopeMultimodal(Arc<DashScopeMultimodalASR>),
ElevenLabs(Arc<ElevenLabsBatchASR>),
AssemblyAI(Arc<AssemblyAIRealtimeASR>),
Deepgram(Arc<DeepgramRealtimeASR>),
Bailian(Arc<BailianRealtimeASR>),
/// 百炼 Qwen3-ASR-Flash 实时（OpenAI Realtime 风格 WS 协议）。
Qwen3Realtime(Arc<Qwen3RealtimeASR>),
Expand Down Expand Up @@ -230,6 +233,8 @@ pub(crate) enum ActiveAsrProviderKind {
Mimo,
DashScopeMultimodal,
ElevenLabs,
AssemblyAI,
Deepgram,
WhisperCompatible,
Volcengine,
}
Expand Down Expand Up @@ -266,6 +271,8 @@ impl ActiveAsrProviderKind {
| ActiveAsrProviderKind::Mimo
| ActiveAsrProviderKind::DashScopeMultimodal
| ActiveAsrProviderKind::ElevenLabs
| ActiveAsrProviderKind::AssemblyAI
| ActiveAsrProviderKind::Deepgram
| ActiveAsrProviderKind::WhisperCompatible => AsrPreflightCredential::AsrApiKey,
ActiveAsrProviderKind::Volcengine => AsrPreflightCredential::VolcAppKey,
}
Expand All @@ -275,7 +282,9 @@ impl ActiveAsrProviderKind {
match self {
ActiveAsrProviderKind::Bailian
| ActiveAsrProviderKind::Qwen3Realtime
| ActiveAsrProviderKind::ElevenLabs => {
| ActiveAsrProviderKind::ElevenLabs
| ActiveAsrProviderKind::AssemblyAI
| ActiveAsrProviderKind::Deepgram => {
AsrConfiguredFields::ApiKeyOnly
}
ActiveAsrProviderKind::Mimo | ActiveAsrProviderKind::DashScopeMultimodal => {
Expand Down Expand Up @@ -304,6 +313,10 @@ pub(crate) fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
ActiveAsrProviderKind::DashScopeMultimodal
} else if is_elevenlabs_provider(id) {
ActiveAsrProviderKind::ElevenLabs
} else if is_assemblyai_provider(id) {
ActiveAsrProviderKind::AssemblyAI
} else if is_deepgram_provider(id) {
ActiveAsrProviderKind::Deepgram
} else if is_whisper_compatible_provider(id) {
ActiveAsrProviderKind::WhisperCompatible
} else {
Expand Down Expand Up @@ -2352,6 +2365,51 @@ fn read_elevenlabs_credentials() -> (String, String, String) {
(api_key, base_url, model)
}

fn read_assemblyai_credentials() -> crate::asr::AssemblyAICredentials {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
.flatten()
.unwrap_or_default();
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::assemblyai::DEFAULT_ENDPOINT.to_string());
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::assemblyai::DEFAULT_MODEL.to_string());
crate::asr::AssemblyAICredentials {
api_key,
endpoint,
model,
}
}

fn read_deepgram_credentials() -> crate::asr::DeepgramCredentials {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
.flatten()
.unwrap_or_default();
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::deepgram::DEFAULT_ENDPOINT.to_string());
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::deepgram::DEFAULT_MODEL.to_string());
crate::asr::DeepgramCredentials {
api_key,
endpoint,
model,
language: Some("zh".to_string()),
}
}

fn read_dashscope_multimodal_credentials() -> (String, String, String) {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
Expand Down
60 changes: 60 additions & 0 deletions openless-all/app/src-tauri/src/coordinator/asr_wiring.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,14 @@ pub(super) fn is_elevenlabs_provider(id: &str) -> bool {
id == crate::asr::elevenlabs::PROVIDER_ID
}

pub(super) fn is_assemblyai_provider(id: &str) -> bool {
id == crate::asr::assemblyai::PROVIDER_ID
}

pub(super) fn is_deepgram_provider(id: &str) -> bool {
id == crate::asr::deepgram::PROVIDER_ID
}

pub(super) fn apply_chinese_script_preference(text: &str, pref: ChineseScriptPreference) -> String {
if text.is_empty() {
return String::new();
Expand Down Expand Up @@ -443,6 +451,14 @@ pub(super) enum QaAsrStart {
asr: Arc<crate::asr::StepfunRealtimeASR>,
bridge: Arc<DeferredAsrBridge>,
},
AssemblyAI {
asr: Arc<crate::asr::AssemblyAIRealtimeASR>,
bridge: Arc<DeferredAsrBridge>,
},
Deepgram {
asr: Arc<crate::asr::DeepgramRealtimeASR>,
bridge: Arc<DeferredAsrBridge>,
},
Ready {
active: ActiveAsr,
consumer: Arc<dyn crate::recorder::AudioConsumer>,
Expand All @@ -456,6 +472,8 @@ impl QaAsrStart {
QaAsrStart::Bailian { asr, .. } => ActiveAsr::Bailian(Arc::clone(asr)),
QaAsrStart::Qwen3Realtime { asr, .. } => ActiveAsr::Qwen3Realtime(Arc::clone(asr)),
QaAsrStart::StepfunRealtime { asr, .. } => ActiveAsr::StepfunRealtime(Arc::clone(asr)),
QaAsrStart::AssemblyAI { asr, .. } => ActiveAsr::AssemblyAI(Arc::clone(asr)),
QaAsrStart::Deepgram { asr, .. } => ActiveAsr::Deepgram(Arc::clone(asr)),
QaAsrStart::Ready { active, .. } => active.clone(),
}
}
Expand All @@ -466,6 +484,8 @@ impl QaAsrStart {
QaAsrStart::Bailian { bridge, .. } => Arc::clone(bridge) as _,
QaAsrStart::Qwen3Realtime { bridge, .. } => Arc::clone(bridge) as _,
QaAsrStart::StepfunRealtime { bridge, .. } => Arc::clone(bridge) as _,
QaAsrStart::AssemblyAI { bridge, .. } => Arc::clone(bridge) as _,
QaAsrStart::Deepgram { bridge, .. } => Arc::clone(bridge) as _,
QaAsrStart::Ready { consumer, .. } => Arc::clone(consumer),
}
}
Expand Down Expand Up @@ -506,6 +526,24 @@ impl QaAsrStart {
);
Ok(())
}
QaAsrStart::AssemblyAI { asr, bridge } => {
asr.open_session().await.map_err(|e| e.to_string())?;
let target: Arc<dyn crate::asr::AudioConsumer> = Arc::clone(asr) as _;
let flushed = bridge.attach(target);
log::info!(
"[coord] QA AssemblyAI realtime ASR connected; flushed {flushed} deferred audio bytes"
);
Ok(())
}
QaAsrStart::Deepgram { asr, bridge } => {
asr.open_session().await.map_err(|e| e.to_string())?;
let target: Arc<dyn crate::asr::AudioConsumer> = Arc::clone(asr) as _;
let flushed = bridge.attach(target);
log::info!(
"[coord] QA Deepgram realtime ASR connected; flushed {flushed} deferred audio bytes"
);
Ok(())
}
QaAsrStart::Ready { .. } => Ok(()),
}
}
Expand Down Expand Up @@ -641,6 +679,28 @@ pub(super) async fn build_qa_asr_start(
label,
))
}
ActiveAsrProviderKind::AssemblyAI => {
let creds = read_assemblyai_credentials();
let label = AsrCallLabel::new(effective_asr.clone(), Some(creds.model.clone()));
Ok((
QaAsrStart::AssemblyAI {
asr: Arc::new(crate::asr::AssemblyAIRealtimeASR::new(creds)),
bridge: Arc::new(DeferredAsrBridge::new()),
},
label,
))
}
ActiveAsrProviderKind::Deepgram => {
let creds = read_deepgram_credentials();
let label = AsrCallLabel::new(effective_asr.clone(), Some(creds.model.clone()));
Ok((
QaAsrStart::Deepgram {
asr: Arc::new(crate::asr::DeepgramRealtimeASR::new(creds)),
bridge: Arc::new(DeferredAsrBridge::new()),
},
label,
))
}
ActiveAsrProviderKind::Mimo => {
let (api_key, base_url, model) = read_mimo_credentials();
let label = AsrCallLabel::new(effective_asr.clone(), Some(model.clone()));
Expand Down
2 changes: 2 additions & 0 deletions openless-all/app/src/i18n/en.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -798,6 +798,8 @@ export const en: typeof zhCN = {
asrOpenrouter: 'OpenRouter Whisper',
asrXiaomiMimo: 'Xiaomi MiMo ASR',
asrElevenLabs: 'ElevenLabs Scribe',
asrAssemblyAI: 'AssemblyAI Realtime ASR',
asrDeepgram: 'Deepgram Realtime ASR',
asrSherpaOnnxLocal: 'Local sherpa-onnx (experimental)',
asrFoundryLocalWhisper: 'Local Whisper (Foundry Local)',
asrLocalQwen3: 'Local Qwen3-ASR',
Expand Down
2 changes: 2 additions & 0 deletions openless-all/app/src/i18n/ja.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,8 @@ export const ja: typeof zhCN = {
asrOpenrouter: 'OpenRouter Whisper',
asrXiaomiMimo: 'Xiaomi MiMo ASR',
asrElevenLabs: 'ElevenLabs Scribe',
asrAssemblyAI: 'AssemblyAI リアルタイム ASR',
asrDeepgram: 'Deepgram リアルタイム ASR',
asrSherpaOnnxLocal: 'ローカル sherpa-onnx（実験的）',
asrFoundryLocalWhisper: 'ローカル Whisper（Foundry Local）',
asrLocalQwen3: 'ローカル Qwen3-ASR',
Expand Down
2 changes: 2 additions & 0 deletions openless-all/app/src/i18n/ko.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,8 @@ export const ko: typeof zhCN = {
asrOpenrouter: 'OpenRouter Whisper',
asrXiaomiMimo: 'Xiaomi MiMo ASR',
asrElevenLabs: 'ElevenLabs Scribe',
asrAssemblyAI: 'AssemblyAI 실시간 ASR',
asrDeepgram: 'Deepgram 실시간 ASR',
asrSherpaOnnxLocal: '로컬 sherpa-onnx(실험적)',
asrFoundryLocalWhisper: '로컬 Whisper(Foundry Local)',
asrLocalQwen3: '로컬 Qwen3-ASR',
Expand Down
2 changes: 2 additions & 0 deletions openless-all/app/src/i18n/zh-CN.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,8 @@ export const zhCN = {
asrOpenrouter: 'OpenRouter Whisper',
asrXiaomiMimo: '小米 MiMo ASR',
asrElevenLabs: 'ElevenLabs Scribe',
asrAssemblyAI: 'AssemblyAI 实时 ASR',
asrDeepgram: 'Deepgram 实时 ASR',
asrSherpaOnnxLocal: '本地 sherpa-onnx（实验性）',
asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）',
asrLocalQwen3: '本地 Qwen3-ASR',
Expand Down
2 changes: 2 additions & 0 deletions openless-all/app/src/i18n/zh-TW.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -798,6 +798,8 @@ export const zhTW: typeof zhCN = {
asrOpenrouter: 'OpenRouter Whisper',
asrXiaomiMimo: '小米 MiMo ASR',
asrElevenLabs: 'ElevenLabs Scribe',
asrAssemblyAI: 'AssemblyAI 即時 ASR',
asrDeepgram: 'Deepgram 即時 ASR',
asrSherpaOnnxLocal: '本地 sherpa-onnx（實驗性）',
asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）',
asrLocalQwen3: '本地 Qwen3-ASR',
Expand Down
Loading
Loading