From 2bc530679549fc17af4ddef79f31160b45feb47f Mon Sep 17 00:00:00 2001 From: Cea <61349137+ceastld@users.noreply.github.com> Date: Fri, 7 Aug 2026 19:47:50 +0800 Subject: [PATCH 1/4] feat: add lightweight native voice preview --- .github/workflows/native-preview.yml | 46 +++ .gitignore | 5 + AGENTS.md | 6 + README.md | 3 + docs/quicker-integration-research.md | 2 + native-app/BENCHMARKS.md | 43 +++ native-app/Directory.Build.props | 10 + native-app/Directory.Build.targets | 3 + native-app/Directory.Packages.props | 10 + native-app/README.md | 50 ++++ native-app/VoxType.Native.slnx | 5 + native-app/build.ps1 | 50 ++++ native-app/src/VoxType.NativeApp/App.xaml | 50 ++++ native-app/src/VoxType.NativeApp/App.xaml.cs | 7 + .../src/VoxType.NativeApp/MainWindow.xaml | 172 +++++++++++ .../src/VoxType.NativeApp/MainWindow.xaml.cs | 278 ++++++++++++++++++ .../VoxType.NativeApp.csproj | 22 ++ .../VoxType.NativeCore/AudioCaptureSession.cs | 110 +++++++ .../src/VoxType.NativeCore/ModelCatalog.cs | 91 ++++++ .../src/VoxType.NativeCore/ModelManager.cs | 206 +++++++++++++ .../VoxType.NativeCore/SenseVoiceEngine.cs | 124 ++++++++ .../VoxType.NativeCore.csproj | 14 + .../src/VoxType.NativeCore/VoxTypePaths.cs | 37 +++ .../src/VoxType.NativeCore/WaveFileAudio.cs | 42 +++ native-app/src/VoxType.NativeSmoke/Program.cs | 56 ++++ .../VoxType.NativeSmoke.csproj | 14 + 26 files changed, 1456 insertions(+) create mode 100644 .github/workflows/native-preview.yml create mode 100644 native-app/BENCHMARKS.md create mode 100644 native-app/Directory.Build.props create mode 100644 native-app/Directory.Build.targets create mode 100644 native-app/Directory.Packages.props create mode 100644 native-app/README.md create mode 100644 native-app/VoxType.Native.slnx create mode 100644 native-app/build.ps1 create mode 100644 native-app/src/VoxType.NativeApp/App.xaml create mode 100644 native-app/src/VoxType.NativeApp/App.xaml.cs create mode 100644 native-app/src/VoxType.NativeApp/MainWindow.xaml create mode 100644 native-app/src/VoxType.NativeApp/MainWindow.xaml.cs create mode 100644 native-app/src/VoxType.NativeApp/VoxType.NativeApp.csproj create mode 100644 native-app/src/VoxType.NativeCore/AudioCaptureSession.cs create mode 100644 native-app/src/VoxType.NativeCore/ModelCatalog.cs create mode 100644 native-app/src/VoxType.NativeCore/ModelManager.cs create mode 100644 native-app/src/VoxType.NativeCore/SenseVoiceEngine.cs create mode 100644 native-app/src/VoxType.NativeCore/VoxType.NativeCore.csproj create mode 100644 native-app/src/VoxType.NativeCore/VoxTypePaths.cs create mode 100644 native-app/src/VoxType.NativeCore/WaveFileAudio.cs create mode 100644 native-app/src/VoxType.NativeSmoke/Program.cs create mode 100644 native-app/src/VoxType.NativeSmoke/VoxType.NativeSmoke.csproj diff --git a/.github/workflows/native-preview.yml b/.github/workflows/native-preview.yml new file mode 100644 index 0000000..3ec1d80 --- /dev/null +++ b/.github/workflows/native-preview.yml @@ -0,0 +1,46 @@ +name: Native Preview + +on: + workflow_dispatch: + pull_request: + paths: + - 'native-app/**' + - 'catalog/models.json' + - '.github/workflows/native-preview.yml' + push: + branches: [main] + paths: + - 'native-app/**' + - 'catalog/models.json' + - '.github/workflows/native-preview.yml' + +permissions: + contents: read + +jobs: + build: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup .NET 10 + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + - name: Build + shell: pwsh + run: dotnet build ./native-app/VoxType.Native.slnx -c Release + + - name: Package framework-dependent preview + shell: pwsh + run: pwsh -NoProfile -File ./native-app/build.ps1 -SkipBuild + + - name: Upload preview zip + uses: actions/upload-artifact@v4 + with: + name: VoxType-NativePreview-win-x64 + path: native-app/publish/VoxType.NativePreview-win-x64.zip + if-no-files-found: error + retention-days: 14 diff --git a/.gitignore b/.gitignore index b943aa1..dc24b8c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,11 @@ app/src-tauri/bundle-resources/runtime/voxtype-runtime/* !app/src-tauri/bundle-resources/runtime/voxtype-runtime/.gitkeep app/src-tauri/bundle-resources/catalog/models.json +# Native .NET preview +native-app/**/bin/ +native-app/**/obj/ +native-app/publish/ + # Plugin plugin/bin/ plugin/obj/ diff --git a/AGENTS.md b/AGENTS.md index 49f3bf0..d5066db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ VoxType 是 Windows 本地语音输入应用,也是 Quicker 语音输入扩展 | 目录 | 职责 | |------|------| | `app/` | Tauri 桌面宿主:热键、录音、浮层、设置、文本提交 | +| `native-app/` | .NET 10/WPF 原生体验版与可复用识别 core;不依赖 Python runtime | | `runtime/` | 独立 ASR sidecar:模型加载、识别、WebSocket 会话 | | `plugin/` | Quicker 兼容适配器;不承载识别引擎 | | `catalog/` | 随安装包发布的模型元数据;模型权重不入库 | @@ -44,6 +45,11 @@ pnpm install pnpm build cargo test --manifest-path src-tauri/Cargo.toml +# Native .NET preview +cd ../../native-app +dotnet build VoxType.Native.slnx -c Release +pwsh ./build.ps1 -SkipBuild + # 模型目录 cd .. pwsh ./scripts/validate-models-catalog.ps1 diff --git a/README.md b/README.md index 7de0e59..6bcffcb 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ ASR runtime uses [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) (SenseVoic | `app/` | Tauri client — settings, overlay, hotkeys, direct typing, Quicker HTTP API | | `plugin/` | Quicker plugin — button triggers dictation | | `catalog/models.json` | Public model catalog for settings UI | +| `native-app/` | .NET 10/WPF native preview — no Python sidecar | ## Quick start (dev) @@ -35,6 +36,8 @@ pnpm tauri dev Data directory: `%LOCALAPPDATA%\VoxType\` (models, settings). +For the smaller native preview intended to validate future `QuickerPc/Quicker` integration, see [`native-app/`](native-app/README.md). + ## Model download Model **weights** are not bundled (too large). Model **download URLs** are written in `catalog/models.json` and ship inside the installer at `catalog/models.json` — the app reads this file locally (no remote catalog fetch). Users download weights from the settings UI on first use. diff --git a/docs/quicker-integration-research.md b/docs/quicker-integration-research.md index 5b37db1..77a6369 100644 --- a/docs/quicker-integration-research.md +++ b/docs/quicker-integration-research.md @@ -17,6 +17,7 @@ Windows Text Services Framework(TSF)适合长期追求输入法级体验, - sherpa-onnx + SenseVoice 已能离线工作,runtime 目录也预留了其它识别后端。 - 当前桌面端直接模拟 Unicode 键入;遇到管理员窗口、特殊编辑器、终端或复杂输入法状态时兼容性有限。 - 当前 Quicker 插件经 `127.0.0.1:6020` 调用 VoxType,只是外部控制桥,不是 Quicker 原生 provider。 +- `native-app/` 已提供 `.NET 10 + WPF + sherpa-onnx C#` 的无 Python 体验版,用于验证更小的原生 worker;它不改变“Quicker 主进程不直接承载模型”的隔离边界。 ### QuickerPc/Quicker @@ -125,6 +126,7 @@ v2 在会话前增加能力握手,至少返回: - 抽出 JSON schema、示例音频和协议一致性测试。 - runtime 增加 capabilities/版本信息和控制 API token。 - 为 .NET 提供薄客户端;不把模型推理依赖带进客户端包。 +- 以 `native-app/VoxType.NativeCore` 为基础增加无 UI worker,完成稳定性与内存驻留测试后,再决定是否替换 Python runtime。 ### P2:Quicker 原生 provider diff --git a/native-app/BENCHMARKS.md b/native-app/BENCHMARKS.md new file mode 100644 index 0000000..50ffcaf --- /dev/null +++ b/native-app/BENCHMARKS.md @@ -0,0 +1,43 @@ +# Native Preview 基线 + +> 日期:2026-08-07。数据用于比较架构,不代表所有设备上的保证值。 + +## 环境 + +- CPU:Intel Core i9-14900HX(24 核 / 32 线程) +- 系统:Windows x64 +- SDK / Runtime:.NET 10.0.301 / .NET Desktop Runtime 10 +- 推理:sherpa-onnx 1.13.4,CPU provider,最多 4 线程 +- 模型:SenseVoice Small INT8,239,233,841-byte ONNX +- 音频:16 kHz mono,5.00 秒中文 WAV + +## 结果 + +| 指标 | 结果 | +|------|------| +| 模型加载 | 1.27–1.63 s | +| 稳态识别 | 0.16–0.23 s,RTF 0.032–0.045 | +| 发布目录 | 21.4 MB | +| zip | 8.1 MB | +| 模型加载后的进程 Working Set | 约 506 MB | +| 模型加载后的 Private Memory | 约 450 MB | + +`v0.1.12` 的 Python/PyInstaller + Tauri NSIS 安装包为 25,001,555 bytes(约 23.8 MiB)。Native Preview 的 zip 为约 8.1 MiB,压缩体积降低约 66%;两者都不包含模型权重。重复运行存在正常调度波动,因此表中保留两轮实测范围。 + +## 解释 + +- 首次加载和后续识别必须分开测量;UI 启动时预热模型,因此用户松开录音后的主要等待接近“稳态识别”。 +- 体积下降主要来自移除 Python、NumPy、aiohttp 和 PyInstaller runtime;发布目录仍主要由 `onnxruntime.dll` 与 `sherpa-onnx-c-api.dll` 构成。 +- 内存主要由 229 MiB 模型、ONNX Runtime 权重/工作区和 WPF 进程构成。后续 Quicker 接入仍建议使用独立原生 worker,以隔离这部分内存和 native DLL 生命周期。 +- 当前只启用 CPU provider。DirectML/CUDA 是否值得加入,要以更多硬件上的启动失败率、驱动兼容和安装包增量为准。 + +## 复现 + +```powershell +pwsh ./build.ps1 + +dotnet run --project ./src/VoxType.NativeSmoke -c Release -- ` + ` + "$env:LOCALAPPDATA\VoxType\models\sensevoice" ` + 3 +``` diff --git a/native-app/Directory.Build.props b/native-app/Directory.Build.props new file mode 100644 index 0000000..b377359 --- /dev/null +++ b/native-app/Directory.Build.props @@ -0,0 +1,10 @@ + + + + latest + true + $(DefaultItemExcludes);publish/**;**/publish/** + true + true + + diff --git a/native-app/Directory.Build.targets b/native-app/Directory.Build.targets new file mode 100644 index 0000000..a25c1a7 --- /dev/null +++ b/native-app/Directory.Build.targets @@ -0,0 +1,3 @@ + + + diff --git a/native-app/Directory.Packages.props b/native-app/Directory.Packages.props new file mode 100644 index 0000000..d4515eb --- /dev/null +++ b/native-app/Directory.Packages.props @@ -0,0 +1,10 @@ + + + true + + + + + + + diff --git a/native-app/README.md b/native-app/README.md new file mode 100644 index 0000000..87bafe8 --- /dev/null +++ b/native-app/README.md @@ -0,0 +1,50 @@ +# VoxType Native Preview + +面向 `QuickerPc/Quicker` 的最小原生语音识别体验版。它使用 `.NET 10 + WPF + NAudio + sherpa-onnx C#`,直接加载 SenseVoice,不启动 Python、aiohttp 或 PyInstaller runtime。 + +## 体验 + +```powershell +cd native-app +dotnet run --project ./src/VoxType.NativeApp -c Release +``` + +首次启动点击“下载模型”。模型约 229 MB,保存到 `%LOCALAPPDATA%\VoxType\models\sensevoice`,与现有 VoxType 共用,不进入应用安装包。模型就绪后: + +1. 点击“开始录音”。 +2. 说话后点击“结束并识别”。 +3. 查看耗时和 RTF,复制识别结果。 + +当前预览版只做录音、识别和复制,不注册全局热键、不自动向其它窗口输入文字。这让首轮测试集中在原生运行稳定性、识别性能和打包体积。 + +## 构建小体积包 + +```powershell +pwsh ./build.ps1 +``` + +产物: + +- `publish/win-x64/`:framework-dependent Windows x64 应用。 +- `publish/VoxType.NativePreview-win-x64.zip`:可直接解压体验的压缩包。 + +首轮实测 zip 为 8.1 MB,模型加载后的 5 秒中文音频稳态识别为 0.16–0.23 秒。环境、内存与复现命令见 [`BENCHMARKS.md`](BENCHMARKS.md)。 + +每次 `main` 上的 native-app 改动也会生成一个保留 14 天的 GitHub Actions artifact:打开仓库的 **Actions → Native Preview**,下载 `VoxType-NativePreview-win-x64`。 + +应用要求已安装 .NET 10 Desktop Runtime。Quicker V2 已使用同一运行时;若未来面向没有 Quicker 的独立用户发布,可另做 self-contained 包,但体积会显著增加。 + +## 命令行识别验证 + +```powershell +dotnet run --project ./src/VoxType.NativeSmoke -c Release -- [model-directory] [repeat-count] +``` + +第二个可选参数是模型目录,第三个参数可在同一进程重复识别,用于区分模型加载耗时和稳态 decode 耗时。该工具用于 CI/开发验证,不进入预览应用压缩包。 + +## 当前边界 + +- 只启用 Windows x64 CPU provider,优先保证部署一致性;GPU provider 留待实测收益与驱动兼容性后再开放。 +- 识别器在进程内只加载一次,每次录音创建独立 stream,避免重复加载模型。 +- 模型按 catalog 下载并支持 `.part` 断点续传。 +- 原生识别调用不可被 sherpa-onnx 中途打断;关闭窗口会阻止新会话,但正在执行的单次 decode 需要自然返回。 diff --git a/native-app/VoxType.Native.slnx b/native-app/VoxType.Native.slnx new file mode 100644 index 0000000..91669aa --- /dev/null +++ b/native-app/VoxType.Native.slnx @@ -0,0 +1,5 @@ + + + + + diff --git a/native-app/build.ps1 b/native-app/build.ps1 new file mode 100644 index 0000000..9e63889 --- /dev/null +++ b/native-app/build.ps1 @@ -0,0 +1,50 @@ +#Requires -Version 7.0 +param( + [switch]$SkipBuild +) + +$ErrorActionPreference = "Stop" +$Root = $PSScriptRoot +$Solution = Join-Path $Root "VoxType.Native.slnx" +$AppProject = Join-Path $Root "src/VoxType.NativeApp/VoxType.NativeApp.csproj" +$PublishDir = Join-Path $Root "publish/win-x64" +$ZipPath = Join-Path $Root "publish/VoxType.NativePreview-win-x64.zip" + +$ResolvedRoot = [System.IO.Path]::GetFullPath($Root).TrimEnd([System.IO.Path]::DirectorySeparatorChar) +$ResolvedPublishDir = [System.IO.Path]::GetFullPath($PublishDir) +if (-not $ResolvedPublishDir.StartsWith($ResolvedRoot + [System.IO.Path]::DirectorySeparatorChar, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Publish directory escaped native-app root: $ResolvedPublishDir" +} +if (Test-Path -LiteralPath $ResolvedPublishDir) { + $PublishItem = Get-Item -LiteralPath $ResolvedPublishDir -Force + if ($PublishItem.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw "Refusing to clean reparse-point publish directory: $ResolvedPublishDir" + } + Remove-Item -LiteralPath $ResolvedPublishDir -Recurse -Force +} + +if (-not $SkipBuild) { + dotnet build $Solution -c Release + if ($LASTEXITCODE -ne 0) { throw "Native app build failed." } +} + +dotnet publish $AppProject ` + -c Release ` + -r win-x64 ` + --self-contained false ` + --no-restore ` + -p:DebugType=None ` + -p:DebugSymbols=false ` + -o $ResolvedPublishDir +if ($LASTEXITCODE -ne 0) { throw "Native app publish failed." } + +if (Test-Path -LiteralPath $ZipPath) { + Remove-Item -LiteralPath $ZipPath -Force +} +Compress-Archive -Path (Join-Path $PublishDir "*") -DestinationPath $ZipPath -CompressionLevel Optimal + +$PublishBytes = (Get-ChildItem -LiteralPath $PublishDir -Recurse -File | Measure-Object Length -Sum).Sum +$ZipBytes = (Get-Item -LiteralPath $ZipPath).Length +Write-Host "Native preview ready:" -ForegroundColor Green +Write-Host " Directory: $PublishDir ($([math]::Round($PublishBytes / 1MB, 1)) MB)" +Write-Host " Zip: $ZipPath ($([math]::Round($ZipBytes / 1MB, 1)) MB)" diff --git a/native-app/src/VoxType.NativeApp/App.xaml b/native-app/src/VoxType.NativeApp/App.xaml new file mode 100644 index 0000000..88fbe7f --- /dev/null +++ b/native-app/src/VoxType.NativeApp/App.xaml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + diff --git a/native-app/src/VoxType.NativeApp/App.xaml.cs b/native-app/src/VoxType.NativeApp/App.xaml.cs new file mode 100644 index 0000000..e10c9d8 --- /dev/null +++ b/native-app/src/VoxType.NativeApp/App.xaml.cs @@ -0,0 +1,7 @@ +using System.Windows; + +namespace VoxType.NativeApp; + +public partial class App : Application +{ +} diff --git a/native-app/src/VoxType.NativeApp/MainWindow.xaml b/native-app/src/VoxType.NativeApp/MainWindow.xaml new file mode 100644 index 0000000..d4d7142 --- /dev/null +++ b/native-app/src/VoxType.NativeApp/MainWindow.xaml @@ -0,0 +1,172 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +