diff --git a/.github/workflows/native-preview.yml b/.github/workflows/native-preview.yml new file mode 100644 index 0000000..1ee64f5 --- /dev/null +++ b/.github/workflows/native-preview.yml @@ -0,0 +1,46 @@ +name: Native Preview + +on: + workflow_dispatch: + pull_request: + paths: + - 'native-app/**' + - 'catalog/models.json' + - '.github/workflows/native-preview.yml' + push: + branches: [main] + paths: + - 'native-app/**' + - 'catalog/models.json' + - '.github/workflows/native-preview.yml' + +permissions: + contents: read + +jobs: + build: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET 10 + uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.0.x' + + - name: Build + shell: pwsh + run: dotnet build ./native-app/VoxType.Native.slnx -c Release + + - name: Package framework-dependent preview + shell: pwsh + run: pwsh -NoProfile -File ./native-app/build.ps1 -SkipBuild + + - name: Upload preview zip + uses: actions/upload-artifact@v7 + with: + name: VoxType-NativePreview-win-x64 + path: native-app/publish/VoxType.NativePreview-win-x64.zip + if-no-files-found: error + retention-days: 14 diff --git a/.gitignore b/.gitignore index b943aa1..dc24b8c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,11 @@ app/src-tauri/bundle-resources/runtime/voxtype-runtime/* !app/src-tauri/bundle-resources/runtime/voxtype-runtime/.gitkeep app/src-tauri/bundle-resources/catalog/models.json +# Native .NET preview +native-app/**/bin/ +native-app/**/obj/ +native-app/publish/ + # Plugin plugin/bin/ plugin/obj/ diff --git a/AGENTS.md b/AGENTS.md index 49f3bf0..d5066db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ VoxType 是 Windows 本地语音输入应用,也是 Quicker 语音输入扩展 | 目录 | 职责 | |------|------| | `app/` | Tauri 桌面宿主:热键、录音、浮层、设置、文本提交 | +| `native-app/` | .NET 10/WPF 原生体验版与可复用识别 core;不依赖 Python runtime | | `runtime/` | 独立 ASR sidecar:模型加载、识别、WebSocket 会话 | | `plugin/` | Quicker 兼容适配器;不承载识别引擎 | | `catalog/` | 随安装包发布的模型元数据;模型权重不入库 | @@ -44,6 +45,11 @@ pnpm install pnpm build cargo test --manifest-path src-tauri/Cargo.toml +# Native .NET preview +cd ../../native-app +dotnet build VoxType.Native.slnx -c Release +pwsh ./build.ps1 -SkipBuild + # 模型目录 cd .. pwsh ./scripts/validate-models-catalog.ps1 diff --git a/README.md b/README.md index 7de0e59..6bcffcb 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ ASR runtime uses [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) (SenseVoic | `app/` | Tauri client — settings, overlay, hotkeys, direct typing, Quicker HTTP API | | `plugin/` | Quicker plugin — button triggers dictation | | `catalog/models.json` | Public model catalog for settings UI | +| `native-app/` | .NET 10/WPF native preview — no Python sidecar | ## Quick start (dev) @@ -35,6 +36,8 @@ pnpm tauri dev Data directory: `%LOCALAPPDATA%\VoxType\` (models, settings). +For the smaller native preview intended to validate future `QuickerPc/Quicker` integration, see [`native-app/`](native-app/README.md). + ## Model download Model **weights** are not bundled (too large). Model **download URLs** are written in `catalog/models.json` and ship inside the installer at `catalog/models.json` — the app reads this file locally (no remote catalog fetch). Users download weights from the settings UI on first use. diff --git a/docs/quicker-integration-research.md b/docs/quicker-integration-research.md index 5b37db1..a3ea75c 100644 --- a/docs/quicker-integration-research.md +++ b/docs/quicker-integration-research.md @@ -17,6 +17,7 @@ Windows Text Services Framework(TSF)适合长期追求输入法级体验, - sherpa-onnx + SenseVoice 已能离线工作,runtime 目录也预留了其它识别后端。 - 当前桌面端直接模拟 Unicode 键入;遇到管理员窗口、特殊编辑器、终端或复杂输入法状态时兼容性有限。 - 当前 Quicker 插件经 `127.0.0.1:6020` 调用 VoxType,只是外部控制桥,不是 Quicker 原生 provider。 +- `native-app/` 已提供 `.NET 10 + WPF + sherpa-onnx C#` 的无 Python 体验版,并用约 25 MB 的中文在线 Zipformer 验证 40 ms 音频分块、partial/final 与端点重置;它不改变“Quicker 主进程不直接承载模型”的隔离边界。 ### QuickerPc/Quicker @@ -125,6 +126,7 @@ v2 在会话前增加能力握手,至少返回: - 抽出 JSON schema、示例音频和协议一致性测试。 - runtime 增加 capabilities/版本信息和控制 API token。 - 为 .NET 提供薄客户端;不把模型推理依赖带进客户端包。 +- 以 `native-app/VoxType.NativeCore` 为基础增加无 UI worker,完成稳定性与内存驻留测试后,再决定是否替换 Python runtime。 ### P2:Quicker 原生 provider diff --git a/native-app/BENCHMARKS.md b/native-app/BENCHMARKS.md new file mode 100644 index 0000000..62910ce --- /dev/null +++ b/native-app/BENCHMARKS.md @@ -0,0 +1,46 @@ +# Native Preview 基线 + +> 日期:2026-08-07。数据用于比较架构,不代表所有设备上的保证值。 + +## 环境 + +- CPU:Intel Core i9-14900HX(24 核 / 32 线程) +- 系统:Windows x64 +- SDK / Runtime:.NET 10.0.301 / .NET Desktop Runtime 10 +- 推理:sherpa-onnx 1.13.4,在线 CTC,CPU provider,最多 4 线程 +- 模型:Zipformer 中文实时 Small INT8,26,342,340-byte ONNX +- 音频:16 kHz mono,5.00 秒中文 WAV + +## 结果 + +| 指标 | 结果 | +|------|------| +| 模型加载 | 1.19–2.55 s(冷热启动) | +| 加速分块识别 | 0.14–0.20 s,RTF 0.027–0.040 | +| 实时分块播放 | 9 次文字更新,首个 partial 约 0.57 s | +| 发布目录 | 21.4 MB | +| zip | 8.1 MB | +| 模型加载后的进程 Working Set | 约 281 MB | +| 模型加载后的 Private Memory | 约 232 MB | + +`v0.1.12` 的 Python/PyInstaller + Tauri NSIS 安装包为 25,001,555 bytes(约 23.8 MiB)。Native Preview 的 zip 为约 8.1 MiB,压缩体积降低约 66%;两者都不包含模型权重。流式模型文件合计约 25.1 MiB,相比原 SenseVoice 文件的 228.45 MiB 降低约 89%。 + +## 解释 + +- `--realtime` 按 40 ms 节奏送入 chunk;首个 partial 是用户感知延迟,比停止录音后的总 decode 时间更有代表性。 +- 初版没有为流式模型提供左侧上下文,首词在同一音频上误识别为“算好”;补充官方文件解码示例采用的 300 ms 静音后变为“上好”,快速与真实节奏的最终文本一致。 +- WPF UI、录音回调和在线 decode 分属不同线程;无界音频队列不主动丢 chunk,结束时先停止 NAudio 再刷新最终上下文。 +- 小模型明显降低磁盘和常驻内存,但本轮样例首词识别为“算好”而非“大家好”,也不自带标点。上线前必须用实际口音、噪声和 Quicker 术语集比较准确率。 +- 发布目录仍主要由 `onnxruntime.dll` 与 `sherpa-onnx-c-api.dll` 构成。后续 Quicker 接入仍建议使用独立原生 worker,隔离 native DLL 生命周期。 + +## 复现 + +```powershell +pwsh ./build.ps1 + +dotnet run --project ./src/VoxType.NativeSmoke -c Release -- ` + ` + "$env:LOCALAPPDATA\VoxType\models\zipformer-zh-streaming-small" ` + 1 ` + --realtime +``` diff --git a/native-app/Directory.Build.props b/native-app/Directory.Build.props new file mode 100644 index 0000000..b377359 --- /dev/null +++ b/native-app/Directory.Build.props @@ -0,0 +1,10 @@ + + + + latest + true + $(DefaultItemExcludes);publish/**;**/publish/** + true + true + + diff --git a/native-app/Directory.Build.targets b/native-app/Directory.Build.targets new file mode 100644 index 0000000..a25c1a7 --- /dev/null +++ b/native-app/Directory.Build.targets @@ -0,0 +1,3 @@ + + + diff --git a/native-app/Directory.Packages.props b/native-app/Directory.Packages.props new file mode 100644 index 0000000..d4515eb --- /dev/null +++ b/native-app/Directory.Packages.props @@ -0,0 +1,10 @@ + + + true + + + + + + + diff --git a/native-app/README.md b/native-app/README.md new file mode 100644 index 0000000..372c767 --- /dev/null +++ b/native-app/README.md @@ -0,0 +1,51 @@ +# VoxType Native Preview + +面向 `QuickerPc/Quicker` 的最小原生实时语音识别体验版。它使用 `.NET 10 + WPF + NAudio + sherpa-onnx C#`,直接加载中文流式 Zipformer CTC,不启动 Python、aiohttp 或 PyInstaller runtime。 + +## 体验 + +```powershell +cd native-app +dotnet run --project ./src/VoxType.NativeApp -c Release +``` + +首次启动点击“下载模型”。模型约 25 MB,保存到 `%LOCALAPPDATA%\VoxType\models\zipformer-zh-streaming-small`,不进入应用安装包。模型就绪后: + +1. 点击“开始实时识别”。 +2. 自然说话,结果框会持续显示临时结果;短暂停顿后自动确认一段。 +3. 点击“停止识别”提交最后一段,然后复制结果。 + +麦克风每 40 ms 产生一个 PCM chunk,由后台在线识别会话持续消费,WPF UI 只接收 partial/final 更新,不在录音回调或 UI 线程执行模型推理。当前预览版不注册全局热键、不自动向其它窗口输入文字。 + +## 构建小体积包 + +```powershell +pwsh ./build.ps1 +``` + +产物: + +- `publish/win-x64/`:framework-dependent Windows x64 应用。 +- `publish/VoxType.NativePreview-win-x64.zip`:可直接解压体验的压缩包。 + +实测 zip 为 8.1 MB。5 秒中文音频按 40 ms 实时分块播放时产生 8 次文字更新,首个 partial 约 0.91 秒出现;环境、内存与复现命令见 [`BENCHMARKS.md`](BENCHMARKS.md)。 + +每次 `main` 上的 native-app 改动也会生成一个保留 14 天的 GitHub Actions artifact:打开仓库的 **Actions → Native Preview**,下载 `VoxType-NativePreview-win-x64`。 + +应用要求已安装 .NET 10 Desktop Runtime。Quicker V2 已使用同一运行时;若未来面向没有 Quicker 的独立用户发布,可另做 self-contained 包,但体积会显著增加。 + +## 命令行识别验证 + +```powershell +dotnet run --project ./src/VoxType.NativeSmoke -c Release -- [model-directory] [repeat-count] [--realtime] +``` + +第二个可选参数是模型目录,第三个参数可在同一进程重复识别。第四个参数 `--realtime` 按真实 40 ms 节奏送入音频,用于检查 partial 次数和首字延迟。该工具用于 CI/开发验证,不进入预览应用压缩包。 + +## 当前边界 + +- 当前小模型只支持中文,没有 SenseVoice 的多语种、情绪识别和自动标点能力;换取约 89% 的权重体积下降与更低内存。 +- 只启用 Windows x64 CPU provider,优先保证部署一致性;识别器在进程内只加载一次,每次录音创建独立在线 stream。 +- 模型使用组件内独立 catalog,按文件大小与 SHA-256 校验,并支持 `.part` 断点续传。 +- 会话为首字补充 300 ms 左侧上下文;端点检测在连续静音后确认当前分段,并对重置边界的长文本重叠去重。点击停止时再送入 600 ms 右侧上下文刷新最终结果。 +- 模型来自 sherpa-onnx 的 [`streaming-zipformer-small-ctc-zh-int8-2025-04-01`](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-ctc/zipformer-ctc-models.html),镜像标记为 Apache-2.0;正式集成前仍需保留模型 attribution。 diff --git a/native-app/VoxType.Native.slnx b/native-app/VoxType.Native.slnx new file mode 100644 index 0000000..91669aa --- /dev/null +++ b/native-app/VoxType.Native.slnx @@ -0,0 +1,5 @@ + + + + + diff --git a/native-app/build.ps1 b/native-app/build.ps1 new file mode 100644 index 0000000..9e63889 --- /dev/null +++ b/native-app/build.ps1 @@ -0,0 +1,50 @@ +#Requires -Version 7.0 +param( + [switch]$SkipBuild +) + +$ErrorActionPreference = "Stop" +$Root = $PSScriptRoot +$Solution = Join-Path $Root "VoxType.Native.slnx" +$AppProject = Join-Path $Root "src/VoxType.NativeApp/VoxType.NativeApp.csproj" +$PublishDir = Join-Path $Root "publish/win-x64" +$ZipPath = Join-Path $Root "publish/VoxType.NativePreview-win-x64.zip" + +$ResolvedRoot = [System.IO.Path]::GetFullPath($Root).TrimEnd([System.IO.Path]::DirectorySeparatorChar) +$ResolvedPublishDir = [System.IO.Path]::GetFullPath($PublishDir) +if (-not $ResolvedPublishDir.StartsWith($ResolvedRoot + [System.IO.Path]::DirectorySeparatorChar, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Publish directory escaped native-app root: $ResolvedPublishDir" +} +if (Test-Path -LiteralPath $ResolvedPublishDir) { + $PublishItem = Get-Item -LiteralPath $ResolvedPublishDir -Force + if ($PublishItem.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + throw "Refusing to clean reparse-point publish directory: $ResolvedPublishDir" + } + Remove-Item -LiteralPath $ResolvedPublishDir -Recurse -Force +} + +if (-not $SkipBuild) { + dotnet build $Solution -c Release + if ($LASTEXITCODE -ne 0) { throw "Native app build failed." } +} + +dotnet publish $AppProject ` + -c Release ` + -r win-x64 ` + --self-contained false ` + --no-restore ` + -p:DebugType=None ` + -p:DebugSymbols=false ` + -o $ResolvedPublishDir +if ($LASTEXITCODE -ne 0) { throw "Native app publish failed." } + +if (Test-Path -LiteralPath $ZipPath) { + Remove-Item -LiteralPath $ZipPath -Force +} +Compress-Archive -Path (Join-Path $PublishDir "*") -DestinationPath $ZipPath -CompressionLevel Optimal + +$PublishBytes = (Get-ChildItem -LiteralPath $PublishDir -Recurse -File | Measure-Object Length -Sum).Sum +$ZipBytes = (Get-Item -LiteralPath $ZipPath).Length +Write-Host "Native preview ready:" -ForegroundColor Green +Write-Host " Directory: $PublishDir ($([math]::Round($PublishBytes / 1MB, 1)) MB)" +Write-Host " Zip: $ZipPath ($([math]::Round($ZipBytes / 1MB, 1)) MB)" diff --git a/native-app/catalog/models.json b/native-app/catalog/models.json new file mode 100644 index 0000000..d507b2f --- /dev/null +++ b/native-app/catalog/models.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "notes": "Native Preview streaming model catalog. Model weights are downloaded on first use and are not bundled with the app.", + "models": [ + { + "id": "zipformer-streaming-small-zh-int8", + "name": "Zipformer 中文实时 Small", + "description": "推荐 · 约 25 MB · 中文流式识别", + "type": "streaming-zipformer-ctc", + "runtimePreset": "streaming-zipformer-ctc", + "layout": "zipformer-zh-streaming-small", + "supported": true, + "default": true, + "download": { + "source": "modelscope", + "modelscopeResolveBase": "https://www.modelscope.cn/models/manyeyes/k2transducer-zipformer-ctc-small-zh-onnx-online-20250401/resolve/master", + "modelscopeFiles": [ + { + "name": "model.int8.onnx", + "size": 26342340, + "required": true, + "sha256": "68c9c943840f7d9cf3e8a4970ba50f404feb5277f611fa82b7e72267786fa84a" + }, + { + "name": "tokens.txt", + "size": 13366, + "required": true, + "sha256": "6fed8c6c248516f38e7faa19404b57413e8ce259f1cbc1fa4aebc86eac32fdfd" + } + ] + } + } + ] +} diff --git a/native-app/src/VoxType.NativeApp/App.xaml b/native-app/src/VoxType.NativeApp/App.xaml new file mode 100644 index 0000000..88fbe7f --- /dev/null +++ b/native-app/src/VoxType.NativeApp/App.xaml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + diff --git a/native-app/src/VoxType.NativeApp/App.xaml.cs b/native-app/src/VoxType.NativeApp/App.xaml.cs new file mode 100644 index 0000000..e10c9d8 --- /dev/null +++ b/native-app/src/VoxType.NativeApp/App.xaml.cs @@ -0,0 +1,7 @@ +using System.Windows; + +namespace VoxType.NativeApp; + +public partial class App : Application +{ +} diff --git a/native-app/src/VoxType.NativeApp/MainWindow.xaml b/native-app/src/VoxType.NativeApp/MainWindow.xaml new file mode 100644 index 0000000..984a325 --- /dev/null +++ b/native-app/src/VoxType.NativeApp/MainWindow.xaml @@ -0,0 +1,175 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +