From 9c2a88b60f5fbdcf954e2f112ab654f4f4ae0312 Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Thu, 20 Aug 2026 13:51:10 +0800 Subject: [PATCH 1/2] Add Phase B ESP32 OCR handoff skeleton --- docs/phase_ab_esp32_ocr_handoff_zh.md | 279 ++++++++++++++++++ runtime/openglass_omni/README.md | 4 + runtime/openglass_omni/esp32_bridge.py | 74 ++++- runtime/openglass_omni/perception/README.md | 19 ++ runtime/openglass_omni/perception/__init__.py | 16 + runtime/openglass_omni/perception/base.py | 41 +++ runtime/openglass_omni/perception/noop.py | 56 ++++ .../openglass_omni/perception/ocr_template.py | 27 ++ runtime/openglass_omni/perception/pipeline.py | 122 ++++++++ runtime/openglass_omni/perception/registry.py | 70 +++++ .../perception/shadow_runtime.py | 74 +++++ .../openglass_omni/perception/yolo_onnx.py | 201 +++++++++++++ runtime/openglass_omni/requirements-cv.txt | 4 + runtime/openglass_omni/tests/__init__.py | 1 + .../openglass_omni/tests/test_perception.py | 95 ++++++ 15 files changed, 1082 insertions(+), 1 deletion(-) create mode 100644 docs/phase_ab_esp32_ocr_handoff_zh.md create mode 100644 runtime/openglass_omni/perception/README.md create mode 100644 runtime/openglass_omni/perception/__init__.py create mode 100644 runtime/openglass_omni/perception/base.py create mode 100644 runtime/openglass_omni/perception/noop.py create mode 100644 runtime/openglass_omni/perception/ocr_template.py create mode 100644 runtime/openglass_omni/perception/pipeline.py create mode 100644 runtime/openglass_omni/perception/registry.py create mode 100644 runtime/openglass_omni/perception/shadow_runtime.py create mode 100644 runtime/openglass_omni/perception/yolo_onnx.py create mode 100644 runtime/openglass_omni/requirements-cv.txt create mode 100644 runtime/openglass_omni/tests/__init__.py create mode 100644 runtime/openglass_omni/tests/test_perception.py diff --git a/docs/phase_ab_esp32_ocr_handoff_zh.md b/docs/phase_ab_esp32_ocr_handoff_zh.md new file mode 100644 index 0000000..ecf0254 --- /dev/null +++ b/docs/phase_ab_esp32_ocr_handoff_zh.md @@ -0,0 +1,279 @@ +# Phase A / Phase B 与 ESP32 OCR 开发交接（V1.1） + +更新时间：2026-08-20 +工作分支：`codex/phase-b-v1.1-esp32-ocr-handoff` + +本文面向接手 OCR、文字检测和轻量 CV 插件的开发者。它记录当前已经验证的能力、Windows Anaconda Prompt 启动顺序，以及哪些文件可以修改。本文中的命令都在 **Anaconda Prompt（cmd）** 中逐条执行，不使用 PowerShell 的反引号续行。 + +> 安全边界：实验性避障不是导航或人身安全功能。CV V1.1 只做 shadow 观测和日志，不直接控制会话、播报或运动决策。 + +## 1. 两个阶段和代码所有权 + +Phase A 和 Phase B 共用同一个控制核心，只替换设备适配层： + +```text +Phase A: Chrome JPEG/PCM -> Browser Adapter --+ + +-> Harness Core / FunASR / Router / StateMachine +Phase B: Rokid 或 ESP32 -> Device Adapter ----+ | + v +llama.cpp-omni -> Worker -> Gateway :8040 -> MiniCPM-o 4.5 -> PC 扬声器 +``` + +- `MiniCPM-o-Demo-Comni/extensions/assistive_harness/` 是唯一的命令 Router、状态机、Skill YAML、prompt 和控制协议实现。 +- OpenGlass 只负责 ESP32/Rokid 的 JPEG、PCM、播放、录制以及可选 CV provider。不要在设备端再复制一套关键词或状态机。 +- RESET 和 Skill 切换只替换 Gateway Session，不能重载 Worker 或模型后端。 +- CV provider 只返回统一的 `CVObservation`。V1.1 的 ESP32 接入是旁路日志，不能阻塞主音频链路。 + +## 2. 当前冻结能力 + +Phase A 浏览器链路已经人工验证： + +- `停一下`：立即停止当前播放； +- `恢复对话`：放开播放门，继续当前 Session； +- `重新开始`：以普通聊天 prompt 热重启 Session； +- `回到普通聊天`：从任意 Skill 切回普通聊天，并更换 Session； +- `帮我找一下手机`、`帮我读一下这个杯子上的字`、`帮我描述一下眼前的场景`、`前面有没有障碍`：分别切换找物、识字、场景描述和实验性避障 Skill。 + +Phase B Rokid + PC 扬声器已经验证真实 JPEG/PCM、STOP、RESUME、RESET 和四种 Skill 切换。V1.1 新增： + +- Session 就绪双音提示默认音量从 `0.16` 提到 `0.32`，可用参数调节； +- EchoGuard 把 PC 播放队列和尾音时间纳入“模型正在说话”，减少扬声器回灌误触发； +- Skill 新 Session 建好后自动注入一次原始任务，例如读字或判断前方障碍，避免只切 prompt 不主动回答； +- 终端输出 `[AssistiveHarness][MODEL]`，同时写入模型文字日志； +- 明确支持口令“帮我描述一下”。 + +仍然保留的边界：一次 Skill 激活目前是一次任务，不是连续 CV 跟踪；PC 扬声器回声只做软件门控，并非声学回声消除；模型生成文本与实际 TTS 发音可能不同。 + +## 3. 仓库与环境 + +协作基线： + +```text +仓库：https://github.com/OpenSQZ/OpenGlass +分支：codex/phase-b-v1.1-esp32-ocr-handoff +``` + +同事第一次获取代码： + +```bat +git clone --branch codex/phase-b-v1.1-esp32-ocr-handoff --single-branch https://github.com/OpenSQZ/OpenGlass.git OpenGlass +cd /d OpenGlass +git status --short --branch +``` + +已有 OpenGlass 工作目录则执行： + +```bat +git fetch origin +git switch codex/phase-b-v1.1-esp32-ocr-handoff +git pull --ff-only +``` + +不要直接在协作基线上长期堆叠两个人的实验代码。OCR 同事从基线创建自己的功能分支： + +```bat +git switch codex/phase-b-v1.1-esp32-ocr-handoff +git pull --ff-only +git switch -c feature/esp32-ocr- +``` + +项目负责人继续 Phase B/debug 时也从同一基线创建独立分支，例如 +`feature/phase-b-rokid-debug`。各自通过 PR 合并回协作基线；确认设备回归后， +再向 OpenGlass `main` 提交上游 PR。 + +需要两个并列仓库： + +```text +C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +C:\Users\Lenovo\AI_WAIC\github\OpenGlass +``` + +OpenGlass 分支检查： + +```bat +cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +git switch codex/phase-b-v1.1-esp32-ocr-handoff +git status --short --branch +``` + +当前本机的 `MiniCPM-o-Demo-Comni` 目录不是 Git 工作树。因此本 OpenGlass +分支只包含设备适配、CV 插件骨架和本文，**不会自动包含 Harness Core 的 +V1.1 修改**。交付给同事时应同时提供经过审阅的 MiniCPM 工作目录归档，或 +后续把它接入一个明确的远端仓库；不要把 Harness Core 复制进 OpenGlass。 + +本次文件边界如下： + +```text +MiniCPM-o-Demo-Comni/ + extensions/assistive_harness/model_log.py # 新增模型回合聚合 + extensions/assistive_harness/server.py # MODEL/playback telemetry + extensions/assistive_harness/telemetry.py # 模型日志文件 + extensions/assistive_harness/registry.py # task_trigger 渲染 + extensions/assistive_harness/config/skills.example.yaml # 四类一次性任务/口令 + extensions/assistive_harness/phase_b/rokid_runtime.py # V1.1 播放与触发 + extensions/assistive_harness/phase_b/README.md # Rokid 使用说明 + extensions/assistive_harness/tests/ # 回归测试 + +OpenGlass/ + runtime/openglass_omni/esp32_bridge.py # 非阻塞 CV 挂点/CLI + runtime/openglass_omni/perception/ # provider/pipeline/YOLO/OCR + runtime/openglass_omni/requirements-cv.txt # 可选 CV 依赖 + runtime/openglass_omni/tests/test_perception.py # 插件隔离/背压测试 + docs/phase_ab_esp32_ocr_handoff_zh.md # 本文 +``` + +ESP32 已烧录连接 `CUDY-D102` 的固件。电脑也要连接同一网络，并从串口日志或路由器页面取得 ESP32 的当前 IPv4 地址。运行参数只使用 IP；不要把 SSID 密码、私人 IP 或模型权重提交到 Git。 + +安装基础与可选 CV 依赖： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +python -m pip install -r runtime\openglass_omni\requirements.txt +python -m pip install -r runtime\openglass_omni\requirements-cv.txt +``` + +OCR 后端若需要 PaddleOCR、ONNX Runtime、TensorRT 等额外依赖，请单独增加 `requirements-ocr-.txt`，不要把重型 OCR 依赖塞入基础 requirements。 + +## 4. 启动 MiniCPM-o、Harness 和 Phase B + +### 4.1 后端：llama.cpp-omni -> Worker -> Gateway :8040 + +打开第一个 Anaconda Prompt： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +start_all.cmd --http +``` + +等待后端、Worker 和 Gateway 就绪。Phase B 本机链路使用 `ws://localhost:8040`；不要在未启用 TLS 时改成 `wss://`。 + +### 4.2 Harness Core :8021 + +打开第二个 Anaconda Prompt： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +python -m extensions.assistive_harness.server --enabled --model-path "%USERPROFILE%\.cache\modelscope\hub\models\iic\speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online" --port 8021 +``` + +成功后会看到 FunASR ready 和 Uvicorn `127.0.0.1:8021`。控制与模型日志位于： + +```text +extensions/assistive_harness/runs//control_events.jsonl +extensions/assistive_harness/runs//session_events.jsonl +extensions/assistive_harness/runs//model_events.jsonl +extensions/assistive_harness/runs//model_transcript.txt +``` + +`model_transcript.txt` 是模型生成文字，不是对 TTS 波形重新做的识别。核对实际播报仍应保留录音或 Session WAV。 + +### 4.3 Chrome Phase A 回归 + +前两个 Prompt 就绪后，用 Chrome 打开带显式开关的页面： + +```text +http://127.0.0.1:8040/omni?assistive_harness=1&v=phase-b-v1-1 +``` + +Harness 默认关闭，不能省略 `assistive_harness=1`。只保留一个 Live 页面，依次复测 +STOP、RESUME、RESET、四种 Skill 和 Skill 互切。原生回归则另开 +`http://127.0.0.1:8040/omni`，确认不带参数时仍是原始 Demo 行为。 + +### 4.4 Rokid Phase B（项目负责人继续测试） + +打开第三个 Anaconda Prompt： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +python demo_rokid_phase_b_harness.py --gateway localhost:8040 --harness-url ws://127.0.0.1:8021/ws/control --input-gain 12 --image-rotate-cw 270 --session-ready-chime-volume 0.32 --playback-echo-tail-s 0.80 +``` + +第一轮仍使用电脑扬声器。音量参数范围为 `0.0` 到 `1.0`；先用 `0.32`，不要为盖过回声无上限增大。`--playback-echo-tail-s` 是播放队列排空后的额外抑制时间，不是声学 AEC。 + +### 4.5 ESP32 主链路 smoke test（OCR 同事） + +先验证 ESP32 输入与 MiniCPM 主链路；把 `` 替换为设备当前地址： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +python runtime\openglass_omni\esp32_bridge.py --esp32-host --esp32-port 80 --gateway localhost:8040 --no-tls --rotate 180 --enable-funasr --funasr-model "%USERPROFILE%\.cache\modelscope\hub\models\iic\SenseVoiceSmall" --funasr-echo-suppress-s 4.0 --prompt "你是智能眼镜助手。用户问什么就简短回答什么；只有用户要求描述场景或寻找物体时才看图回答，不要主动描述。" +``` + +若现有固件、Gateway revision 或端口与本机验证环境不同，以当前健康检查和设备日志为准，但不要改写 Router/状态机。先确认音频连续、JPEG 在更新、Gateway 正常收到数据，再叠加 OCR。 + +## 5. YOLO 参考插件 smoke test + +YOLO 只是验证 provider 接口、异常隔离和非阻塞队列，不是最终找物策略。模型路径使用本机实际位置： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +python runtime\openglass_omni\esp32_bridge.py --esp32-host --esp32-port 80 --gateway localhost:8040 --no-tls --rotate 180 --cv-shadow-provider yolo_onnx --cv-shadow-options-json "{\"model_path\":\"C:\\Users\\Lenovo\\AI_Glasses_0618\\OmniHarness\\mini_omni_harness\\models\\yolo26n.onnx\",\"device\":\"cpu\"}" --cv-shadow-skill find_object --cv-shadow-slots-json "{\"target\":\"手机\"}" --cv-shadow-timeout-ms 2000 --cv-shadow-log logs\cv_yolo.jsonl +``` + +预期：终端持续出现 `[CV]`，`logs\cv_yolo.jsonl` 每行是一条 `CVObservation`；即使模型加载失败或超时，ESP32 PCM 与 MiniCPM 音频仍应继续。队列容量固定为 1：推理忙时丢弃旧帧，只保留最新等待帧。 + +## 6. OCR provider 的唯一改动路径 + +目录：`runtime/openglass_omni/perception/` + +1. 复制 `ocr_template.py` 为 `ocr_.py`。 +2. 实现一个同步的 `OcrProvider.analyze()`。模型可延迟加载，但不能创建第二套音频/会话逻辑。 +3. 返回 `CVObservation`。建议 `values` 至少包含： + +```python +{ + "status": "ok", + "text": "识别到的完整文本", + "regions": [ + {"text": "局部文字", "confidence": 0.96, "bbox_xyxy": [x1, y1, x2, y2]} + ], + "latency_ms": 83.4, + "model": "模型名称或版本", + "device": "cpu" +} +``` + +4. 用动态路径启动，不必修改 registry： + +```bat +python runtime\openglass_omni\esp32_bridge.py --esp32-host --gateway localhost:8040 --no-tls --rotate 180 --cv-shadow-provider perception.ocr_:OcrProvider --cv-shadow-options-json "{\"model_path\":\"D:\\models\\ocr\"}" --cv-shadow-skill read_text --cv-shadow-log logs\cv_ocr.jsonl +``` + +5. 模型依赖、权重、缓存和样例图片不要提交；只提交 provider、最小测试、依赖清单和不含隐私的示例结果。 + +OCR V1 完成标准：真实 ESP32 JPEG 能产出统一结果；异常和超时只形成 observation；音频主链不中断；连续输入发生背压时 `dropped_frames` 增长而内存不增长；至少有一张含中英文的固定测试图和预期文本。 + +## 7. 测试 + +OpenGlass CV 骨架： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +python -m unittest discover runtime\openglass_omni\tests -v +``` + +Harness 核心与 Rokid Phase B： + +```bat +conda activate ai_glasses +cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +python -m unittest extensions.assistive_harness.tests.test_core extensions.assistive_harness.tests.test_phase_b_rokid -v +``` + +设备验收顺序：普通问答 -> `停一下` -> `恢复对话` -> `重新开始` -> 找物 -> 识字 -> 场景描述 -> 实验性避障 -> Skill 直接互切 -> 断网重连。每次 RESET/Skill 切换应看到 generation 增加、新 Session ID、`restart_complete`，Skill 还应看到一次 `task_trigger_sent`。 + +## 8. 协作与后续集成 + +- OCR 同事只在本 OpenGlass 分支新增 provider、测试和依赖文件；不要修改 Phase A 的关键词和状态机。 +- 项目负责人继续在 Rokid 真实设备上校准回声、提示音和 Session 稳定性。 +- OCR observation 反馈给 Skill、连续读字和主动播报属于下一阶段。先冻结 shadow 输出格式，再设计结果消费者，避免模型实现与控制策略耦合。 +- 当前 timeout 会及时产生日志并让音频继续，但 Python 线程不能强制终止已经进入原生库的推理调用；需要硬隔离的模型应在后续改成独立进程 provider。 +- `codex/phase-b-v1.1-esp32-ocr-handoff` 是 Phase B/ESP32/OCR 的共同起点； + 新实验通过短期功能分支和 PR 回合，不直接重写基线历史。 diff --git a/runtime/openglass_omni/README.md b/runtime/openglass_omni/README.md index afcaf9b..286008b 100644 --- a/runtime/openglass_omni/README.md +++ b/runtime/openglass_omni/README.md @@ -17,11 +17,15 @@ This is an experimental research integration. It is not production-ready, not a | `bridge_ui.py` | Local web server (default `http://localhost:8080`) for the live first-person view embedded in the panel, plus a `/replay` session browser. Without it, the panel's right pane is blank. | | `recorder_live.py` | Records every session to `sessions/` (video, user/AI audio tracks, `events.jsonl` subtitles, `meta.json`). | | `rerun_source.py` | Replays a recorded session back through the model (see [Rerun mode](#rerun-mode-command-line)). Not wired into the panel. | +| `perception/` | Optional non-blocking CV shadow providers. Includes a YOLO ONNX reference and an OCR provider template. | | `devices.json` | Glasses IP / rotation table. The panel's device dropdown follows this file. | | `templates/` | `live.html`, `replay.html`, `replay_index.html` — served by `bridge_ui.py`. | The entry point is the repository-root `glasses_panel.py`, an 8-line shim that calls `runtime.openglass_omni.panel:main`. +For the Phase A/Phase B control contract, ESP32 commands, and the OCR plugin +handoff, see the [Chinese V1.1 guide](../../docs/phase_ab_esp32_ocr_handoff_zh.md). + ## Process chain ```text diff --git a/runtime/openglass_omni/esp32_bridge.py b/runtime/openglass_omni/esp32_bridge.py index 3d50c75..1cecad4 100644 --- a/runtime/openglass_omni/esp32_bridge.py +++ b/runtime/openglass_omni/esp32_bridge.py @@ -112,6 +112,8 @@ local_pcm_reader = None LocalImageSource = None +from perception.shadow_runtime import ShadowPerceptionRuntime + ASR_RUNTIME_DIR = Path(__file__).resolve().parent / "ASR" if str(ASR_RUNTIME_DIR) not in sys.path: sys.path.insert(0, str(ASR_RUNTIME_DIR)) @@ -1090,6 +1092,7 @@ async def audio_first_image_cache_loop( image_state: AudioFirstImageState, stop_evt: asyncio.Event, stats: dict[str, Any], + cv_shadow: Optional[ShadowPerceptionRuntime] = None, ) -> None: if not args.use_image: return @@ -1173,6 +1176,12 @@ async def audio_first_image_cache_loop( if img_jpeg: stats["image_ok"] += 1 LOGGER.info("[IMG] cache updated seq=%d age=%dms", seq, age_ms) + if cv_shadow is not None: + cv_shadow.submit( + img_jpeg, + frame_id=f"esp32_{seq}_{int(time.time() * 1000)}", + timestamp_ms=time.time() * 1000.0, + ) else: stats["image_fail"] += 1 except asyncio.CancelledError: @@ -1929,6 +1938,29 @@ async def run_bridge(args) -> None: noise_level_db=args.echo_noise_db, ) stop_evt = asyncio.Event() + cv_shadow: Optional[ShadowPerceptionRuntime] = None + if args.cv_shadow_provider: + try: + provider_options = json.loads(args.cv_shadow_options_json) + provider_slots = json.loads(args.cv_shadow_slots_json) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid CV shadow JSON: {exc}") from exc + if not isinstance(provider_options, dict) or not isinstance(provider_slots, dict): + raise ValueError("CV shadow options and slots must be JSON objects") + cv_shadow = ShadowPerceptionRuntime( + args.cv_shadow_provider, + options=provider_options, + skill_id=args.cv_shadow_skill, + slots=provider_slots, + log_path=args.cv_shadow_log, + inference_timeout_ms=args.cv_shadow_timeout_ms, + ) + LOGGER.info( + "[CV] shadow enabled provider=%s skill=%s log=%s", + args.cv_shadow_provider, + args.cv_shadow_skill, + args.cv_shadow_log, + ) # v6.6 移植：把 SIGINT/SIGTERM 改成"设旗子让主循环干净退", # 避免 KeyboardInterrupt 在 cleanup 中段抛出导致 finalize_mp4 被跳过。 @@ -2326,7 +2358,9 @@ async def _send_session_init() -> None: # v6.6 移植：live 用图像优化路（abort/cache）；rerun 用直读路，不启动该 loop if rerun_img_source is None: image_cache_task = asyncio.create_task( - audio_first_image_cache_loop(args, image_state, stop_evt, run_stats) + audio_first_image_cache_loop( + args, image_state, stop_evt, run_stats, cv_shadow + ) ) else: image_cache_task = None @@ -2906,6 +2940,13 @@ async def handle_result( except Exception as e: LOGGER.warning("[ASR] stop err: %r", e) + if cv_shadow is not None: + try: + await cv_shadow.close() + LOGGER.info("[CV] shadow stopped stats=%s", cv_shadow.snapshot()) + except Exception as e: + LOGGER.warning("[CV] shadow close err: %r", e) + for _t in (player_monitor, audio_reader_task, recv_task, image_cache_task): if _t is None: continue @@ -3075,6 +3116,37 @@ def build_arg_parser() -> argparse.ArgumentParser: help="取图超时时本轮不带任何图、只发音频(默认关:超时则沿用上一帧旧图)") p.add_argument("--use-image", action="store_true", default=True) p.add_argument("--no-image", dest="use_image", action="store_false") + p.add_argument( + "--cv-shadow-provider", + default="", + help="可选 CV 旁路 provider：内置别名或 package.module:Class；空值为关闭", + ) + p.add_argument( + "--cv-shadow-options-json", + default="{}", + help="传给 provider 构造函数的 JSON 对象", + ) + p.add_argument( + "--cv-shadow-skill", + default="idle_chat", + help="写入 CVObservation 的 skill_id", + ) + p.add_argument( + "--cv-shadow-slots-json", + default="{}", + help="传给 provider analyze() 的 slots JSON 对象", + ) + p.add_argument( + "--cv-shadow-timeout-ms", + type=float, + default=2000.0, + help="单次旁路推理超时，仅记录错误，不阻塞音频主链路", + ) + p.add_argument( + "--cv-shadow-log", + default="logs/cv_events.jsonl", + help="CVObservation JSONL 输出路径", + ) p.add_argument("--audio-wait-timeout-s", type=float, default=3.0, help="log every N seconds while waiting for timestamped audio backfill") p.add_argument("--audio-max-gap-fill-ms", type=int, default=0, diff --git a/runtime/openglass_omni/perception/README.md b/runtime/openglass_omni/perception/README.md new file mode 100644 index 0000000..8acdfb7 --- /dev/null +++ b/runtime/openglass_omni/perception/README.md @@ -0,0 +1,19 @@ +# Lightweight CV shadow providers + +This package is the transport-neutral, optional computer-vision side lane for +the OpenGlass ESP32 bridge. V1 has four deliberately small guarantees: + +1. providers implement one synchronous `analyze()` contract; +2. inference runs in a dedicated executor, never on the audio event loop; +3. a capacity-one queue drops stale frames and keeps the newest waiting frame; +4. provider errors and timeouts become `CVObservation` log records and never + control MiniCPM or navigation behavior. + +Built-in references are `noop`, `yolo_onnx`, and the intentionally incomplete +`ocr_template`. Copy `ocr_template.py` to a backend-specific module and load it +with `--cv-shadow-provider perception.ocr_:OcrProvider`. See the +[Chinese Phase A/B handoff](../../../docs/phase_ab_esp32_ocr_handoff_zh.md) for +the complete command and output contract. + +The V1 lane is **shadow-only**. Consuming observations in a Skill or generating +spoken guidance is a later, separately tested integration step. diff --git a/runtime/openglass_omni/perception/__init__.py b/runtime/openglass_omni/perception/__init__.py new file mode 100644 index 0000000..987c8cb --- /dev/null +++ b/runtime/openglass_omni/perception/__init__.py @@ -0,0 +1,16 @@ +"""Transport-neutral, non-blocking CV shadow plugins for OpenGlass.""" + +from .base import CVObservation, FrameEnvelope, PerceptionProvider +from .pipeline import CVPipeline +from .registry import CVProviderRegistry, load_provider +from .shadow_runtime import ShadowPerceptionRuntime + +__all__ = [ + "CVObservation", + "CVPipeline", + "CVProviderRegistry", + "FrameEnvelope", + "PerceptionProvider", + "ShadowPerceptionRuntime", + "load_provider", +] diff --git a/runtime/openglass_omni/perception/base.py b/runtime/openglass_omni/perception/base.py new file mode 100644 index 0000000..28d5c92 --- /dev/null +++ b/runtime/openglass_omni/perception/base.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Protocol + + +@dataclass(slots=True) +class FrameEnvelope: + frame: bytes | None + frame_id: str + timestamp_ms: float + skill_id: str + slots: dict[str, Any] = field(default_factory=dict) + mode: str = "shadow" + provider_id: str = "noop" + + +@dataclass(slots=True) +class CVObservation: + frame_id: str + timestamp_ms: float + skill_id: str + provider: str + values: dict[str, Any] = field(default_factory=dict) + error: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +class PerceptionProvider(Protocol): + """Synchronous plugin contract; the pipeline always calls it off-loop.""" + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: ... diff --git a/runtime/openglass_omni/perception/noop.py b/runtime/openglass_omni/perception/noop.py new file mode 100644 index 0000000..f2fefc4 --- /dev/null +++ b/runtime/openglass_omni/perception/noop.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from typing import Any + +from .base import CVObservation, PerceptionProvider + + +class NoOpProvider: + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + del frame, slots + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider="noop", + values={"status": "noop"}, + ) + + +class ShadowProvider: + """Turn provider failures into observations instead of control failures.""" + + def __init__(self, provider: PerceptionProvider, provider_id: str): + self.provider = provider + self.provider_id = provider_id + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + try: + result = self.provider.analyze( + frame, frame_id, timestamp_ms, skill_id, slots + ) + result.provider = f"shadow:{result.provider}" + return result + except Exception as exc: + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider=f"shadow:{self.provider_id}", + values={"status": "error"}, + error=f"{type(exc).__name__}: {exc}", + ) diff --git a/runtime/openglass_omni/perception/ocr_template.py b/runtime/openglass_omni/perception/ocr_template.py new file mode 100644 index 0000000..2371922 --- /dev/null +++ b/runtime/openglass_omni/perception/ocr_template.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Any + +from .base import CVObservation + + +class OcrTemplateProvider: + """Copy this provider and replace only the model-specific inference body.""" + + def __init__(self, model_path: str = "") -> None: + self.model_path = model_path + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + del slots + if not frame: + raise ValueError("OCR requires a non-empty JPEG frame") + raise NotImplementedError( + "copy ocr_template.py to ocr_.py and implement analyze()" + ) diff --git a/runtime/openglass_omni/perception/pipeline.py b/runtime/openglass_omni/perception/pipeline.py new file mode 100644 index 0000000..7c60807 --- /dev/null +++ b/runtime/openglass_omni/perception/pipeline.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import asyncio +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Callable + +from .base import CVObservation, FrameEnvelope +from .registry import CVProviderRegistry + + +class CVPipeline: + """Capacity-one latest-frame lane; inference never runs on the audio loop.""" + + def __init__( + self, + providers: CVProviderRegistry, + *, + on_observation: Callable[[CVObservation], None], + inference_timeout_ms: float = 2000.0, + worker_name: str = "openglass-cv", + ) -> None: + self.providers = providers + self.on_observation = on_observation + self.inference_timeout_ms = max(1.0, float(inference_timeout_ms)) + self.queue: asyncio.Queue[FrameEnvelope] = asyncio.Queue(maxsize=1) + self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix=worker_name) + self.worker_task: asyncio.Task[None] | None = None + self.closed = False + self.submitted_frames = 0 + self.processed_frames = 0 + self.dropped_frames = 0 + self.timeout_count = 0 + + def submit(self, envelope: FrameEnvelope) -> bool: + if self.closed or envelope.mode == "disabled": + return False + if self.worker_task is None or self.worker_task.done(): + self.worker_task = asyncio.create_task(self._run()) + if self.queue.full(): + try: + self.queue.get_nowait() + self.queue.task_done() + self.dropped_frames += 1 + except asyncio.QueueEmpty: + pass + self.queue.put_nowait(envelope) + self.submitted_frames += 1 + return True + + def snapshot(self) -> dict[str, int | bool]: + return { + "closed": self.closed, + "queued_frames": self.queue.qsize(), + "submitted_frames": self.submitted_frames, + "processed_frames": self.processed_frames, + "dropped_frames": self.dropped_frames, + "timeout_count": self.timeout_count, + } + + async def close(self) -> None: + if self.closed: + return + self.closed = True + if self.worker_task is not None: + self.worker_task.cancel() + await asyncio.gather(self.worker_task, return_exceptions=True) + self.executor.shutdown(wait=False, cancel_futures=True) + + async def _run(self) -> None: + loop = asyncio.get_running_loop() + while True: + envelope = await self.queue.get() + started = time.perf_counter() + future = loop.run_in_executor(self.executor, self.providers.analyze, envelope) + try: + observation = await asyncio.wait_for( + asyncio.shield(future), self.inference_timeout_ms / 1000.0 + ) + except asyncio.TimeoutError: + self.timeout_count += 1 + observation = CVObservation( + envelope.frame_id, + envelope.timestamp_ms, + envelope.skill_id, + f"{envelope.mode}:{envelope.provider_id}", + {"status": "timeout", "timeout_ms": self.inference_timeout_ms}, + f"TimeoutError: exceeded {self.inference_timeout_ms:.0f} ms", + ) + self._emit(observation) + try: + await future + except Exception: + pass + except asyncio.CancelledError: + raise + except Exception as exc: + self._emit( + CVObservation( + envelope.frame_id, + envelope.timestamp_ms, + envelope.skill_id, + f"{envelope.mode}:{envelope.provider_id}", + {"status": "error"}, + f"{type(exc).__name__}: {exc}", + ) + ) + else: + observation.values.setdefault( + "pipeline_latency_ms", + round((time.perf_counter() - started) * 1000.0, 1), + ) + self.processed_frames += 1 + self._emit(observation) + finally: + self.queue.task_done() + + def _emit(self, observation: CVObservation) -> None: + try: + self.on_observation(observation) + except Exception: + pass diff --git a/runtime/openglass_omni/perception/registry.py b/runtime/openglass_omni/perception/registry.py new file mode 100644 index 0000000..4f68c84 --- /dev/null +++ b/runtime/openglass_omni/perception/registry.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import importlib +import inspect +from typing import Any + +from .base import CVObservation, FrameEnvelope, PerceptionProvider +from .noop import NoOpProvider, ShadowProvider + + +BUILTIN_PROVIDERS = { + "noop": f"{__package__}.noop:NoOpProvider", + "ocr_template": f"{__package__}.ocr_template:OcrTemplateProvider", + "yolo_onnx": f"{__package__}.yolo_onnx:YoloOnnxProvider", +} + + +def load_provider(reference: str, options: dict[str, Any] | None = None) -> PerceptionProvider: + """Load `package.module:Class` or one of the small built-in aliases.""" + target = BUILTIN_PROVIDERS.get(reference, reference) + if ":" not in target: + raise ValueError( + "provider must be a built-in alias or package.module:Class" + ) + module_name, attribute_name = target.split(":", 1) + module = importlib.import_module(module_name) + factory = getattr(module, attribute_name) + kwargs = dict(options or {}) + instance = factory(**kwargs) if inspect.isclass(factory) else factory(kwargs) + if not callable(getattr(instance, "analyze", None)): + raise TypeError(f"provider {reference!r} does not implement analyze()") + return instance + + +class CVProviderRegistry: + def __init__(self, providers: dict[str, PerceptionProvider] | None = None): + raw = {"noop": NoOpProvider(), **(providers or {})} + self.providers = raw + self.shadow = { + provider_id: ShadowProvider(provider, provider_id) + for provider_id, provider in raw.items() + } + + def analyze(self, envelope: FrameEnvelope) -> CVObservation: + if envelope.mode != "shadow": + return CVObservation( + envelope.frame_id, + envelope.timestamp_ms, + envelope.skill_id, + f"{envelope.mode}:{envelope.provider_id}", + {"status": "skipped", "reason": "unsupported_mode"}, + f"Unsupported mode: {envelope.mode}", + ) + provider = self.shadow.get(envelope.provider_id) + if provider is None: + return CVObservation( + envelope.frame_id, + envelope.timestamp_ms, + envelope.skill_id, + f"shadow:{envelope.provider_id}", + {"status": "skipped", "reason": "unknown_provider"}, + f"Unknown provider: {envelope.provider_id}", + ) + return provider.analyze( + envelope.frame, + envelope.frame_id, + envelope.timestamp_ms, + envelope.skill_id, + envelope.slots, + ) diff --git a/runtime/openglass_omni/perception/shadow_runtime.py b/runtime/openglass_omni/perception/shadow_runtime.py new file mode 100644 index 0000000..faff6fe --- /dev/null +++ b/runtime/openglass_omni/perception/shadow_runtime.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import json +import logging +import threading +import time +from pathlib import Path +from typing import Any + +from .base import CVObservation, FrameEnvelope +from .pipeline import CVPipeline +from .registry import CVProviderRegistry, load_provider + + +LOG = logging.getLogger("openglass.perception") + + +class ShadowPerceptionRuntime: + """Bridge hook that logs provider output but never controls MiniCPM.""" + + def __init__( + self, + provider_reference: str, + *, + options: dict[str, Any] | None = None, + skill_id: str = "idle_chat", + slots: dict[str, Any] | None = None, + log_path: str | Path = "logs/cv_events.jsonl", + inference_timeout_ms: float = 2000.0, + ) -> None: + self.provider_id = provider_reference + self.skill_id = skill_id + self.slots = dict(slots or {}) + self.log_path = Path(log_path) + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self._write_lock = threading.Lock() + provider = load_provider(provider_reference, options) + self.pipeline = CVPipeline( + CVProviderRegistry({provider_reference: provider}), + on_observation=self._record, + inference_timeout_ms=inference_timeout_ms, + ) + + def submit(self, jpeg: bytes, frame_id: str, timestamp_ms: float) -> bool: + return self.pipeline.submit( + FrameEnvelope( + frame=jpeg, + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=self.skill_id, + slots=dict(self.slots), + mode="shadow", + provider_id=self.provider_id, + ) + ) + + def snapshot(self) -> dict[str, int | bool]: + return self.pipeline.snapshot() + + async def close(self) -> None: + await self.pipeline.close() + + def _record(self, observation: CVObservation) -> None: + record = {"logged_at_ms": time.time() * 1000.0, **observation.to_dict()} + with self._write_lock: + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + LOG.info( + "[CV] provider=%s frame=%s error=%s values=%s", + observation.provider, + observation.frame_id, + observation.error or "-", + observation.values, + ) diff --git a/runtime/openglass_omni/perception/yolo_onnx.py b/runtime/openglass_omni/perception/yolo_onnx.py new file mode 100644 index 0000000..d1a35f9 --- /dev/null +++ b/runtime/openglass_omni/perception/yolo_onnx.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import threading +import time +from pathlib import Path +from typing import Any + +import numpy as np + +from .base import CVObservation + + +COCO_NAMES = ( + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", + "truck", "boat", "traffic light", "fire hydrant", "stop sign", + "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", + "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", + "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", + "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", + "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", + "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", + "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", + "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", + "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", + "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", + "teddy bear", "hair drier", "toothbrush", +) + +DEFAULT_TARGET_ALIASES = { + "手机": "cell phone", "电话": "cell phone", "智能手机": "cell phone", + "phone": "cell phone", "cellphone": "cell phone", "cell phone": "cell phone", + "书": "book", "书本": "book", "图书": "book", "book": "book", + "杯子": "cup", "水杯": "cup", "茶杯": "cup", "cup": "cup", + "瓶子": "bottle", "水瓶": "bottle", "bottle": "bottle", + "椅子": "chair", "chair": "chair", "人": "person", "行人": "person", + "person": "person", "电脑": "laptop", "笔记本电脑": "laptop", + "laptop": "laptop", "遥控器": "remote", "remote": "remote", + "键盘": "keyboard", "keyboard": "keyboard", "鼠标": "mouse", + "mouse": "mouse", "背包": "backpack", "书包": "backpack", + "backpack": "backpack", +} + + +class YoloOnnxProvider: + """CPU-first YOLO reference plugin; it only produces shadow observations.""" + + def __init__( + self, + model_path: str | Path, + *, + device: str = "cpu", + confidence: float = 0.25, + image_size: int = 640, + target_aliases: dict[str, str] | None = None, + ) -> None: + if device.lower() != "cpu": + raise ValueError("V1 YOLO reference accepts only device=cpu") + self.model_path = Path(model_path).resolve() + self.confidence = float(confidence) + self.image_size = int(image_size) + self.target_aliases = { + **DEFAULT_TARGET_ALIASES, + **{str(k).lower(): str(v) for k, v in (target_aliases or {}).items()}, + } + self._session: Any = None + self._input_name: str | None = None + self._cv2: Any = None + self._lock = threading.Lock() + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + if not frame: + raise ValueError("YOLO requires a non-empty JPEG frame") + self._load() + target = str(slots.get("target") or "").strip() + target_label = self._resolve_target(target) if target else None + if target and target_label is None: + return CVObservation( + frame_id, timestamp_ms, skill_id, "yolo_onnx", + {"status": "skipped", "reason": "unsupported_target", "target": target}, + ) + + image = self._cv2.imdecode( + np.frombuffer(frame, dtype=np.uint8), self._cv2.IMREAD_COLOR + ) + if image is None: + raise ValueError("YOLO could not decode the JPEG frame") + height, width = image.shape[:2] + started = time.perf_counter() + tensor, scale, pad_x, pad_y = self._prepare_input(image) + with self._lock: + output = self._session.run(None, {self._input_name: tensor})[0] + if output.ndim != 3 or output.shape[0] != 1 or output.shape[2] != 6: + raise ValueError(f"Unexpected YOLO output shape: {output.shape}") + + target_id = COCO_NAMES.index(target_label) if target_label else None + detections: list[dict[str, Any]] = [] + for x1, y1, x2, y2, confidence, raw_class in output[0]: + class_id = round(float(raw_class)) + if float(confidence) < self.confidence or not 0 <= class_id < len(COCO_NAMES): + continue + if target_id is not None and class_id != target_id: + continue + left = max(0.0, min(width, (float(x1) - pad_x) / scale)) + top = max(0.0, min(height, (float(y1) - pad_y) / scale)) + right = max(0.0, min(width, (float(x2) - pad_x) / scale)) + bottom = max(0.0, min(height, (float(y2) - pad_y) / scale)) + if right <= left or bottom <= top: + continue + center_x = ((left + right) / 2.0) / width + center_y = ((top + bottom) / 2.0) / height + detections.append( + { + "label": COCO_NAMES[class_id], + "confidence": round(float(confidence), 4), + "bbox_xyxy": [round(left), round(top), round(right), round(bottom)], + "center_normalized": [round(center_x, 4), round(center_y, 4)], + "position": self._position(center_x, center_y), + } + ) + detections.sort(key=lambda item: float(item["confidence"]), reverse=True) + return CVObservation( + frame_id, + timestamp_ms, + skill_id, + "yolo_onnx", + { + "status": "ok", + "found": bool(detections), + "target": target or None, + "canonical_label": target_label, + "best_detection": detections[0] if detections else None, + "detections": detections, + "image": {"width": width, "height": height}, + "latency_ms": round((time.perf_counter() - started) * 1000.0, 1), + "model": self.model_path.name, + "device": "cpu", + }, + ) + + def _load(self) -> None: + if self._session is not None: + return + if not self.model_path.is_file(): + raise FileNotFoundError(f"YOLO weights not found: {self.model_path}") + import cv2 + import onnxruntime as ort + + session = ort.InferenceSession( + str(self.model_path), providers=["CPUExecutionProvider"] + ) + model_input = session.get_inputs()[0] + self._cv2 = cv2 + self._session = session + self._input_name = model_input.name + + def _prepare_input( + self, image: np.ndarray + ) -> tuple[np.ndarray, float, float, float]: + height, width = image.shape[:2] + scale = min(self.image_size / width, self.image_size / height) + resized_width, resized_height = round(width * scale), round(height * scale) + resized = self._cv2.resize(image, (resized_width, resized_height)) + pad_x = (self.image_size - resized_width) / 2.0 + pad_y = (self.image_size - resized_height) / 2.0 + left, top = round(pad_x - 0.1), round(pad_y - 0.1) + right = self.image_size - resized_width - left + bottom = self.image_size - resized_height - top + padded = self._cv2.copyMakeBorder( + resized, top, bottom, left, right, + self._cv2.BORDER_CONSTANT, value=(114, 114, 114), + ) + rgb = self._cv2.cvtColor(padded, self._cv2.COLOR_BGR2RGB) + tensor = np.ascontiguousarray(rgb.transpose(2, 0, 1), dtype=np.float32) + return np.expand_dims(tensor / 255.0, axis=0), scale, float(left), float(top) + + def _resolve_target(self, target: str) -> str | None: + normalized = target.lower().replace(" ", "") + for alias in sorted(self.target_aliases, key=len, reverse=True): + if alias.lower().replace(" ", "") in normalized: + label = self.target_aliases[alias] + return label if label in COCO_NAMES else None + return None + + @staticmethod + def _position(center_x: float, center_y: float) -> str: + horizontal = "左" if center_x < 1 / 3 else "右" if center_x > 2 / 3 else "中" + vertical = "上" if center_y < 1 / 3 else "下" if center_y > 2 / 3 else "中" + return { + ("左", "上"): "左上方", ("中", "上"): "正上方", + ("右", "上"): "右上方", ("左", "中"): "左侧", + ("中", "中"): "中央", ("右", "中"): "右侧", + ("左", "下"): "左下方", ("中", "下"): "正下方", + ("右", "下"): "右下方", + }[(horizontal, vertical)] diff --git a/runtime/openglass_omni/requirements-cv.txt b/runtime/openglass_omni/requirements-cv.txt new file mode 100644 index 0000000..89c94b0 --- /dev/null +++ b/runtime/openglass_omni/requirements-cv.txt @@ -0,0 +1,4 @@ +# Optional dependencies for the bundled YOLO ONNX reference provider. +# OCR providers should declare their model-specific dependencies separately. +onnxruntime>=1.18 +opencv-python>=4.9 diff --git a/runtime/openglass_omni/tests/__init__.py b/runtime/openglass_omni/tests/__init__.py new file mode 100644 index 0000000..eec37af --- /dev/null +++ b/runtime/openglass_omni/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the OpenGlass Omni runtime.""" diff --git a/runtime/openglass_omni/tests/test_perception.py b/runtime/openglass_omni/tests/test_perception.py new file mode 100644 index 0000000..7e32f5e --- /dev/null +++ b/runtime/openglass_omni/tests/test_perception.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import asyncio +import threading +import unittest + +from runtime.openglass_omni.perception.base import CVObservation, FrameEnvelope +from runtime.openglass_omni.perception.pipeline import CVPipeline +from runtime.openglass_omni.perception.registry import CVProviderRegistry, load_provider + + +class _FailingProvider: + def analyze(self, frame, frame_id, timestamp_ms, skill_id, slots): + del frame, frame_id, timestamp_ms, skill_id, slots + raise RuntimeError("model failed") + + +class _BlockingProvider: + def __init__(self) -> None: + self.started = threading.Event() + self.release = threading.Event() + + def analyze(self, frame, frame_id, timestamp_ms, skill_id, slots): + del frame, slots + if frame_id == "first": + self.started.set() + self.release.wait(timeout=2.0) + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider="blocking", + values={"status": "ok"}, + ) + + +class PerceptionRegistryTests(unittest.TestCase): + def test_builtin_provider_can_be_loaded_dynamically(self) -> None: + provider = load_provider("noop") + observation = provider.analyze(b"jpeg", "f1", 1.0, "idle_chat", {}) + self.assertEqual(observation.values["status"], "noop") + + def test_shadow_provider_isolates_plugin_failure(self) -> None: + registry = CVProviderRegistry({"broken": _FailingProvider()}) + observation = registry.analyze( + FrameEnvelope( + frame=b"jpeg", + frame_id="f1", + timestamp_ms=1.0, + skill_id="read_text", + provider_id="broken", + ) + ) + self.assertEqual(observation.values["status"], "error") + self.assertIn("RuntimeError", observation.error or "") + + +class PerceptionPipelineTests(unittest.IsolatedAsyncioTestCase): + async def test_capacity_one_keeps_latest_waiting_frame(self) -> None: + provider = _BlockingProvider() + observations: list[CVObservation] = [] + pipeline = CVPipeline( + CVProviderRegistry({"blocking": provider}), + on_observation=observations.append, + inference_timeout_ms=2000, + ) + try: + pipeline.submit(self._envelope("first")) + started = await asyncio.to_thread(provider.started.wait, 1.0) + self.assertTrue(started) + pipeline.submit(self._envelope("middle")) + pipeline.submit(self._envelope("latest")) + provider.release.set() + await asyncio.wait_for(pipeline.queue.join(), timeout=2.0) + finally: + provider.release.set() + await pipeline.close() + + self.assertEqual([item.frame_id for item in observations], ["first", "latest"]) + self.assertEqual(pipeline.snapshot()["dropped_frames"], 1) + + @staticmethod + def _envelope(frame_id: str) -> FrameEnvelope: + return FrameEnvelope( + frame=b"jpeg", + frame_id=frame_id, + timestamp_ms=1.0, + skill_id="read_text", + mode="shadow", + provider_id="blocking", + ) + + +if __name__ == "__main__": + unittest.main() From 8baff556d3cecb0de9877cb2c62509a3080ced6e Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Thu, 20 Aug 2026 16:02:11 +0800 Subject: [PATCH 2/2] Vendor Phase B Harness core into OpenGlass --- demo_rokid_phase_b_harness.py | 7 + docs/phase_ab_esp32_ocr_handoff_zh.md | 94 +- extensions/__init__.py | 1 + extensions/assistive_harness/.gitignore | 4 + .../OPENGLASS_MIGRATION_MAP.md | 25 + extensions/assistive_harness/README.md | 79 + extensions/assistive_harness/__init__.py | 13 + extensions/assistive_harness/asr/__init__.py | 5 + extensions/assistive_harness/asr/base.py | 29 + .../assistive_harness/asr/energy_vad.py | 91 + .../assistive_harness/asr/funasr_engine.py | 84 + .../config/skills.example.yaml | 128 ++ extensions/assistive_harness/cv/README.md | 171 ++ extensions/assistive_harness/cv/__init__.py | 14 + extensions/assistive_harness/cv/base.py | 44 + extensions/assistive_harness/cv/noop.py | 60 + extensions/assistive_harness/cv/pipeline.py | 159 ++ extensions/assistive_harness/cv/registry.py | 89 + .../assistive_harness/cv/requirements-cv.txt | 2 + extensions/assistive_harness/cv/yolo_onnx.py | 220 +++ .../download_modelscope_model.py | 69 + extensions/assistive_harness/echo_guard.py | 75 + extensions/assistive_harness/model_log.py | 72 + .../assistive_harness/phase_b/README.md | 114 ++ .../assistive_harness/phase_b/__init__.py | 5 + .../phase_b/requirements-phase-b.txt | 3 + .../phase_b/rokid_runtime.py | 1652 +++++++++++++++++ .../assistive_harness/prompts/README.md | 39 + .../prompts/describe_scene_zh.txt | 8 + .../prompts/find_object_zh.txt | 9 + .../prompts/idle_chat_zh.txt | 8 + .../prompts/obstacle_avoidance_zh.txt | 9 + .../prompts/read_text_zh.txt | 7 + extensions/assistive_harness/registry.py | 117 ++ extensions/assistive_harness/requirements.txt | 8 + extensions/assistive_harness/router.py | 262 +++ extensions/assistive_harness/schemas.py | 67 + extensions/assistive_harness/server.py | 571 ++++++ extensions/assistive_harness/state_machine.py | 115 ++ .../assistive_harness/summarize_reset_run.py | 97 + extensions/assistive_harness/telemetry.py | 67 + .../assistive_harness/tests/__init__.py | 1 + .../assistive_harness/tests/test_core.py | 402 ++++ .../assistive_harness/tests/test_cv_yolo.py | 35 + .../tests/test_phase_b_rokid.py | 352 ++++ integrations/minicpm_browser/README_zh.md | 20 + .../assistive_harness/acceptance-testbed.html | 17 + .../assistive_harness/acceptance-testbed.js | 115 ++ .../audio-mirror-processor.js | 36 + .../browser-session-adapter.js | 307 +++ .../browser-session-adapter.test.mjs | 202 ++ .../assistive_harness/harness-client.js | 212 +++ .../static/assistive_harness/package.json | 4 + models/README.md | 6 + runtime/openglass_omni/README.md | 10 +- 55 files changed, 6384 insertions(+), 28 deletions(-) create mode 100644 demo_rokid_phase_b_harness.py create mode 100644 extensions/__init__.py create mode 100644 extensions/assistive_harness/.gitignore create mode 100644 extensions/assistive_harness/OPENGLASS_MIGRATION_MAP.md create mode 100644 extensions/assistive_harness/README.md create mode 100644 extensions/assistive_harness/__init__.py create mode 100644 extensions/assistive_harness/asr/__init__.py create mode 100644 extensions/assistive_harness/asr/base.py create mode 100644 extensions/assistive_harness/asr/energy_vad.py create mode 100644 extensions/assistive_harness/asr/funasr_engine.py create mode 100644 extensions/assistive_harness/config/skills.example.yaml create mode 100644 extensions/assistive_harness/cv/README.md create mode 100644 extensions/assistive_harness/cv/__init__.py create mode 100644 extensions/assistive_harness/cv/base.py create mode 100644 extensions/assistive_harness/cv/noop.py create mode 100644 extensions/assistive_harness/cv/pipeline.py create mode 100644 extensions/assistive_harness/cv/registry.py create mode 100644 extensions/assistive_harness/cv/requirements-cv.txt create mode 100644 extensions/assistive_harness/cv/yolo_onnx.py create mode 100644 extensions/assistive_harness/download_modelscope_model.py create mode 100644 extensions/assistive_harness/echo_guard.py create mode 100644 extensions/assistive_harness/model_log.py create mode 100644 extensions/assistive_harness/phase_b/README.md create mode 100644 extensions/assistive_harness/phase_b/__init__.py create mode 100644 extensions/assistive_harness/phase_b/requirements-phase-b.txt create mode 100644 extensions/assistive_harness/phase_b/rokid_runtime.py create mode 100644 extensions/assistive_harness/prompts/README.md create mode 100644 extensions/assistive_harness/prompts/describe_scene_zh.txt create mode 100644 extensions/assistive_harness/prompts/find_object_zh.txt create mode 100644 extensions/assistive_harness/prompts/idle_chat_zh.txt create mode 100644 extensions/assistive_harness/prompts/obstacle_avoidance_zh.txt create mode 100644 extensions/assistive_harness/prompts/read_text_zh.txt create mode 100644 extensions/assistive_harness/registry.py create mode 100644 extensions/assistive_harness/requirements.txt create mode 100644 extensions/assistive_harness/router.py create mode 100644 extensions/assistive_harness/schemas.py create mode 100644 extensions/assistive_harness/server.py create mode 100644 extensions/assistive_harness/state_machine.py create mode 100644 extensions/assistive_harness/summarize_reset_run.py create mode 100644 extensions/assistive_harness/telemetry.py create mode 100644 extensions/assistive_harness/tests/__init__.py create mode 100644 extensions/assistive_harness/tests/test_core.py create mode 100644 extensions/assistive_harness/tests/test_cv_yolo.py create mode 100644 extensions/assistive_harness/tests/test_phase_b_rokid.py create mode 100644 integrations/minicpm_browser/README_zh.md create mode 100644 integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.html create mode 100644 integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.js create mode 100644 integrations/minicpm_browser/static/assistive_harness/audio-mirror-processor.js create mode 100644 integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.js create mode 100644 integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.test.mjs create mode 100644 integrations/minicpm_browser/static/assistive_harness/harness-client.js create mode 100644 integrations/minicpm_browser/static/assistive_harness/package.json create mode 100644 models/README.md diff --git a/demo_rokid_phase_b_harness.py b/demo_rokid_phase_b_harness.py new file mode 100644 index 0000000..13f1fc6 --- /dev/null +++ b/demo_rokid_phase_b_harness.py @@ -0,0 +1,7 @@ +"""Stable entry point for the Rokid Phase B Harness runtime.""" + +from extensions.assistive_harness.phase_b.rokid_runtime import main + + +if __name__ == "__main__": + main() diff --git a/docs/phase_ab_esp32_ocr_handoff_zh.md b/docs/phase_ab_esp32_ocr_handoff_zh.md index ecf0254..2779434 100644 --- a/docs/phase_ab_esp32_ocr_handoff_zh.md +++ b/docs/phase_ab_esp32_ocr_handoff_zh.md @@ -19,7 +19,7 @@ Phase B: Rokid 或 ESP32 -> Device Adapter ----+ | llama.cpp-omni -> Worker -> Gateway :8040 -> MiniCPM-o 4.5 -> PC 扬声器 ``` -- `MiniCPM-o-Demo-Comni/extensions/assistive_harness/` 是唯一的命令 Router、状态机、Skill YAML、prompt 和控制协议实现。 +- `OpenGlass/extensions/assistive_harness/` 是唯一的命令 Router、状态机、Skill YAML、prompt 和控制协议实现。 - OpenGlass 只负责 ESP32/Rokid 的 JPEG、PCM、播放、录制以及可选 CV provider。不要在设备端再复制一套关键词或状态机。 - RESET 和 Skill 切换只替换 Gateway Session，不能重载 Worker 或模型后端。 - CV provider 只返回统一的 `CVObservation`。V1.1 的 ESP32 接入是旁路日志，不能阻塞主音频链路。 @@ -81,30 +81,30 @@ git switch -c feature/esp32-ocr- `feature/phase-b-rokid-debug`。各自通过 PR 合并回协作基线；确认设备回归后， 再向 OpenGlass `main` 提交上游 PR。 -需要两个并列仓库： +完整 MiniCPM 后端仍是外部依赖，但 Harness 与 OpenGlass 设备/CV 代码现在都由 +这一个协作分支提供： ```text -C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni -C:\Users\Lenovo\AI_WAIC\github\OpenGlass +<任意目录>\MiniCPM-o-Demo-Comni # 外部 Worker/Gateway/网页后端 +<任意目录>\OpenGlass # 本分支：Harness + Phase B + CV/OCR ``` OpenGlass 分支检查： ```bat -cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +cd /d C:\path\to\OpenGlass git switch codex/phase-b-v1.1-esp32-ocr-handoff git status --short --branch ``` -当前本机的 `MiniCPM-o-Demo-Comni` 目录不是 Git 工作树。因此本 OpenGlass -分支只包含设备适配、CV 插件骨架和本文，**不会自动包含 Harness Core 的 -V1.1 修改**。交付给同事时应同时提供经过审阅的 MiniCPM 工作目录归档，或 -后续把它接入一个明确的远端仓库；不要把 Harness Core 复制进 OpenGlass。 +本分支已经包含完整 Harness Core、Rokid Phase B 入口、浏览器薄适配资产和 +ESP32/CV 插件骨架。同事不再需要接收本机 MiniCPM 工作目录归档。只有 +`llama.cpp-omni -> Worker -> Gateway` 后端和模型权重继续作为外部依赖。 本次文件边界如下： ```text -MiniCPM-o-Demo-Comni/ +OpenGlass/ extensions/assistive_harness/model_log.py # 新增模型回合聚合 extensions/assistive_harness/server.py # MODEL/playback telemetry extensions/assistive_harness/telemetry.py # 模型日志文件 @@ -113,8 +113,8 @@ MiniCPM-o-Demo-Comni/ extensions/assistive_harness/phase_b/rokid_runtime.py # V1.1 播放与触发 extensions/assistive_harness/phase_b/README.md # Rokid 使用说明 extensions/assistive_harness/tests/ # 回归测试 - -OpenGlass/ + demo_rokid_phase_b_harness.py # Rokid 稳定入口 + integrations/minicpm_browser/ # Phase A 浏览器薄适配资产 runtime/openglass_omni/esp32_bridge.py # 非阻塞 CV 挂点/CLI runtime/openglass_omni/perception/ # provider/pipeline/YOLO/OCR runtime/openglass_omni/requirements-cv.txt # 可选 CV 依赖 @@ -128,9 +128,11 @@ ESP32 已烧录连接 `CUDY-D102` 的固件。电脑也要连接同一网络， ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +cd /d C:\path\to\OpenGlass python -m pip install -r runtime\openglass_omni\requirements.txt python -m pip install -r runtime\openglass_omni\requirements-cv.txt +python -m pip install -r extensions\assistive_harness\requirements.txt +python -m pip install -r extensions\assistive_harness\phase_b\requirements-phase-b.txt ``` OCR 后端若需要 PaddleOCR、ONNX Runtime、TensorRT 等额外依赖，请单独增加 `requirements-ocr-.txt`，不要把重型 OCR 依赖塞入基础 requirements。 @@ -143,7 +145,7 @@ OCR 后端若需要 PaddleOCR、ONNX Runtime、TensorRT 等额外依赖，请单 ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +cd /d C:\path\to\MiniCPM-o-Demo-Comni start_all.cmd --http ``` @@ -151,11 +153,21 @@ start_all.cmd --http ### 4.2 Harness Core :8021 -打开第二个 Anaconda Prompt： +`--model-path` 表示本机目录，不是 ModelScope 模型 ID，也不会触发隐式下载。 +第一次使用时，在 OpenGlass 根目录显式下载公开模型： ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +cd /d C:\path\to\OpenGlass +python -m extensions.assistive_harness.download_modelscope_model +``` + +ModelScope 默认下载到 `%USERPROFILE%\.cache\modelscope\hub`，命令末尾会打印 +实际 `MODEL_PATH` 和可直接复制的启动命令。之后打开第二个 Anaconda Prompt： + +```bat +conda activate ai_glasses +cd /d C:\path\to\OpenGlass python -m extensions.assistive_harness.server --enabled --model-path "%USERPROFILE%\.cache\modelscope\hub\models\iic\speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online" --port 8021 ``` @@ -172,7 +184,9 @@ extensions/assistive_harness/runs//model_transcript.txt ### 4.3 Chrome Phase A 回归 -前两个 Prompt 就绪后，用 Chrome 打开带显式开关的页面： +前两个 Prompt 就绪后，用 Chrome 打开带显式开关的页面。浏览器薄适配资产 +位于 `integrations/minicpm_browser/`，必须先按其中 README 接入目标版本的 +MiniCPM-o-Demo；不要用一个完整 `omni-app.js` 覆盖不同上游版本。 ```text http://127.0.0.1:8040/omni?assistive_harness=1&v=phase-b-v1-1 @@ -188,7 +202,7 @@ STOP、RESUME、RESET、四种 Skill 和 Skill 互切。原生回归则另开 ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +cd /d C:\path\to\OpenGlass python demo_rokid_phase_b_harness.py --gateway localhost:8040 --harness-url ws://127.0.0.1:8021/ws/control --input-gain 12 --image-rotate-cw 270 --session-ready-chime-volume 0.32 --playback-echo-tail-s 0.80 ``` @@ -196,11 +210,21 @@ python demo_rokid_phase_b_harness.py --gateway localhost:8040 --harness-url ws:/ ### 4.5 ESP32 主链路 smoke test（OCR 同事） -先验证 ESP32 输入与 MiniCPM 主链路；把 `` 替换为设备当前地址： +先验证 ESP32 输入与 MiniCPM 主链路；把 `` 替换为设备当前地址。 +这里的 `SenseVoiceSmall` 是 ESP32 bridge 的可选本地 ASR 模型，与 Harness +使用的 Paraformer 模型不是同一个目录。第一次使用时显式下载： ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +cd /d C:\path\to\OpenGlass +python -m extensions.assistive_harness.download_modelscope_model --model-id iic/SenseVoiceSmall +``` + +下载程序会打印实际目录。把该目录传给 `--funasr-model`： + +```bat +conda activate ai_glasses +cd /d C:\path\to\OpenGlass python runtime\openglass_omni\esp32_bridge.py --esp32-host --esp32-port 80 --gateway localhost:8040 --no-tls --rotate 180 --enable-funasr --funasr-model "%USERPROFILE%\.cache\modelscope\hub\models\iic\SenseVoiceSmall" --funasr-echo-suppress-s 4.0 --prompt "你是智能眼镜助手。用户问什么就简短回答什么；只有用户要求描述场景或寻找物体时才看图回答，不要主动描述。" ``` @@ -212,8 +236,8 @@ YOLO 只是验证 provider 接口、异常隔离和非阻塞队列，不是最 ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass -python runtime\openglass_omni\esp32_bridge.py --esp32-host --esp32-port 80 --gateway localhost:8040 --no-tls --rotate 180 --cv-shadow-provider yolo_onnx --cv-shadow-options-json "{\"model_path\":\"C:\\Users\\Lenovo\\AI_Glasses_0618\\OmniHarness\\mini_omni_harness\\models\\yolo26n.onnx\",\"device\":\"cpu\"}" --cv-shadow-skill find_object --cv-shadow-slots-json "{\"target\":\"手机\"}" --cv-shadow-timeout-ms 2000 --cv-shadow-log logs\cv_yolo.jsonl +cd /d C:\path\to\OpenGlass +python runtime\openglass_omni\esp32_bridge.py --esp32-host --esp32-port 80 --gateway localhost:8040 --no-tls --rotate 180 --cv-shadow-provider yolo_onnx --cv-shadow-options-json "{\"model_path\":\"C:\\path\\to\\OpenGlass\\models\\yolo26n.onnx\",\"device\":\"cpu\"}" --cv-shadow-skill find_object --cv-shadow-slots-json "{\"target\":\"手机\"}" --cv-shadow-timeout-ms 2000 --cv-shadow-log logs\cv_yolo.jsonl ``` 预期：终端持续出现 `[CV]`，`logs\cv_yolo.jsonl` 每行是一条 `CVObservation`；即使模型加载失败或超时，ESP32 PCM 与 MiniCPM 音频仍应继续。队列容量固定为 1：推理忙时丢弃旧帧，只保留最新等待帧。 @@ -249,13 +273,31 @@ python runtime\openglass_omni\esp32_bridge.py --esp32-host --gateway OCR V1 完成标准：真实 ESP32 JPEG 能产出统一结果；异常和超时只形成 observation；音频主链不中断；连续输入发生背压时 `dropped_frames` 增长而内存不增长；至少有一张含中英文的固定测试图和预期文本。 -## 7. 测试 +## 7. GitHub 不包含的文件及解决办法 + +代码、配置模板、测试和启动入口都在本分支中；以下大文件或本机状态有意不进入 +Git，不属于代码遗漏： + +| 类别 | 原因 | 获取/放置方式 | +|---|---|---| +| Harness Paraformer ASR 权重 | 公开模型体积较大 | 运行 `download_modelscope_model`，使用程序打印的本地路径 | +| ESP32 `SenseVoiceSmall` 权重 | 可选 bridge ASR | 用同一下载程序加 `--model-id iic/SenseVoiceSmall` | +| YOLO 权重 | CV 参考插件的可替换资产 | 自行取得兼容 ONNX 权重，放入 `OpenGlass\models\`，不要提交 | +| OCR 权重/字典 | 由具体 OCR 后端决定 | 放在开发者本机模型目录，通过 provider options 传路径 | +| MiniCPM-o 4.5/llama.cpp-omni 权重和 Worker/Gateway | 独立后端、体积大 | 按后端仓库安装并启动 `:8040`，OpenGlass 不复制模型 | +| Wi-Fi 密码、设备 IP、日志、录音 | 私密或运行时数据 | 只保存在本机，通过命令行参数传入 | + +因此，克隆本分支后可以直接安装依赖、运行单元测试、启动 Harness 和设备适配层； +要得到完整 AI 回答，还必须在本机准备公开 ASR 权重，并启动外部 MiniCPM +Worker/Gateway。OCR 同事只需额外提供其选择的 OCR 依赖和权重。 + +## 8. 测试 OpenGlass CV 骨架： ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_WAIC\github\OpenGlass +cd /d C:\path\to\OpenGlass python -m unittest discover runtime\openglass_omni\tests -v ``` @@ -263,13 +305,13 @@ Harness 核心与 Rokid Phase B： ```bat conda activate ai_glasses -cd /d C:\Users\Lenovo\AI_Glasses_0618\MiniCPM-o-Demo-Comni +cd /d C:\path\to\OpenGlass python -m unittest extensions.assistive_harness.tests.test_core extensions.assistive_harness.tests.test_phase_b_rokid -v ``` 设备验收顺序：普通问答 -> `停一下` -> `恢复对话` -> `重新开始` -> 找物 -> 识字 -> 场景描述 -> 实验性避障 -> Skill 直接互切 -> 断网重连。每次 RESET/Skill 切换应看到 generation 增加、新 Session ID、`restart_complete`，Skill 还应看到一次 `task_trigger_sent`。 -## 8. 协作与后续集成 +## 9. 协作与后续集成 - OCR 同事只在本 OpenGlass 分支新增 provider、测试和依赖文件；不要修改 Phase A 的关键词和状态机。 - 项目负责人继续在 Rokid 真实设备上校准回声、提示音和 Session 稳定性。 diff --git a/extensions/__init__.py b/extensions/__init__.py new file mode 100644 index 0000000..250d3b6 --- /dev/null +++ b/extensions/__init__.py @@ -0,0 +1 @@ +"""OpenGlass extension packages.""" diff --git a/extensions/assistive_harness/.gitignore b/extensions/assistive_harness/.gitignore new file mode 100644 index 0000000..e607f1c --- /dev/null +++ b/extensions/assistive_harness/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +runs/ +test_artifacts/ diff --git a/extensions/assistive_harness/OPENGLASS_MIGRATION_MAP.md b/extensions/assistive_harness/OPENGLASS_MIGRATION_MAP.md new file mode 100644 index 0000000..1110736 --- /dev/null +++ b/extensions/assistive_harness/OPENGLASS_MIGRATION_MAP.md @@ -0,0 +1,25 @@ +# OpenGlass Migration Map + +Phase A deliberately separates portable policy from browser-specific transport. + +| Phase A component | OpenGlass-side destination | +|---|---| +| `schemas.py` | shared structured control-event schema | +| `router.py` | local deterministic voice-command router | +| `registry.py` + prompts | configurable Skill registry/prompt bundle | +| `echo_guard.py` | output/playback-aware ASR echo filter | +| `state_machine.py` | device-independent control priority and dedupe | +| `asr/` | phone/host local ASR service behind the same interface | +| `cv/base.py` | future advisory perception provider interface | +| `server.py` WS messages | replaceable host/phone transport adapter | +| `browser-session-adapter.js` | reference semantics for an OpenGlass session adapter | + +OpenGlass migration should keep `ControlEvent`, `SkillRegistry`, prompt SHA, +generation fence and STOP priority stable. Replace only AudioMirror capture, +session lifecycle calls, and the control transport. No dependency on ESP32, Rokid, +DOM layout or the MiniCPM private wire format exists in the Python core. + +The first Rokid implementation of this boundary now lives in +[`phase_b/`](phase_b/README.md). It consumes the existing APK JPEG/PCM endpoints, +reuses this Core and the existing Gateway, and implements the browser adapter's +STOP/RESUME/RESET/Skill generation contract in Python. diff --git a/extensions/assistive_harness/README.md b/extensions/assistive_harness/README.md new file mode 100644 index 0000000..904aedf --- /dev/null +++ b/extensions/assistive_harness/README.md @@ -0,0 +1,79 @@ +# Assistive Voice Skill Harness (Phase A / Phase B Core) + +This directory is the transport-neutral control Core shared by the optional +MiniCPM-o browser sidecar and OpenGlass Phase B device adapters. It is disabled +by default and does not replace the native microphone/video path. + +## Clean-clone setup + +From the OpenGlass repository root, install the Core and explicitly download +the public ASR model: + +```powershell +python -m pip install -r extensions/assistive_harness/requirements.txt +python -m extensions.assistive_harness.download_modelscope_model +``` + +The second command prints the downloaded local directory and a complete start +command. Model weights are not stored in this Git repository. The runtime never +downloads a model implicitly: `--model-path` must point to an existing local +directory, otherwise startup fails clearly. + +Start the sidecar explicitly: + +```powershell +python -m extensions.assistive_harness.server --enabled ` + --model-path "C:\path\to\a\local\FunASR\model" +``` + +Then opt the browser tab in with `?assistive_harness=1`. Browser integration +assets and their hook contract live in +[`../../integrations/minicpm_browser/`](../../integrations/minicpm_browser/). +Test-only transcript +injection additionally requires `--allow-test-injection`; it is never enabled by +the normal command above. + +The browser hook is deliberately thin. Control, ASR, prompt registry, echo guard, +state machine, metrics, and CV shadow interfaces live under this directory so the +module can later be moved behind another transport (for example OpenGlass). + +The first Rokid/OpenGlass transport is implemented in +[`phase_b/`](phase_b/README.md). It preserves this Core and the existing 8040 +Gateway while replacing browser microphone, camera, playback, and Session calls +with a Python device adapter. + +## User-editable Skills + +System prompts live in `extensions/assistive_harness/prompts/`. Existing prompt +files are read on every activation, so users can replace a prompt and activate +the Skill again without restarting the sidecar. Skill IDs, enable flags and +voice phrases are configured in `config/skills.example.yaml`; changing that YAML +does require restarting the sidecar. + +Enabled by default: + +- `帮我找<物体>` -> `find_object` -> hot Session restart with `{{target}}` +- `读一下` / `帮我识字` -> `read_text` -> hot Session restart +- `描述一下` / `看看周围` -> `describe_scene` -> hot Session restart +- `帮我避障` / `前面有障碍吗` -> `obstacle_avoidance` -> hot Session restart +- `回到聊天` / `恢复普通聊天` -> `idle_chat` -> hot Session restart + +`obstacle_avoidance` contains the frozen AAAI_SI prompt and is enabled only for +stationary, supervised validation. Its mobility safety has not been accepted; +never treat this Demo as a navigation or safety device. + +With the sidecar running, `GET http://127.0.0.1:8021/skills` reports the active +configuration and absolute prompt path for every registered Skill. + +## CV V1 shadow pipeline + +Browser JPEG mirrors enter a capacity-one latest-frame queue and are analyzed +on a dedicated single-thread worker. `cv_mode: disabled` drops frames before +inference; `cv_mode: shadow` writes `CVObservation` records without changing a +Skill or taking ownership of MiniCPM output. Slow, timed-out, or failed plugins +remain outside the audio/control receive path. + +`find_object` uses the local `yolo_onnx` reference provider. Other Skills keep +the `noop` provider until a task-specific plugin is registered. Provider setup, +the observation schema, and an OCR implementation template are documented in +[`cv/README.md`](cv/README.md). diff --git a/extensions/assistive_harness/__init__.py b/extensions/assistive_harness/__init__.py new file mode 100644 index 0000000..f40e11f --- /dev/null +++ b/extensions/assistive_harness/__init__.py @@ -0,0 +1,13 @@ +"""Portable voice-controlled Skill Harness core for Phase A.""" + +from .router import RuleIntentRouter +from .registry import SkillRegistry +from .schemas import ASREvent, ControlEvent, ControlIntent + +__all__ = [ + "ASREvent", + "ControlEvent", + "ControlIntent", + "RuleIntentRouter", + "SkillRegistry", +] diff --git a/extensions/assistive_harness/asr/__init__.py b/extensions/assistive_harness/asr/__init__.py new file mode 100644 index 0000000..c3bbd54 --- /dev/null +++ b/extensions/assistive_harness/asr/__init__.py @@ -0,0 +1,5 @@ +from .base import ASREngine, ASRResult +from .energy_vad import EnergyVAD, UtteranceAudio +from .funasr_engine import FunASREngine + +__all__ = ["ASREngine", "ASRResult", "EnergyVAD", "FunASREngine", "UtteranceAudio"] diff --git a/extensions/assistive_harness/asr/base.py b/extensions/assistive_harness/asr/base.py new file mode 100644 index 0000000..511f465 --- /dev/null +++ b/extensions/assistive_harness/asr/base.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +import numpy as np + + +@dataclass(frozen=True, slots=True) +class ASRResult: + text: str + confidence: float + model: str + device: str + + +class ASREngine(Protocol): + def transcribe(self, audio: np.ndarray, sample_rate: int) -> ASRResult: + ... + + +class ScriptedASREngine: + def __init__(self, transcripts: list[str]): + self.transcripts = list(transcripts) + + def transcribe(self, audio: np.ndarray, sample_rate: int) -> ASRResult: + del audio, sample_rate + text = self.transcripts.pop(0) if self.transcripts else "" + return ASRResult(text=text, confidence=1.0, model="scripted", device="cpu") diff --git a/extensions/assistive_harness/asr/energy_vad.py b/extensions/assistive_harness/asr/energy_vad.py new file mode 100644 index 0000000..2de6ef1 --- /dev/null +++ b/extensions/assistive_harness/asr/energy_vad.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass + +import numpy as np + + +@dataclass(frozen=True, slots=True) +class UtteranceAudio: + audio: np.ndarray + started_at_ms: float + ended_at_ms: float + + +class EnergyVAD: + """Small streaming endpoint detector; it does not perform recognition.""" + + def __init__( + self, + sample_rate: int = 16_000, + rms_threshold: float = 0.012, + min_speech_ms: int = 180, + end_silence_ms: int = 450, + max_utterance_ms: int = 8_000, + preroll_ms: int = 200, + ): + self.sample_rate = sample_rate + self.rms_threshold = rms_threshold + self.min_speech_ms = min_speech_ms + self.end_silence_ms = end_silence_ms + self.max_utterance_ms = max_utterance_ms + self.preroll_ms = preroll_ms + self._preroll: deque[tuple[np.ndarray, float]] = deque() + self._active: list[np.ndarray] = [] + self._started_at_ms: float | None = None + self._last_voice_ms: float | None = None + + def _duration_ms(self, audio: np.ndarray) -> float: + return float(audio.size) * 1000.0 / self.sample_rate + + def feed(self, audio: np.ndarray, frame_started_at_ms: float) -> UtteranceAudio | None: + frame = np.asarray(audio, dtype=np.float32).reshape(-1).copy() + if frame.size == 0: + return None + duration_ms = self._duration_ms(frame) + frame_end_ms = frame_started_at_ms + duration_ms + rms = float(np.sqrt(np.mean(np.square(frame, dtype=np.float64)))) + voiced = rms >= self.rms_threshold + + if self._started_at_ms is None: + self._preroll.append((frame, frame_started_at_ms)) + while self._preroll and frame_end_ms - self._preroll[0][1] > self.preroll_ms: + self._preroll.popleft() + if not voiced: + return None + self._started_at_ms = self._preroll[0][1] if self._preroll else frame_started_at_ms + self._active = [item[0] for item in self._preroll] + self._preroll.clear() + self._last_voice_ms = frame_end_ms + else: + self._active.append(frame) + if voiced: + self._last_voice_ms = frame_end_ms + + active_ms = frame_end_ms - float(self._started_at_ms) + silence_ms = frame_end_ms - float(self._last_voice_ms or frame_end_ms) + if active_ms >= self.max_utterance_ms or silence_ms >= self.end_silence_ms: + return self._finish(frame_end_ms) + return None + + def _finish(self, ended_at_ms: float) -> UtteranceAudio | None: + if self._started_at_ms is None or not self._active: + self.reset() + return None + audio = np.concatenate(self._active).astype(np.float32, copy=False) + started = self._started_at_ms + voiced_duration = max(0.0, float(self._last_voice_ms or ended_at_ms) - started) + self.reset() + if voiced_duration < self.min_speech_ms: + return None + return UtteranceAudio(audio=audio, started_at_ms=started, ended_at_ms=ended_at_ms) + + def flush(self, ended_at_ms: float) -> UtteranceAudio | None: + return self._finish(ended_at_ms) + + def reset(self) -> None: + self._preroll.clear() + self._active = [] + self._started_at_ms = None + self._last_voice_ms = None diff --git a/extensions/assistive_harness/asr/funasr_engine.py b/extensions/assistive_harness/asr/funasr_engine.py new file mode 100644 index 0000000..c7a126d --- /dev/null +++ b/extensions/assistive_harness/asr/funasr_engine.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import threading +from pathlib import Path +from typing import Any + +import numpy as np + +from .base import ASRResult + + +class FunASREngine: + """Lazy local FunASR adapter. It never downloads a model implicitly.""" + + def __init__(self, model_path: str, device: str = "cpu", model_kwargs: dict[str, Any] | None = None): + path = Path(model_path).expanduser().resolve() + if not path.is_dir(): + raise FileNotFoundError(f"FunASR model path does not exist: {path}") + self.model_path = str(path) + self.device = device + self.model_kwargs = dict(model_kwargs or {}) + self._model: Any = None + self._load_lock = threading.Lock() + self._infer_lock = threading.Lock() + + @property + def loaded(self) -> bool: + return self._model is not None + + def ensure_loaded(self) -> None: + if self._model is not None: + return + with self._load_lock: + if self._model is not None: + return + from funasr import AutoModel + + self._model = AutoModel( + model=self.model_path, + device=self.device, + disable_update=True, + **self.model_kwargs, + ) + + def warm_up(self) -> None: + """Load the model and run one short silent inference before serving clients.""" + self.ensure_loaded() + silence = np.zeros(8_000, dtype=np.float32) + with self._infer_lock: + self._model.generate( + input=silence, + cache={}, + is_final=True, + batch_size_s=0, + ) + + def transcribe(self, audio: np.ndarray, sample_rate: int) -> ASRResult: + if sample_rate != 16_000: + raise ValueError(f"FunASR Phase A requires 16 kHz audio, got {sample_rate}") + self.ensure_loaded() + waveform = np.asarray(audio, dtype=np.float32).reshape(-1) + with self._infer_lock: + results = self._model.generate( + input=waveform, + cache={}, + is_final=True, + batch_size_s=0, + ) + text = "" + confidence = 1.0 + if isinstance(results, list) and results: + first = results[0] + if isinstance(first, dict): + text = str(first.get("text") or "").strip() + if isinstance(first.get("confidence"), (int, float)): + confidence = float(first["confidence"]) + else: + text = str(first).strip() + return ASRResult( + text=text, + confidence=confidence, + model=self.model_path, + device=self.device, + ) diff --git a/extensions/assistive_harness/config/skills.example.yaml b/extensions/assistive_harness/config/skills.example.yaml new file mode 100644 index 0000000..f2b4c76 --- /dev/null +++ b/extensions/assistive_harness/config/skills.example.yaml @@ -0,0 +1,128 @@ +version: 2 +default_skill: idle_chat + +harness: + enabled: false + host: 127.0.0.1 + port: 8021 + allow_test_injection: false + +asr: + engine: funasr + model_path: "" + device: cpu + rms_threshold: 0.012 + min_speech_ms: 180 + end_silence_ms: 450 + max_utterance_ms: 8000 + preroll_ms: 200 + +echo_guard: + window_ms: 20000 + similarity_threshold: 0.86 + +cv: + mode: shadow + max_fps: 1 + queue_size: 1 + inference_timeout_ms: 2000 + providers: + noop: + type: noop + yolo_onnx: + type: yolo_onnx + # Resolved relative to this YAML. Weights stay in OpenGlass/models and + # are ignored by Git; the Core never downloads them implicitly. + model_path: ../../../models/yolo26n.onnx + device: cpu + confidence: 0.25 + image_size: 640 + +control: + cooldown_ms: 1200 + stop_speech: + # `同一下` / `等一(一)下` are bounded aliases observed from the local + # FunASR model when the user clearly says the short command `停一下`. + # They are command-only aliases, not arbitrary substring anchors. + phrases: [停一下, 同一下, 等一下, 等一一下, 别说了, 闭嘴, 停止播报, 安静] + # Literal substring protocol: any transcript containing this anchor stops. + embedded_phrases: [停一下] + resume_speech: + # Literal substring protocol: any transcript containing this anchor resumes. + embedded_phrases: [恢复对话] + reset_session: + phrases: [重新开始] + # Literal substring protocol for real voice RESET. Product deployments can + # later prepend a wake name (for example "乐奇重新开始") without changing + # the deterministic router. + embedded_phrases: [重新开始] + return_to_chat: + phrases: [回到聊天, 回到普通聊天, 退出技能, 普通聊天, 恢复普通聊天] + cancel_skill: + phrases: [取消任务, 不找了, 不读了] + +skills: + idle_chat: + enabled: true + prompt_file: ../prompts/idle_chat_zh.txt + requires_session_restart: true + cooldown_ms: 1200 + cv_mode: disabled + cv_provider: noop + activation_phrases: [普通聊天, 回到聊天] + + find_object: + enabled: true + description: 使用第一视角画面寻找语音中指定的物体 + prompt_source: AAAI_SI/C1_OBJECT_FINDING + prompt_file: ../prompts/find_object_zh.txt + requires_session_restart: true + cooldown_ms: 1500 + cv_mode: shadow + cv_provider: yolo_onnx + task_trigger: '请立即根据当前画面寻找“{{target}}”，只回答它的位置；如果没看到就说没看到。' + slot_schema: + target: {type: string, required: true} + activation_patterns: + - "帮我找(?:一下)?(?:我的)?(?P.+)" + - "帮我找(?P.+)" + - "(?P.+)在哪" + - "(?P.+)在哪里" + - "看到(?P.+)了吗" + - "有没有(?P.+)" + + read_text: + enabled: true + description: 读取当前画面中清晰可见的文字 + prompt_source: AAAI_SI/C1_TEXT_READING + prompt_file: ../prompts/read_text_zh.txt + requires_session_restart: true + cooldown_ms: 1500 + cv_mode: shadow + cv_provider: noop + task_trigger: 请立即读取当前画面中最明显的文字，只读看到的内容。 + activation_phrases: [读一下, 念一下, 帮我读一下, 帮我识字, 帮我读字, 上面写了什么, 这是什么字, 读文字] + + describe_scene: + enabled: true + prompt_file: ../prompts/describe_scene_zh.txt + requires_session_restart: true + cooldown_ms: 1500 + cv_mode: shadow + cv_provider: noop + task_trigger: 请立即用一两句话简短描述当前画面。 + activation_phrases: [描述一下, 帮我描述一下, 帮我描述场景, 前面有什么, 看看周围, 画面里有什么] + + obstacle_avoidance: + # Experimental: enabled only for stationary, supervised browser validation. + # It is not a mobility-safety guarantee and must not be used for navigation. + enabled: true + description: 识别前方可见障碍并给出保守提示（实验） + prompt_source: AAAI_SI/C1_OBSTACLE_AVOID_DIRECT_V1 + prompt_file: ../prompts/obstacle_avoidance_zh.txt + requires_session_restart: true + cooldown_ms: 1500 + cv_mode: shadow + cv_provider: noop + task_trigger: 请立即判断当前画面前方是否有明显障碍，并给出简短、保守的安全提示。 + activation_phrases: [帮我避障, 前面有障碍吗, 前面有没有障碍, 看一下前方障碍, 前面安全吗] diff --git a/extensions/assistive_harness/cv/README.md b/extensions/assistive_harness/cv/README.md new file mode 100644 index 0000000..2165f26 --- /dev/null +++ b/extensions/assistive_harness/cv/README.md @@ -0,0 +1,171 @@ +# Lightweight CV provider framework (V1) + +This directory is the transport-neutral CV shadow path used by the Assistive +Harness. Browser and future OpenGlass Adapters only provide JPEG frames. They do +not call YOLO, OCR, depth estimation, or control rules directly. + +## V1 data flow + +```text +Browser / ESP32 Adapter + | + | FrameEnvelope(JPEG, timestamp, skill_id, slots) + v +CVPipeline.submit() # returns immediately + | + | capacity-one latest-frame queue (old queued frame is dropped) + v +single background worker + | + | provider.analyze() in a dedicated CPU thread + v +CVObservation -> cv_events.jsonl # shadow only; no control decision +``` + +The Router, state machine, Session Adapter, and MiniCPM audio/video path do not +depend on a concrete CV model. + +## Guarantees and deliberate limits + +- `submit()` never waits for inference. +- The queue is bounded and latest-frame wins; overload cannot grow memory + without limit. +- Only one inference call runs at a time per client pipeline. +- Provider errors become `CVObservation.error` and never escape to the control + socket. +- A timeout is recorded without blocking STOP/RESET. Python cannot kill an + already-running native ONNX call safely, so that one worker lane remains + occupied until the native call returns; new queued frames continue to + collapse to the latest frame. +- V1 accepts only `disabled` and `shadow`. Shadow observations never switch a + Skill, suppress MiniCPM, or speak to the user. +- Temporal voting, result fusion, navigation decisions, and accuracy tuning are + intentionally outside V1. + +## Configuration + +Providers are declared once under `cv.providers`. Each Skill selects one with +`cv_provider`: + +```yaml +cv: + max_fps: 1 + queue_size: 1 + inference_timeout_ms: 2000 + providers: + noop: + type: noop + yolo_onnx: + type: yolo_onnx + model_path: path/to/yolo.onnx + device: cpu + confidence: 0.25 + image_size: 640 + +skills: + find_object: + cv_mode: shadow + cv_provider: yolo_onnx +``` + +Relative model paths are resolved against the YAML directory. Weights are not +owned by the Core. The shipped sample resolves to the Git-ignored local file +`OpenGlass/models/yolo26n.onnx`. + +## Provider contract + +A provider is synchronous. `CVPipeline` is responsible for running it outside +the asyncio/control loop: + +Provider instances are shared so model weights load only once. If a backend is +not safe for concurrent calls from multiple connected clients, the provider +must protect that backend with its own lock, as `YoloOnnxProvider` does. + +```python +from typing import Any +from extensions.assistive_harness.cv.base import CVObservation + + +class OcrOnnxProvider: + def __init__(self, model_path): + self.model_path = model_path + self._session = None # lazy-load on first frame + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + if not frame: + raise ValueError("OCR requires a JPEG frame") + # 1. Decode JPEG. + # 2. Lazy-load and run the local model. + # 3. Return JSON-serializable values only. + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider="ocr_onnx", + values={ + "status": "ok", + "text": "recognized text", + "regions": [], + "latency_ms": 12.3, + "model": self.model_path.name, + "device": "cpu", + }, + ) +``` + +To add OCR: + +1. Put the implementation in `cv/ocr_onnx.py` and keep model-specific imports + inside that plugin. +2. Add one `ocr_onnx` factory branch in `build_provider_registry()` in + `cv/registry.py`. +3. Add its config under `cv.providers`. +4. Set `read_text.cv_provider: ocr_onnx`; leave `cv_mode: shadow` during tests. +5. Add a blank/synthetic inference smoke plus one recorded real-frame test. + +No changes are required in `router.py`, `state_machine.py`, the browser Adapter, +or the future ESP32 Frame Adapter. + +## Observation schema + +Every plugin produces the same envelope: + +```json +{ + "frame_id": "browser_123", + "timestamp_ms": 123.0, + "skill_id": "find_object", + "provider": "shadow:yolo_onnx", + "values": { + "status": "ok", + "found": true, + "detections": [], + "latency_ms": 31.1, + "pipeline_latency_ms": 42.0 + }, + "error": null +} +``` + +Plugin-specific data belongs under `values`; the outer keys stay stable for +logging, replay, and later OpenGlass migration. + +## Validation + +```powershell +python -m unittest ` + extensions.assistive_harness.tests.test_core ` + extensions.assistive_harness.tests.test_cv_yolo +``` + +The tests cover failure isolation, latest-frame dropping, timeout reporting, +non-blocking submission, and a real ONNX Runtime YOLO inference. A browser +acceptance run should additionally verify that `find_object` produces +`provider=shadow:yolo_onnx` records while STOP/RESET remain responsive. diff --git a/extensions/assistive_harness/cv/__init__.py b/extensions/assistive_harness/cv/__init__.py new file mode 100644 index 0000000..dec2042 --- /dev/null +++ b/extensions/assistive_harness/cv/__init__.py @@ -0,0 +1,14 @@ +from .base import CVObservation, FrameEnvelope, PerceptionProvider +from .noop import NoOpCVProvider, ShadowCVProvider +from .pipeline import CVPipeline +from .registry import CVProviderRegistry + +__all__ = [ + "CVObservation", + "CVPipeline", + "CVProviderRegistry", + "FrameEnvelope", + "NoOpCVProvider", + "PerceptionProvider", + "ShadowCVProvider", +] diff --git a/extensions/assistive_harness/cv/base.py b/extensions/assistive_harness/cv/base.py new file mode 100644 index 0000000..5c1e153 --- /dev/null +++ b/extensions/assistive_harness/cv/base.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Protocol + + +@dataclass(slots=True) +class FrameEnvelope: + """Transport-neutral frame handed from an Adapter to the CV worker.""" + + frame: bytes | None + frame_id: str + timestamp_ms: float + skill_id: str + slots: dict[str, Any] = field(default_factory=dict) + mode: str = "shadow" + provider_id: str = "noop" + + +@dataclass(slots=True) +class CVObservation: + frame_id: str + timestamp_ms: float + skill_id: str + provider: str + values: dict[str, Any] = field(default_factory=dict) + error: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +class PerceptionProvider(Protocol): + """Synchronous plugin contract; CVPipeline always calls it off-loop.""" + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + ... diff --git a/extensions/assistive_harness/cv/noop.py b/extensions/assistive_harness/cv/noop.py new file mode 100644 index 0000000..1770c62 --- /dev/null +++ b/extensions/assistive_harness/cv/noop.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import Any + +from .base import CVObservation, PerceptionProvider + + +class NoOpCVProvider: + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + del frame, slots + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider="noop", + values={}, + ) + + +class ShadowCVProvider: + """Failure-isolated observer. Its result is never a control decision.""" + + def __init__( + self, + provider: PerceptionProvider | None = None, + provider_id: str | None = None, + ): + self.provider = provider or NoOpCVProvider() + self.provider_id = provider_id or type(self.provider).__name__ + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + try: + result = self.provider.analyze( + frame, frame_id, timestamp_ms, skill_id, slots + ) + result.provider = f"shadow:{result.provider}" + return result + except Exception as exc: # shadow failures must not escape + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider=f"shadow:{self.provider_id}", + values={}, + error=f"{type(exc).__name__}: {exc}", + ) diff --git a/extensions/assistive_harness/cv/pipeline.py b/extensions/assistive_harness/cv/pipeline.py new file mode 100644 index 0000000..f17f9cf --- /dev/null +++ b/extensions/assistive_harness/cv/pipeline.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import asyncio +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Callable + +from .base import CVObservation, FrameEnvelope +from .registry import CVProviderRegistry + + +ObservationCallback = Callable[[CVObservation], None] +MetricCallback = Callable[[str, float], None] + + +class CVPipeline: + """Latest-frame-only worker that keeps CV inference off the control loop.""" + + def __init__( + self, + providers: CVProviderRegistry, + *, + on_observation: ObservationCallback, + on_metric: MetricCallback | None = None, + queue_size: int = 1, + inference_timeout_ms: float = 2000.0, + worker_name: str = "assistive-cv", + ) -> None: + if int(queue_size) != 1: + raise ValueError("CV V1 requires queue_size=1 (latest-frame-only)") + self.providers = providers + self.on_observation = on_observation + self.on_metric = on_metric + self.queue: asyncio.Queue[FrameEnvelope] = asyncio.Queue(maxsize=1) + self.inference_timeout_ms = max(1.0, float(inference_timeout_ms)) + self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix=worker_name) + self.worker_task: asyncio.Task[None] | None = None + self.closed = False + self.submitted_frames = 0 + self.processed_frames = 0 + self.dropped_frames = 0 + self.timeout_count = 0 + + def submit(self, envelope: FrameEnvelope) -> bool: + """Enqueue without awaiting inference; replace the oldest queued frame.""" + if self.closed or envelope.mode == "disabled": + return False + self._ensure_worker() + if self.queue.full(): + try: + self.queue.get_nowait() + self.queue.task_done() + self.dropped_frames += 1 + self._metric("cv_frames_dropped", 1.0) + except asyncio.QueueEmpty: + pass + self.queue.put_nowait(envelope) + self.submitted_frames += 1 + return True + + def snapshot(self) -> dict[str, int | bool]: + return { + "closed": self.closed, + "queued_frames": self.queue.qsize(), + "submitted_frames": self.submitted_frames, + "processed_frames": self.processed_frames, + "dropped_frames": self.dropped_frames, + "timeout_count": self.timeout_count, + } + + async def close(self) -> None: + if self.closed: + return + self.closed = True + if self.worker_task is not None: + self.worker_task.cancel() + await asyncio.gather(self.worker_task, return_exceptions=True) + self.executor.shutdown(wait=False, cancel_futures=True) + + def _ensure_worker(self) -> None: + if self.worker_task is None or self.worker_task.done(): + self.worker_task = asyncio.create_task(self._run()) + + async def _run(self) -> None: + loop = asyncio.get_running_loop() + while True: + envelope = await self.queue.get() + started = time.perf_counter() + future = loop.run_in_executor(self.executor, self.providers.analyze, envelope) + try: + observation = await asyncio.wait_for( + asyncio.shield(future), self.inference_timeout_ms / 1000.0 + ) + except asyncio.TimeoutError: + self.timeout_count += 1 + self._metric("cv_inference_timeout", 1.0) + self._emit( + CVObservation( + frame_id=envelope.frame_id, + timestamp_ms=envelope.timestamp_ms, + skill_id=envelope.skill_id, + provider=f"{envelope.mode}:{envelope.provider_id}", + values={ + "status": "timeout", + "timeout_ms": self.inference_timeout_ms, + }, + error=( + "TimeoutError: CV inference exceeded " + f"{self.inference_timeout_ms:.0f} ms" + ), + ) + ) + # Python cannot safely kill an in-flight native inference call. + # Keep this one-worker lane occupied until it exits so timeouts + # never create concurrent calls or an unbounded executor queue. + try: + await future + except Exception: + pass + except asyncio.CancelledError: + raise + except Exception as exc: + self._emit( + CVObservation( + frame_id=envelope.frame_id, + timestamp_ms=envelope.timestamp_ms, + skill_id=envelope.skill_id, + provider=f"{envelope.mode}:{envelope.provider_id}", + values={"status": "error"}, + error=f"{type(exc).__name__}: {exc}", + ) + ) + else: + observation.values.setdefault( + "pipeline_latency_ms", round((time.perf_counter() - started) * 1000, 1) + ) + self.processed_frames += 1 + self._metric( + "cv_pipeline_latency_ms", + float(observation.values["pipeline_latency_ms"]), + ) + self._emit(observation) + finally: + self.queue.task_done() + + def _emit(self, observation: CVObservation) -> None: + try: + self.on_observation(observation) + except Exception: + # Telemetry/consumer failures are also outside the control path. + pass + + def _metric(self, name: str, value: float) -> None: + if self.on_metric is None: + return + try: + self.on_metric(name, value) + except Exception: + pass diff --git a/extensions/assistive_harness/cv/registry.py b/extensions/assistive_harness/cv/registry.py new file mode 100644 index 0000000..dbe3b4e --- /dev/null +++ b/extensions/assistive_harness/cv/registry.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .base import CVObservation, FrameEnvelope, PerceptionProvider +from .noop import NoOpCVProvider, ShadowCVProvider + + +class CVProviderRegistry: + """Small explicit registry shared by browser and future device Adapters.""" + + def __init__(self, providers: dict[str, PerceptionProvider] | None = None): + self.providers: dict[str, PerceptionProvider] = { + "noop": NoOpCVProvider(), + **(providers or {}), + } + self._shadow = { + provider_id: ShadowCVProvider(provider, provider_id) + for provider_id, provider in self.providers.items() + } + + def register(self, provider_id: str, provider: PerceptionProvider) -> None: + normalized = provider_id.strip() + if not normalized: + raise ValueError("provider_id cannot be empty") + self.providers[normalized] = provider + self._shadow[normalized] = ShadowCVProvider(provider, normalized) + + def analyze(self, envelope: FrameEnvelope) -> CVObservation: + if envelope.mode != "shadow": + return CVObservation( + frame_id=envelope.frame_id, + timestamp_ms=envelope.timestamp_ms, + skill_id=envelope.skill_id, + provider=f"{envelope.mode}:{envelope.provider_id}", + values={"status": "skipped", "reason": "unsupported_cv_mode"}, + error=f"Unsupported cv_mode: {envelope.mode}", + ) + provider = self._shadow.get(envelope.provider_id) + if provider is None: + return CVObservation( + frame_id=envelope.frame_id, + timestamp_ms=envelope.timestamp_ms, + skill_id=envelope.skill_id, + provider=f"shadow:{envelope.provider_id}", + values={"status": "skipped", "reason": "unknown_provider"}, + error=f"Unknown CV provider: {envelope.provider_id}", + ) + return provider.analyze( + envelope.frame, + envelope.frame_id, + envelope.timestamp_ms, + envelope.skill_id, + envelope.slots, + ) + + +def build_provider_registry( + config: dict[str, Any], *, config_dir: Path +) -> CVProviderRegistry: + registry = CVProviderRegistry() + providers = dict((config.get("cv") or {}).get("providers") or {}) + for provider_id, raw_spec in providers.items(): + spec = dict(raw_spec or {}) + provider_type = str(spec.get("type") or provider_id) + if provider_type == "noop": + registry.register(str(provider_id), NoOpCVProvider()) + continue + if provider_type == "yolo_onnx": + from .yolo_onnx import YoloOnnxProvider + + raw_path = Path(str(spec.get("model_path") or "")) + model_path = raw_path if raw_path.is_absolute() else config_dir / raw_path + registry.register( + str(provider_id), + YoloOnnxProvider( + model_path.resolve(), + device=str(spec.get("device") or "cpu"), + confidence=float(spec.get("confidence", 0.25)), + image_size=int(spec.get("image_size", 640)), + target_aliases=dict(spec.get("target_aliases") or {}), + ), + ) + continue + raise ValueError( + f"Unsupported CV provider type {provider_type!r} for {provider_id!r}" + ) + return registry diff --git a/extensions/assistive_harness/cv/requirements-cv.txt b/extensions/assistive_harness/cv/requirements-cv.txt new file mode 100644 index 0000000..72c2319 --- /dev/null +++ b/extensions/assistive_harness/cv/requirements-cv.txt @@ -0,0 +1,2 @@ +opencv-python-headless +onnxruntime==1.21.0 diff --git a/extensions/assistive_harness/cv/yolo_onnx.py b/extensions/assistive_harness/cv/yolo_onnx.py new file mode 100644 index 0000000..01ca1c8 --- /dev/null +++ b/extensions/assistive_harness/cv/yolo_onnx.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import threading +import time +from pathlib import Path +from typing import Any + +import numpy as np + +from .base import CVObservation + + +COCO_NAMES = ( + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", + "truck", "boat", "traffic light", "fire hydrant", "stop sign", + "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", + "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", + "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", + "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", + "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", + "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", + "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", + "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", + "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", + "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", + "teddy bear", "hair drier", "toothbrush", +) + +DEFAULT_TARGET_ALIASES = { + "手机": "cell phone", "电话": "cell phone", "智能手机": "cell phone", + "phone": "cell phone", "cellphone": "cell phone", "cell phone": "cell phone", + "书": "book", "书本": "book", "图书": "book", "book": "book", + "杯子": "cup", "水杯": "cup", "茶杯": "cup", "cup": "cup", + "瓶子": "bottle", "水瓶": "bottle", "bottle": "bottle", + "椅子": "chair", "chair": "chair", "人": "person", "行人": "person", + "person": "person", "电脑": "laptop", "笔记本电脑": "laptop", + "laptop": "laptop", "遥控器": "remote", "remote": "remote", + "键盘": "keyboard", "keyboard": "keyboard", "鼠标": "mouse", + "mouse": "mouse", "背包": "backpack", "书包": "backpack", + "backpack": "backpack", +} + + +class YoloOnnxProvider: + """CPU-first YOLO ONNX reference plugin with no control-side effects.""" + + def __init__( + self, + model_path: Path, + *, + device: str = "cpu", + confidence: float = 0.25, + image_size: int = 640, + target_aliases: dict[str, str] | None = None, + ) -> None: + if device.lower() != "cpu": + raise ValueError("CV V1 only accepts device=cpu") + self.model_path = Path(model_path).resolve() + self.device = "cpu" + self.confidence = float(confidence) + self.image_size = int(image_size) + self.target_aliases = { + **DEFAULT_TARGET_ALIASES, + **{str(k).lower(): str(v) for k, v in (target_aliases or {}).items()}, + } + self._session: Any = None + self._input_name: str | None = None + self._cv2: Any = None + self._lock = threading.Lock() + + def analyze( + self, + frame: bytes | None, + frame_id: str, + timestamp_ms: float, + skill_id: str, + slots: dict[str, Any], + ) -> CVObservation: + if not frame: + raise ValueError("YOLO requires a non-empty JPEG frame") + self._load() + target = str(slots.get("target") or "").strip() + target_label = self._resolve_target(target) if target else None + if target and target_label is None: + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider="yolo_onnx", + values={ + "status": "skipped", + "reason": "unsupported_target", + "target": target, + }, + ) + + encoded = np.frombuffer(frame, dtype=np.uint8) + image = self._cv2.imdecode(encoded, self._cv2.IMREAD_COLOR) + if image is None: + raise ValueError("YOLO could not decode the JPEG frame") + height, width = image.shape[:2] + started = time.perf_counter() + tensor, scale, pad_x, pad_y = self._prepare_input(image) + with self._lock: + output = self._session.run(None, {self._input_name: tensor})[0] + latency_ms = (time.perf_counter() - started) * 1000.0 + if output.ndim != 3 or output.shape[0] != 1 or output.shape[2] != 6: + raise ValueError(f"Unexpected YOLO output shape: {output.shape}") + + target_id = COCO_NAMES.index(target_label) if target_label else None + detections: list[dict[str, Any]] = [] + for row in output[0]: + x1, y1, x2, y2, confidence, raw_class = row + class_id = round(float(raw_class)) + if float(confidence) < self.confidence or not 0 <= class_id < len(COCO_NAMES): + continue + if target_id is not None and class_id != target_id: + continue + left = max(0.0, min(width, (float(x1) - pad_x) / scale)) + top = max(0.0, min(height, (float(y1) - pad_y) / scale)) + right = max(0.0, min(width, (float(x2) - pad_x) / scale)) + bottom = max(0.0, min(height, (float(y2) - pad_y) / scale)) + if right <= left or bottom <= top: + continue + center_x = ((left + right) / 2.0) / width + center_y = ((top + bottom) / 2.0) / height + detections.append( + { + "label": COCO_NAMES[class_id], + "confidence": round(float(confidence), 4), + "bbox_xyxy": [round(left), round(top), round(right), round(bottom)], + "bbox_normalized": [ + round(left / width, 4), round(top / height, 4), + round(right / width, 4), round(bottom / height, 4), + ], + "center_normalized": [round(center_x, 4), round(center_y, 4)], + "position": self._position(center_x, center_y), + } + ) + detections.sort(key=lambda item: float(item["confidence"]), reverse=True) + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider="yolo_onnx", + values={ + "status": "ok", + "found": bool(detections), + "target": target or None, + "canonical_label": target_label, + "best_detection": detections[0] if detections else None, + "detections": detections, + "image": {"width": width, "height": height}, + "latency_ms": round(latency_ms, 1), + "frame_age_ms": round(max(0.0, time.time() * 1000.0 - timestamp_ms), 1), + "model": self.model_path.name, + "device": self.device, + }, + ) + + def _load(self) -> None: + if self._session is not None: + return + if not self.model_path.is_file(): + raise FileNotFoundError(f"YOLO weights not found: {self.model_path}") + import cv2 + import onnxruntime as ort + + session = ort.InferenceSession( + str(self.model_path), providers=["CPUExecutionProvider"] + ) + model_input = session.get_inputs()[0] + if model_input.shape[-2:] != [self.image_size, self.image_size]: + raise ValueError( + f"YOLO input is {model_input.shape[-2:]}, expected " + f"[{self.image_size}, {self.image_size}]" + ) + self._cv2 = cv2 + self._session = session + self._input_name = model_input.name + + def _prepare_input( + self, image: np.ndarray + ) -> tuple[np.ndarray, float, float, float]: + height, width = image.shape[:2] + scale = min(self.image_size / width, self.image_size / height) + resized_width, resized_height = round(width * scale), round(height * scale) + resized = self._cv2.resize(image, (resized_width, resized_height)) + pad_x = (self.image_size - resized_width) / 2.0 + pad_y = (self.image_size - resized_height) / 2.0 + left, top = round(pad_x - 0.1), round(pad_y - 0.1) + right = self.image_size - resized_width - left + bottom = self.image_size - resized_height - top + padded = self._cv2.copyMakeBorder( + resized, top, bottom, left, right, + self._cv2.BORDER_CONSTANT, value=(114, 114, 114), + ) + rgb = self._cv2.cvtColor(padded, self._cv2.COLOR_BGR2RGB) + tensor = np.ascontiguousarray(rgb.transpose(2, 0, 1), dtype=np.float32) + return np.expand_dims(tensor / 255.0, axis=0), scale, float(left), float(top) + + def _resolve_target(self, target: str) -> str | None: + normalized = target.lower().replace(" ", "") + for alias in sorted(self.target_aliases, key=len, reverse=True): + if alias.lower().replace(" ", "") in normalized: + label = self.target_aliases[alias] + return label if label in COCO_NAMES else None + return None + + @staticmethod + def _position(center_x: float, center_y: float) -> str: + horizontal = "左" if center_x < 1 / 3 else "右" if center_x > 2 / 3 else "中" + vertical = "上" if center_y < 1 / 3 else "下" if center_y > 2 / 3 else "中" + return { + ("左", "上"): "左上方", ("中", "上"): "正上方", + ("右", "上"): "右上方", ("左", "中"): "左侧", + ("中", "中"): "中央", ("右", "中"): "右侧", + ("左", "下"): "左下方", ("中", "下"): "正下方", + ("右", "下"): "右下方", + }[(horizontal, vertical)] diff --git a/extensions/assistive_harness/download_modelscope_model.py b/extensions/assistive_harness/download_modelscope_model.py new file mode 100644 index 0000000..3d51a37 --- /dev/null +++ b/extensions/assistive_harness/download_modelscope_model.py @@ -0,0 +1,69 @@ +"""Explicitly download a ModelScope model for the local Harness. + +The runtime itself deliberately accepts only a local ``--model-path``. Keeping +download and inference as separate commands makes network access visible and +keeps device runs reproducible. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +DEFAULT_MODEL_ID = ( + "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online" +) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Download a ModelScope model and print its local directory" + ) + parser.add_argument("--model-id", default=DEFAULT_MODEL_ID) + parser.add_argument("--revision", default=None) + location = parser.add_mutually_exclusive_group() + location.add_argument( + "--cache-dir", + default=None, + help="ModelScope cache root; the model ID is appended below it", + ) + location.add_argument( + "--local-dir", + default=None, + help="exact destination directory for this model", + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + try: + from modelscope.hub.snapshot_download import snapshot_download + except ImportError as exc: + raise SystemExit( + "ModelScope is not installed. Run: " + "python -m pip install -r extensions/assistive_harness/requirements.txt" + ) from exc + + kwargs: dict[str, str] = {} + if args.revision: + kwargs["revision"] = args.revision + if args.cache_dir: + kwargs["cache_dir"] = str(Path(args.cache_dir).expanduser().resolve()) + if args.local_dir: + kwargs["local_dir"] = str(Path(args.local_dir).expanduser().resolve()) + + print(f"Downloading ModelScope model: {args.model_id}", flush=True) + model_path = Path(snapshot_download(args.model_id, **kwargs)).resolve() + print(f"MODEL_PATH={model_path}") + print("Start the Harness with:") + print( + f'"{sys.executable}" -m extensions.assistive_harness.server ' + f'--enabled --model-path "{model_path}" --port 8021' + ) + + +if __name__ == "__main__": + main() diff --git a/extensions/assistive_harness/echo_guard.py b/extensions/assistive_harness/echo_guard.py new file mode 100644 index 0000000..fc3d25e --- /dev/null +++ b/extensions/assistive_harness/echo_guard.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import time +from collections import deque +from dataclasses import dataclass +from difflib import SequenceMatcher + +from .router import normalize_text +from .schemas import ControlIntent + + +@dataclass(frozen=True, slots=True) +class EchoDecision: + allow: bool + reason: str + similarity: float = 0.0 + + +class EchoGuard: + def __init__(self, window_ms: int = 20_000, similarity_threshold: float = 0.86): + self.window_ms = int(window_ms) + self.similarity_threshold = float(similarity_threshold) + self._model_text: deque[tuple[float, str]] = deque() + self.ai_speaking = False + + def note_model_text(self, text: str, at_ms: float | None = None) -> None: + normalized = normalize_text(text) + if not normalized: + return + now = at_ms if at_ms is not None else time.time() * 1000 + self._model_text.append((now, normalized)) + self._prune(now) + + def set_ai_speaking(self, active: bool) -> None: + self.ai_speaking = bool(active) + + def _prune(self, now_ms: float) -> None: + cutoff = now_ms - self.window_ms + while self._model_text and self._model_text[0][0] < cutoff: + self._model_text.popleft() + + def evaluate( + self, + utterance: str, + intent: ControlIntent, + at_ms: float | None = None, + ) -> EchoDecision: + now = at_ms if at_ms is not None else time.time() * 1000 + self._prune(now) + normalized = normalize_text(utterance) + if len(normalized) < 2: + return EchoDecision(False, "too_short") + + best = 0.0 + for _, model_text in self._model_text: + if normalized in model_text or model_text in normalized: + best = 1.0 + break + best = max(best, SequenceMatcher(None, normalized, model_text).ratio()) + if best >= self.similarity_threshold: + return EchoDecision(False, "matches_recent_model_echo", best) + + if self.ai_speaking and intent not in { + ControlIntent.STOP_SPEECH, + ControlIntent.RESUME_SPEECH, + ControlIntent.RESET_SESSION, + ControlIntent.CANCEL_SKILL, + ControlIntent.RETURN_TO_CHAT, + # A positively routed Skill command is also a control-plane + # interruption. Recent-model-text similarity is checked above, + # so model echo is still rejected before this exception applies. + ControlIntent.ACTIVATE_SKILL, + }: + return EchoDecision(False, "ordinary_skill_suppressed_while_ai_speaking", best) + return EchoDecision(True, "allowed", best) diff --git a/extensions/assistive_harness/model_log.py b/extensions/assistive_harness/model_log.py new file mode 100644 index 0000000..43045e4 --- /dev/null +++ b/extensions/assistive_harness/model_log.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(slots=True) +class ModelTurnAccumulator: + """Aggregate streamed Gateway fragments into auditable model turns.""" + + turn_index: int = 0 + _key: tuple[Any, ...] | None = None + _text: list[str] = field(default_factory=list) + _audio_ms: float = 0.0 + _metadata: dict[str, Any] = field(default_factory=dict) + + def _flush(self) -> dict[str, Any] | None: + if self._key is None: + return None + self.turn_index += 1 + record = { + "type": "model.turn", + "turn_index": self.turn_index, + **self._metadata, + "text": "".join(self._text), + "audio_ms": round(self._audio_ms, 1), + } + self._key = None + self._text.clear() + self._audio_ms = 0.0 + self._metadata = {} + return record + + def feed(self, message: dict[str, Any]) -> list[dict[str, Any]]: + state = str(message.get("state") or "unknown") + key = ( + message.get("session_id"), + int(message.get("generation") or 0), + str(message.get("skill_id") or ""), + state, + ) + completed: list[dict[str, Any]] = [] + if self._key is not None and key != self._key: + record = self._flush() + if record is not None: + completed.append(record) + + text = str(message.get("text") or "") + audio_ms = float(message.get("audio_ms") or 0.0) + end_of_turn = bool(message.get("end_of_turn")) + if self._key is None and (text or audio_ms > 0): + self._key = key + self._metadata = { + "role": "assistant" if state == "speak" else "listen", + "state": state, + "session_id": message.get("session_id"), + "generation": int(message.get("generation") or 0), + "skill_id": str(message.get("skill_id") or ""), + "slots": dict(message.get("slots") or {}), + } + if self._key is not None: + if text: + self._text.append(text) + self._audio_ms += max(0.0, audio_ms) + if end_of_turn: + record = self._flush() + if record is not None: + completed.append(record) + return completed + + def flush(self) -> dict[str, Any] | None: + return self._flush() diff --git a/extensions/assistive_harness/phase_b/README.md b/extensions/assistive_harness/phase_b/README.md new file mode 100644 index 0000000..b7ffe10 --- /dev/null +++ b/extensions/assistive_harness/phase_b/README.md @@ -0,0 +1,114 @@ +# Rokid Phase B Adapter + +This adapter replaces the Phase A browser transport while preserving the +frozen Harness policy and the existing MiniCPM backend: + +```text +Rokid APK --JPEG/PCM over Wi-Fi--> Phase B Adapter :18080 + |-- audio.mirror/frame.shadow --> Harness :8021 + |-- audio_chunk + JPEG --------> Gateway :8040 + `-- MiniCPM audio -------------> PC speaker +``` + +The backend remains `llama.cpp-omni -> Worker -> Gateway :8040 -> MiniCPM-o +4.5`. The old ESP32 and Rokid bridge scripts remain reference implementations; +`demo_rokid_phase_b_harness.py` is the Phase B entry point. + +## Frozen control contract + +- `STOP`: block and flush PC playback immediately. Device PCM continues to + reach Harness ASR and Gateway with `force_listen=true`. +- `RESUME`: release the playback gate without replacing the Session. +- `RESET`: stop the old Gateway Session with light cleanup, increment the + generation, create a new Session using the idle prompt, and reject stale + output. After `restart_complete`, the PC plays a short two-note ready cue; + initial startup stays silent. Use `--no-session-ready-chime` to disable it, + or `--session-ready-chime-volume` (default `0.32`) to tune it. +- Skill activation, cancellation, and return-to-chat use the same replacement + flow with the prompt and slots supplied by Harness. +- After a Skill Session reaches `restart_complete`, Harness injects the + original one-shot task into that new Session. This makes read-text and + experimental obstacle activation answer once instead of only changing the + system prompt. +- Rokid PCM is repacketized into 100 ms float32 `audio.mirror` frames, matching + Phase A. A bounded drop-oldest queue prevents device capture from being + blocked by Gateway latency. +- PC speaker queue depth plus `--playback-echo-tail-s` (default `0.80`) keeps + EchoGuard active through buffered playback and its short acoustic tail. + +## Prerequisites + +Run the existing Worker/Gateway on port 8040 and the Assistive Harness on port +8021. Install optional adapter dependencies from `requirements-phase-b.txt` in +the same Python environment. + +The current Rokid APK must target the host computer on port 18080 and provide: + +- `POST /rokid/image` with raw JPEG bytes; +- `WS /rokid/audio` with 16 kHz mono signed PCM16 little-endian packets. + +## Start + +From the repository root: + +```powershell +python .\demo_rokid_phase_b_harness.py ` + --gateway localhost:8040 ` + --harness-url ws://127.0.0.1:8021/ws/control ` + --input-gain 12 ` + --image-rotate-cw 270 ` + --session-ready-chime-volume 0.32 ` + --playback-echo-tail-s 0.80 +``` + +The local Gateway currently uses plain `ws://`. Use `--gateway-tls` only when +the deployed Gateway endpoint actually serves `wss://`. First-round model audio +plays on the computer; `--no-play` disables playback for diagnostics. + +The Rokid SDK PCM observed on the current device is materially quieter than the +browser microphone signal. `--input-gain 12` is the initial device calibration; +gain is applied before both Harness and Gateway and saturates instead of wrapping +PCM16. Recalibrate from real-device RMS logs rather than lowering the shared +Phase A VAD threshold. + +The ready cue briefly suppresses Rokid PCM forwarding for the cue plus a short +guard interval so the laptop speaker notification is not immediately recycled +into Harness or MiniCPM-o. + +Streaming model text is aggregated by Session, generation and end-of-turn. It +is printed as `[AssistiveHarness][MODEL]` and written under the current run: + +- `model_events.jsonl` for structured records; +- `model_transcript.txt` for quick human review. + +This is generated model text, not ASR over the final TTS waveform. Keep a +session recording when exact spoken audio must be audited. + +Health and the normalized latest frame are available at: + +- `GET http://127.0.0.1:18080/health` +- `GET http://127.0.0.1:18080/capture` + +Healthy input is approximately 25 audio packets/s and 1 JPEG/s, with one audio +client, zero sustained queue drops, `harness_connected=true`, and +`gateway_status=running`. `device_input_ready=true` confirms that at least one +PCM packet or JPEG has reached this process. If `[ROKID][NO_INPUT]` appears, +restart Sensor Mode on the glasses (`STOP -> RUN`) after the PC listener is up; +an APK WebSocket disconnected by a PC restart may not reconnect by itself. + +Stop the adapter with `Ctrl+C` (or `Ctrl+Break` on Windows). The HTTP listener, +Rokid WebSocket handler, Gateway Session, Harness connection, and PC speaker are +closed with bounded waits. A 10-second process watchdog is the final fallback, +so a native audio cleanup stall cannot leave port 18080 occupied indefinitely. + +## First device acceptance order + +1. Ordinary speech produces a MiniCPM response on the PC speaker. +2. While it speaks, say `停一下`; playback must stop and Harness must ACK + `stop_speech`. +3. Say `恢复对话`; the same Session resumes. +4. Say `重新开始`; the Session ID and generation must change. +5. Activate `找物`, `识字`, `场景描述`, and supervised experimental `避障`; + each switch must complete with a new Session and no old-generation output. + +Experimental obstacle output is not a navigation or safety guarantee. diff --git a/extensions/assistive_harness/phase_b/__init__.py b/extensions/assistive_harness/phase_b/__init__.py new file mode 100644 index 0000000..80d861b --- /dev/null +++ b/extensions/assistive_harness/phase_b/__init__.py @@ -0,0 +1,5 @@ +"""Device adapters for the Phase B glasses runtime.""" + +from .rokid_runtime import PhaseBRokidRuntime, RokidRuntimeConfig + +__all__ = ["PhaseBRokidRuntime", "RokidRuntimeConfig"] diff --git a/extensions/assistive_harness/phase_b/requirements-phase-b.txt b/extensions/assistive_harness/phase_b/requirements-phase-b.txt new file mode 100644 index 0000000..88c2364 --- /dev/null +++ b/extensions/assistive_harness/phase_b/requirements-phase-b.txt @@ -0,0 +1,3 @@ +aiohttp>=3.13 +Pillow>=10 +sounddevice>=0.5 diff --git a/extensions/assistive_harness/phase_b/rokid_runtime.py b/extensions/assistive_harness/phase_b/rokid_runtime.py new file mode 100644 index 0000000..52885d8 --- /dev/null +++ b/extensions/assistive_harness/phase_b/rokid_runtime.py @@ -0,0 +1,1652 @@ +from __future__ import annotations + +import argparse +import asyncio +import base64 +import io +import json +import logging +import os +import signal +import ssl +import threading +import time +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Awaitable, Callable, Protocol + +import aiohttp +from aiohttp import web +import numpy as np + +from ..registry import SkillRegistry + +try: + from PIL import Image +except Exception: # pragma: no cover - optional at import time + Image = None + + +LOG = logging.getLogger("assistive_harness.phase_b.rokid") +SAMPLE_RATE_IN = 16_000 +SAMPLE_RATE_OUT = 24_000 +PCM16_WIDTH = 2 +CTRL_C_HARD_EXIT_S = 10.0 + + +def now_ms() -> float: + return time.time() * 1000.0 + + +class CtrlCExitWatchdog: + """Guarantee that a Windows Ctrl+C cannot leave the Rokid port behind.""" + + def __init__(self, hard_exit_s: float = CTRL_C_HARD_EXIT_S): + self.hard_exit_s = max(1.0, float(hard_exit_s)) + self._armed = False + self._lock = threading.Lock() + + @property + def armed(self) -> bool: + return self._armed + + def arm(self) -> None: + with self._lock: + if self._armed: + return + self._armed = True + LOG.info( + "Console interrupt received; closing Rokid listener and runtime " + "(hard cutoff %.1fs)", + self.hard_exit_s, + ) + threading.Thread( + target=self._force_exit_after_deadline, + name="rokid-ctrl-c-watchdog", + daemon=True, + ).start() + + def _force_exit_after_deadline(self) -> None: + time.sleep(self.hard_exit_s) + LOG.error( + "Rokid shutdown exceeded %.1fs; forcing process exit so port 18080 " + "cannot remain occupied", + self.hard_exit_s, + ) + logging.shutdown() + os._exit(130) + + +def pcm16le_to_float32(raw: bytes) -> np.ndarray: + if len(raw) % PCM16_WIDTH: + raw = raw[:-1] + if not raw: + return np.zeros(0, dtype=np.float32) + return ( + np.frombuffer(raw, dtype=" bytes: + """Apply device-specific gain with saturation while preserving PCM16 LE.""" + if gain <= 0: + raise ValueError("input gain must be greater than zero") + if gain == 1.0 or not raw: + return raw + samples = np.frombuffer(raw, dtype=" np.ndarray: + """Return a short, non-speech two-note cue for restart_complete.""" + + amplitude = min(1.0, max(0.0, float(amplitude))) + + def tone(frequency_hz: float, duration_s: float) -> np.ndarray: + count = max(2, int(sample_rate * duration_s)) + phase = np.arange(count, dtype=np.float32) / float(sample_rate) + envelope = np.sin(np.linspace(0.0, np.pi, count, dtype=np.float32)) ** 2 + return ( + amplitude + * envelope + * np.sin(2.0 * np.pi * frequency_hz * phase) + ).astype(np.float32) + + gap = np.zeros(max(1, int(sample_rate * 0.030)), dtype=np.float32) + return np.concatenate((tone(880.0, 0.080), gap, tone(1174.66, 0.105))) + + +def float32_to_base64(samples: np.ndarray) -> str: + return base64.b64encode( + samples.astype(np.float32, copy=False).tobytes() + ).decode("ascii") + + +def same_slots(left: dict[str, Any], right: dict[str, Any]) -> bool: + return {str(key): str(value) for key, value in left.items()} == { + str(key): str(value) for key, value in right.items() + } + + +def rotate_jpeg_clockwise(jpeg: bytes, degrees: int, quality: int = 95) -> bytes: + degrees %= 360 + if degrees == 0 or Image is None: + return jpeg + methods = { + 90: Image.Transpose.ROTATE_270, + 180: Image.Transpose.ROTATE_180, + 270: Image.Transpose.ROTATE_90, + } + method = methods.get(degrees) + if method is None: + raise ValueError("image rotation must be 0, 90, 180, or 270") + with Image.open(io.BytesIO(jpeg)) as image: + output = io.BytesIO() + image.convert("RGB").transpose(method).save( + output, format="JPEG", quality=quality + ) + return output.getvalue() + + +class DropOldestAudioQueue: + """Bounded device-audio queue that always preserves the newest packets.""" + + def __init__(self, max_packets: int): + self._queue: asyncio.Queue[bytes] = asyncio.Queue(maxsize=max(1, max_packets)) + self.dropped_packets = 0 + + @property + def size(self) -> int: + return self._queue.qsize() + + def put_nowait(self, raw: bytes) -> None: + if self._queue.full(): + try: + self._queue.get_nowait() + self.dropped_packets += 1 + except asyncio.QueueEmpty: + pass + self._queue.put_nowait(raw) + + async def get(self, timeout_s: float) -> bytes | None: + try: + return await asyncio.wait_for(self._queue.get(), timeout=timeout_s) + except asyncio.TimeoutError: + return None + + def clear(self) -> int: + cleared = 0 + while True: + try: + self._queue.get_nowait() + cleared += 1 + except asyncio.QueueEmpty: + return cleared + + +class AudioMirrorChunker: + """Repacketize Rokid PCM into the same 100 ms frames used in Phase A.""" + + def __init__(self, samples_per_frame: int = 1_600): + self.target_bytes = max(1, samples_per_frame) * PCM16_WIDTH + self._carry = bytearray() + + def feed(self, raw: bytes) -> list[np.ndarray]: + if len(raw) % PCM16_WIDTH: + raw = raw[:-1] + self._carry.extend(raw) + frames: list[np.ndarray] = [] + while len(self._carry) >= self.target_bytes: + frame = bytes(self._carry[: self.target_bytes]) + del self._carry[: self.target_bytes] + frames.append(pcm16le_to_float32(frame)) + return frames + + def clear(self) -> None: + self._carry.clear() + + +@dataclass(slots=True) +class LatestFrame: + jpeg: bytes | None = None + timestamp_ms: float = 0.0 + sequence: int = 0 + + def set(self, jpeg: bytes, timestamp_ms: float) -> None: + self.jpeg = jpeg + self.timestamp_ms = timestamp_ms + self.sequence += 1 + + +@dataclass(slots=True) +class OutputGate: + generation: int = 0 + current_skill: str = "idle_chat" + current_slots: dict[str, Any] | None = None + speech_hold_active: bool = False + drop_output_until_listen: bool = False + restart_in_progress: bool = False + dropped_old_text: int = 0 + dropped_old_audio: int = 0 + stop_count: int = 0 + local_cue_mute_until_mono: float = 0.0 + + def __post_init__(self) -> None: + if self.current_slots is None: + self.current_slots = {} + + def stop(self) -> None: + self.stop_count += 1 + self.speech_hold_active = True + self.drop_output_until_listen = True + + def resume(self) -> None: + self.speech_hold_active = False + self.drop_output_until_listen = False + + def replacement_ready(self) -> None: + self.speech_hold_active = False + self.drop_output_until_listen = False + self.restart_in_progress = False + + +class SpeakerSink(Protocol): + async def start(self) -> None: ... + + async def enqueue(self, pcm: np.ndarray, generation: int) -> None: ... + + async def block_and_flush(self) -> None: ... + + async def resume(self) -> None: ... + + async def close(self) -> None: ... + + def pending_ms(self) -> float: ... + + +class NullSpeaker: + async def start(self) -> None: + return None + + async def enqueue(self, pcm: np.ndarray, generation: int) -> None: + return None + + async def block_and_flush(self) -> None: + return None + + async def resume(self) -> None: + return None + + async def close(self) -> None: + return None + + def pending_ms(self) -> float: + return 0.0 + + +class PCSpeaker: + """Interruptible 24 kHz PC playback with a short write quantum.""" + + def __init__(self, block_ms: int = 50): + self.block_samples = max(240, int(SAMPLE_RATE_OUT * block_ms / 1000)) + self.queue: asyncio.Queue[tuple[int, int, np.ndarray]] = asyncio.Queue(maxsize=32) + self.blocked = False + self.epoch = 0 + self._stream: Any = None + self._worker: asyncio.Task[None] | None = None + self._io_lock = asyncio.Lock() + self._pending_samples = 0 + + def _open_stream(self) -> None: + import sounddevice as sd # type: ignore + + stream = sd.OutputStream( + samplerate=SAMPLE_RATE_OUT, + channels=1, + dtype="float32", + blocksize=self.block_samples, + latency="low", + ) + stream.start() + self._stream = stream + + def _close_stream(self) -> None: + stream = self._stream + self._stream = None + if stream is None: + return + try: + stream.abort() + finally: + stream.close() + + async def start(self) -> None: + try: + await asyncio.to_thread(self._open_stream) + self._worker = asyncio.create_task(self._play_loop()) + LOG.info("PC speaker output ready") + except Exception as exc: # pragma: no cover - hardware dependent + self._stream = None + LOG.warning("PC speaker disabled: %s", exc) + + async def enqueue(self, pcm: np.ndarray, generation: int) -> None: + if self._stream is None or self.blocked or pcm.size == 0: + return + if self.queue.full(): + try: + _epoch, _generation, dropped = self.queue.get_nowait() + self._pending_samples = max( + 0, self._pending_samples - int(dropped.size) + ) + except asyncio.QueueEmpty: + pass + copied = pcm.astype(np.float32, copy=True) + self._pending_samples += int(copied.size) + self.queue.put_nowait((self.epoch, generation, copied)) + + async def _play_loop(self) -> None: + while True: + epoch, _generation, pcm = await self.queue.get() + try: + for offset in range(0, pcm.size, self.block_samples): + if self.blocked or epoch != self.epoch or self._stream is None: + break + chunk = pcm[offset : offset + self.block_samples].reshape(-1, 1) + stream = self._stream + try: + async with self._io_lock: + if self.blocked or epoch != self.epoch or self._stream is not stream: + break + await asyncio.to_thread(stream.write, chunk) + except asyncio.CancelledError: + raise + except Exception as exc: # pragma: no cover - hardware dependent + LOG.warning("speaker write failed: %s", exc) + break + finally: + self._pending_samples = max( + 0, self._pending_samples - int(pcm.size) + ) + + async def block_and_flush(self) -> None: + self.blocked = True + self.epoch += 1 + self._pending_samples = 0 + while True: + try: + self.queue.get_nowait() + except asyncio.QueueEmpty: + break + if self._stream is not None: + try: + # Windows MME may reject start() immediately after abort() while + # the previous buffer is still retiring. Close the device here + # and create a fresh stream only when output is resumed. + async with self._io_lock: + await asyncio.to_thread(self._close_stream) + except Exception as exc: # pragma: no cover - hardware dependent + LOG.warning("speaker flush failed: %s", exc) + + async def resume(self) -> None: + if self._stream is None: + try: + async with self._io_lock: + if self._stream is None: + await asyncio.to_thread(self._open_stream) + except Exception as exc: # pragma: no cover - hardware dependent + LOG.warning("speaker resume failed: %s", exc) + self.blocked = False + + async def close(self) -> None: + self.blocked = True + self._pending_samples = 0 + if self._worker: + self._worker.cancel() + await asyncio.gather(self._worker, return_exceptions=True) + if self._stream is not None: + try: + async with self._io_lock: + await asyncio.to_thread(self._close_stream) + except Exception: + pass + + def pending_ms(self) -> float: + return self._pending_samples * 1000.0 / SAMPLE_RATE_OUT + + +@dataclass(slots=True) +class RokidRuntimeConfig: + host: str = "0.0.0.0" + port: int = 18_080 + gateway: str = "localhost:8040" + gateway_tls: bool = False + harness_url: str = "ws://127.0.0.1:8021/ws/control" + harness_client_id: str = "rokid-phase-b" + skills_config: str = "" + cleanup_mode: str = "light" + chunk_ms: int = 1_000 + force_listen_count: int = 3 + max_new_speak_tokens_per_chunk: int = 20 + length_penalty: float = 1.1 + max_slice_nums: int = 1 + audio_queue_packets: int = 96 + input_gain: float = 12.0 + image_rotate_cw: int = 270 + image_jpeg_quality: int = 95 + image_resend_s: float = 0.5 + image_max_age_s: float = 30.0 + prepare_timeout_s: float = 120.0 + close_timeout_s: float = 2.0 + reconnect_s: float = 1.5 + play_audio: bool = True + session_ready_chime: bool = True + session_ready_chime_volume: float = 0.32 + session_ready_chime_feedback_guard_s: float = 0.65 + playback_echo_tail_s: float = 0.80 + + +@dataclass(slots=True) +class SessionSpec: + generation: int + skill_id: str + slots: dict[str, Any] + system_prompt: str + + +class SessionTelemetry(Protocol): + connected: bool + + async def send(self, payload: dict[str, Any]) -> None: ... + + +class GatewayDuplexSession: + """One generation-fenced connection to the existing MiniCPM Gateway.""" + + def __init__( + self, + config: RokidRuntimeConfig, + spec: SessionSpec, + audio_queue: DropOldestAudioQueue, + latest_frame: LatestFrame, + gate: OutputGate, + on_result: Callable[["GatewayDuplexSession", dict[str, Any]], Awaitable[None]], + ): + self.config = config + self.spec = spec + self.audio_queue = audio_queue + self.latest_frame = latest_frame + self.gate = gate + self.on_result = on_result + self.session_id = ( + f"omni_rokid_pb_g{spec.generation}_{int(time.time())}_{uuid.uuid4().hex[:6]}" + ) + self.status = "created" + self.last_error = "" + self.ws: aiohttp.ClientWebSocketResponse | None = None + self._task: asyncio.Task[None] | None = None + self._ready: asyncio.Future[None] | None = None + self._stopped = asyncio.Event() + self._closing = False + self._send_lock = asyncio.Lock() + + async def _send_json(self, payload: dict[str, Any]) -> None: + ws = self.ws + if ws is None or ws.closed: + raise RuntimeError("gateway session is not connected") + async with self._send_lock: + await ws.send_json(payload) + + def _ssl_context(self) -> ssl.SSLContext | None: + if not self.config.gateway_tls: + return None + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + + async def start(self) -> None: + loop = asyncio.get_running_loop() + self._ready = loop.create_future() + self._ready.add_done_callback( + lambda future: None if future.cancelled() else future.exception() + ) + self._task = asyncio.create_task(self._run()) + await asyncio.wait_for( + asyncio.shield(self._ready), timeout=self.config.prepare_timeout_s + ) + + async def _run(self) -> None: + scheme = "wss" if self.config.gateway_tls else "ws" + url = f"{scheme}://{self.config.gateway}/ws/duplex/{self.session_id}" + self.status = "connecting" + LOG.info("[GW] connecting generation=%d %s", self.spec.generation, url) + try: + async with aiohttp.ClientSession() as client: + async with client.ws_connect( + url, + heartbeat=30, + max_msg_size=0, + ssl=self._ssl_context(), + ) as ws: + self.ws = ws + await self._prepare(ws) + self.status = "running" + if self._ready and not self._ready.done(): + self._ready.set_result(None) + send_task = asyncio.create_task(self._send_loop(ws)) + receive_task = asyncio.create_task(self._receive_loop(ws)) + done, pending = await asyncio.wait( + (send_task, receive_task), + return_when=asyncio.FIRST_COMPLETED, + ) + for task in pending: + task.cancel() + await asyncio.gather(*done, *pending, return_exceptions=True) + except asyncio.CancelledError: + raise + except Exception as exc: + self.status = "error" + self.last_error = str(exc) + if self._ready and not self._ready.done(): + self._ready.set_exception(exc) + if not self._closing: + LOG.warning("[GW] session %s failed: %s", self.session_id, exc) + finally: + self.ws = None + self._stopped.set() + if self.status != "error": + self.status = "stopped" + + async def _prepare(self, ws: aiohttp.ClientWebSocketResponse) -> None: + self.status = "queued" + while True: + message = await ws.receive() + if message.type != aiohttp.WSMsgType.TEXT: + if message.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR): + raise RuntimeError("gateway closed while queued") + continue + payload = json.loads(message.data) + message_type = payload.get("type") + if message_type == "queue_done": + break + if message_type == "error": + raise RuntimeError(payload.get("error") or "gateway queue error") + if message_type in {"queued", "queue_update"}: + LOG.info( + "[GW] queue position=%s eta=%s", + payload.get("position"), + payload.get("estimated_wait_s"), + ) + + self.status = "preparing" + await self._send_json( + { + "type": "prepare", + "system_prompt": self.spec.system_prompt, + "config": { + "force_listen_count": self.config.force_listen_count, + "chunk_ms": self.config.chunk_ms, + "generate_audio": True, + "max_new_speak_tokens_per_chunk": ( + self.config.max_new_speak_tokens_per_chunk + ), + "length_penalty": self.config.length_penalty, + }, + "max_slice_nums": self.config.max_slice_nums, + "deferred_finalize": True, + } + ) + while True: + message = await ws.receive() + if message.type != aiohttp.WSMsgType.TEXT: + if message.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR): + raise RuntimeError("gateway closed while preparing") + continue + payload = json.loads(message.data) + if payload.get("type") == "prepared": + LOG.info( + "[GW] prepared session=%s generation=%d skill=%s", + self.session_id, + self.spec.generation, + self.spec.skill_id, + ) + return + if payload.get("type") == "error": + raise RuntimeError(payload.get("error") or "gateway prepare error") + + async def _next_audio_chunk(self, carry: bytearray) -> np.ndarray | None: + samples = int(SAMPLE_RATE_IN * self.config.chunk_ms / 1000) + target_bytes = samples * PCM16_WIDTH + output = bytearray() + if carry: + take = min(target_bytes, len(carry)) + output.extend(carry[:take]) + del carry[:take] + deadline = time.monotonic() + max(0.1, self.config.chunk_ms / 1000 * 1.5) + while len(output) < target_bytes: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + raw = await self.audio_queue.get(remaining) + if raw is None: + break + needed = target_bytes - len(output) + output.extend(raw[:needed]) + if len(raw) > needed: + carry.extend(raw[needed:]) + if not output: + return None + if len(output) < target_bytes: + output.extend(b"\x00" * (target_bytes - len(output))) + return pcm16le_to_float32(bytes(output)) + + async def _send_loop(self, ws: aiohttp.ClientWebSocketResponse) -> None: + carry = bytearray() + last_frame_sequence = -1 + last_frame_sent = 0.0 + while not self._closing: + audio = await self._next_audio_chunk(carry) + if audio is None: + continue + payload: dict[str, Any] = { + "type": "audio_chunk", + "audio_base64": float32_to_base64(audio), + } + if self.gate.speech_hold_active: + payload["force_listen"] = True + frame = self.latest_frame + frame_age_ms = now_ms() - frame.timestamp_ms + frame_due = time.monotonic() - last_frame_sent >= self.config.image_resend_s + if ( + frame.jpeg + and frame_age_ms <= self.config.image_max_age_s * 1000 + and (frame.sequence != last_frame_sequence or frame_due) + ): + payload["frame_base64_list"] = [ + base64.b64encode(frame.jpeg).decode("ascii") + ] + last_frame_sequence = frame.sequence + last_frame_sent = time.monotonic() + await self._send_json(payload) + + async def inject_task(self, text: str) -> bool: + """Send one text+latest-frame task to a newly prepared Skill Session.""" + text = text.strip() + if not text: + return False + payload: dict[str, Any] = { + "type": "audio_chunk", + "inject_text": text, + "max_slice_nums": self.config.max_slice_nums, + } + frame = self.latest_frame + frame_age_ms = now_ms() - frame.timestamp_ms + has_frame = bool( + frame.jpeg + and frame_age_ms <= self.config.image_max_age_s * 1000 + ) + if has_frame and frame.jpeg is not None: + payload["frame_base64_list"] = [ + base64.b64encode(frame.jpeg).decode("ascii") + ] + await self._send_json(payload) + LOG.info( + "[TASK] injected generation=%d skill=%s frame=%s text=%r", + self.spec.generation, + self.spec.skill_id, + has_frame, + text, + ) + return has_frame + + async def _receive_loop(self, ws: aiohttp.ClientWebSocketResponse) -> None: + async for message in ws: + if message.type == aiohttp.WSMsgType.TEXT: + payload = json.loads(message.data) + message_type = payload.get("type") + if message_type in {"result", "audio_only"}: + await self.on_result(self, payload) + elif message_type == "stopped": + return + elif message_type in {"timeout", "error"}: + raise RuntimeError( + payload.get("error") or payload.get("reason") or message_type + ) + elif message.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR): + return + + async def stop(self, cleanup_mode: str) -> None: + self._closing = True + if self.ws is not None and not self.ws.closed: + try: + await self._send_json({"type": "stop", "cleanup_mode": cleanup_mode}) + except Exception: + pass + try: + await asyncio.wait_for(self._stopped.wait(), timeout=self.config.close_timeout_s) + except asyncio.TimeoutError: + pass + if self.ws is not None and not self.ws.closed: + await self.ws.close() + if self._task and not self._task.done(): + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + + +SessionFactory = Callable[ + [ + RokidRuntimeConfig, + SessionSpec, + DropOldestAudioQueue, + LatestFrame, + OutputGate, + Callable[[GatewayDuplexSession, dict[str, Any]], Awaitable[None]], + ], + GatewayDuplexSession, +] + + +class GatewaySessionManager: + """Python counterpart of the frozen BrowserSessionAdapter contract.""" + + def __init__( + self, + config: RokidRuntimeConfig, + registry: SkillRegistry, + audio_queue: DropOldestAudioQueue, + latest_frame: LatestFrame, + speaker: SpeakerSink, + session_factory: SessionFactory = GatewayDuplexSession, + ): + self.config = config + self.registry = registry + self.audio_queue = audio_queue + self.latest_frame = latest_frame + self.speaker = speaker + self.session_factory = session_factory + self.gate = OutputGate(current_skill=registry.default_skill) + self.harness: SessionTelemetry | None = None + self.active: GatewayDuplexSession | None = None + self._restart_lock = asyncio.Lock() + self._playback_epoch = 0 + self._playback_release_task: asyncio.Task[None] | None = None + + async def send_telemetry(self, payload: dict[str, Any]) -> None: + if self.harness is not None: + await self.harness.send(payload) + + async def _set_playback_active(self, active: bool, source: str) -> None: + await self.send_telemetry( + { + "type": "playback.state", + "active": active, + "source": source, + "generation": self.gate.generation, + "pending_ms": round(self.speaker.pending_ms(), 1), + } + ) + + async def _mark_playback_started(self, source: str) -> None: + self._playback_epoch += 1 + epoch = self._playback_epoch + if self._playback_release_task is not None: + self._playback_release_task.cancel() + await self._set_playback_active(True, source) + + async def release_after_drain() -> None: + try: + while self.speaker.pending_ms() > 0: + await asyncio.sleep( + max(0.02, min(0.25, self.speaker.pending_ms() / 1000.0)) + ) + await asyncio.sleep(max(0.0, self.config.playback_echo_tail_s)) + if epoch == self._playback_epoch: + await self._set_playback_active(False, source) + except asyncio.CancelledError: + raise + + self._playback_release_task = asyncio.create_task(release_after_drain()) + + async def _clear_playback_state(self, source: str) -> None: + self._playback_epoch += 1 + if self._playback_release_task is not None: + self._playback_release_task.cancel() + await asyncio.gather(self._playback_release_task, return_exceptions=True) + self._playback_release_task = None + await self._set_playback_active(False, source) + + def _make_spec( + self, + skill_id: str, + slots: dict[str, Any], + system_prompt: str | None = None, + ) -> SessionSpec: + rendered = self.registry.render(skill_id, slots) + return SessionSpec( + generation=self.gate.generation, + skill_id=skill_id, + slots=dict(slots), + system_prompt=system_prompt or rendered.text, + ) + + async def start_initial(self) -> None: + async with self._restart_lock: + if self.active is not None: + return + spec = self._make_spec(self.registry.default_skill, {}) + session = self.session_factory( + self.config, + spec, + self.audio_queue, + self.latest_frame, + self.gate, + self.handle_result, + ) + self.active = session + try: + await session.start() + except Exception: + self.active = None + raise + await self._emit_bound() + + async def _emit_bound(self) -> None: + await self.send_telemetry( + { + "type": "session.state", + "phase": "bound", + "generation": self.gate.generation, + "skill_id": self.gate.current_skill, + "slots": dict(self.gate.current_slots or {}), + "session_id": self.active.session_id if self.active else None, + } + ) + + async def emit_recovery_sync(self) -> None: + if self.active is None: + return + await self.send_telemetry( + { + "type": "session.state", + "phase": "restart_complete", + "generation": self.gate.generation, + "skill_id": self.gate.current_skill, + "slots": dict(self.gate.current_slots or {}), + "new_session_id": self.active.session_id, + "cleanup_mode": "reconnect_sync", + } + ) + await self._emit_bound() + + async def handle_control(self, event: dict[str, Any]) -> dict[str, Any]: + if not event.get("accepted"): + return {"ok": False, "ignored": True} + intent = str(event.get("intent") or "") + if intent == "stop_speech": + return await self.stop_speech(event) + if intent == "resume_speech": + return await self.resume_speech(event) + if intent in { + "reset_session", + "activate_skill", + "cancel_skill", + "return_to_chat", + }: + return await self.restart(event) + return {"ok": False, "ignored": True} + + def _ack_base(self, event: dict[str, Any]) -> dict[str, Any]: + return { + "type": "control.ack", + "event_id": event.get("event_id"), + "intent": event.get("intent"), + "generation": self.gate.generation, + "dropped_old_text": self.gate.dropped_old_text, + "dropped_old_audio": self.gate.dropped_old_audio, + } + + async def stop_speech( + self, event: dict[str, Any], *, emit_ack: bool = True + ) -> dict[str, Any]: + self.gate.stop() + await self.speaker.block_and_flush() + await self._clear_playback_state("stop") + ack = {**self._ack_base(event), "ok": True} + if emit_ack: + await self.send_telemetry(ack) + LOG.info("[CONTROL] STOP event=%s", event.get("event_id")) + return ack + + async def resume_speech(self, event: dict[str, Any]) -> dict[str, Any]: + self.gate.resume() + await self.speaker.resume() + ack = {**self._ack_base(event), "ok": True} + await self.send_telemetry(ack) + LOG.info("[CONTROL] RESUME event=%s", event.get("event_id")) + return ack + + async def restart(self, event: dict[str, Any]) -> dict[str, Any]: + async with self._restart_lock: + requested_skill = str(event.get("skill_id") or self.registry.default_skill) + requested_slots = dict(event.get("slots") or {}) + if ( + str(event.get("intent")) != "reset_session" + and requested_skill == self.gate.current_skill + and same_slots(requested_slots, dict(self.gate.current_slots or {})) + ): + ack = {**self._ack_base(event), "ok": True, "no_restart": True} + await self.send_telemetry(ack) + return ack + + started = time.monotonic() + await self.stop_speech(event, emit_ack=False) + self.gate.restart_in_progress = True + old_session = self.active + old_session_id = old_session.session_id if old_session else None + self.gate.generation += 1 + await self.send_telemetry( + { + "type": "session.state", + "phase": "restart_started", + "generation": self.gate.generation, + "event_id": event.get("event_id"), + "old_session_id": old_session_id, + } + ) + self.active = None + try: + if old_session: + await old_session.stop(self.config.cleanup_mode) + self.audio_queue.clear() + spec = self._make_spec( + requested_skill, + requested_slots, + str(event.get("system_prompt") or "") or None, + ) + session = self.session_factory( + self.config, + spec, + self.audio_queue, + self.latest_frame, + self.gate, + self.handle_result, + ) + self.active = session + await session.start() + self.gate.current_skill = requested_skill + self.gate.current_slots = requested_slots + self.gate.replacement_ready() + await self.speaker.resume() + await self._emit_bound() + latency_ms = (time.monotonic() - started) * 1000.0 + state = { + "type": "session.state", + "phase": "restart_complete", + "generation": self.gate.generation, + "skill_id": requested_skill, + "slots": requested_slots, + "old_session_id": old_session_id, + "new_session_id": session.session_id, + "cleanup_mode": self.config.cleanup_mode, + } + await self.send_telemetry(state) + if self.config.play_audio and self.config.session_ready_chime: + cue = make_session_ready_chime( + amplitude=self.config.session_ready_chime_volume + ) + self.gate.local_cue_mute_until_mono = max( + self.gate.local_cue_mute_until_mono, + time.monotonic() + + cue.size / SAMPLE_RATE_OUT + + self.config.session_ready_chime_feedback_guard_s, + ) + await self.speaker.enqueue(cue, self.gate.generation) + await self._mark_playback_started("session_ready_chime") + LOG.info( + "[CUE] session ready generation=%d skill=%s", + self.gate.generation, + requested_skill, + ) + trigger = self.registry.task_trigger(requested_skill, requested_slots) + trigger_frame = False + trigger_error = "" + if trigger: + try: + trigger_frame = await session.inject_task(trigger) + await self.send_telemetry( + { + "type": "session.state", + "phase": "task_trigger_sent", + "generation": self.gate.generation, + "skill_id": requested_skill, + "slots": requested_slots, + "session_id": session.session_id, + "has_frame": trigger_frame, + } + ) + except Exception as exc: + trigger_error = str(exc) + LOG.warning( + "[TASK] one-shot trigger failed skill=%s: %s", + requested_skill, + exc, + ) + ack = { + **self._ack_base(event), + "ok": True, + "restart_latency_ms": latency_ms, + "skill_id": requested_skill, + "slots": requested_slots, + "old_session_id": old_session_id, + "new_session_id": session.session_id, + "cleanup_mode": self.config.cleanup_mode, + "task_trigger_sent": bool(trigger and not trigger_error), + "task_trigger_has_frame": trigger_frame, + "task_trigger_error": trigger_error, + } + await self.send_telemetry(ack) + LOG.info( + "[CONTROL] %s ready generation=%d session=%s latency_ms=%.1f", + requested_skill, + self.gate.generation, + session.session_id, + latency_ms, + ) + return ack + except Exception as exc: + if self.active: + await self.active.stop(self.config.cleanup_mode) + self.active = None + self.gate.restart_in_progress = False + ack = {**self._ack_base(event), "ok": False, "error": str(exc)} + await self.send_telemetry(ack) + LOG.exception("[CONTROL] restart failed") + return ack + + async def handle_result( + self, session: GatewayDuplexSession, result: dict[str, Any] + ) -> None: + is_listen = bool(result.get("is_listen")) + text = str(result.get("text") or "") + audio_b64 = str(result.get("audio_data") or "") + stale = ( + session is not self.active + or session.spec.generation != self.gate.generation + ) + if stale or (self.gate.drop_output_until_listen and not is_listen): + if text: + self.gate.dropped_old_text += 1 + if audio_b64: + self.gate.dropped_old_audio += 1 + return + if is_listen and self.gate.drop_output_until_listen: + if not self.gate.speech_hold_active: + self.gate.drop_output_until_listen = False + await self.speaker.resume() + audio_samples = 0 + if audio_b64 and not is_listen: + try: + pcm = np.frombuffer(base64.b64decode(audio_b64), dtype=np.float32) + audio_samples = int(pcm.size) + await self.speaker.enqueue(pcm, session.spec.generation) + if self.config.play_audio: + await self._mark_playback_started("model") + except Exception as exc: + LOG.warning("model audio decode failed: %s", exc) + await self.send_telemetry( + { + "type": "model.state", + "state": "listen" if is_listen else "speak", + "text": text, + "generation": session.spec.generation, + "session_id": session.session_id, + "skill_id": session.spec.skill_id, + "slots": dict(session.spec.slots), + # llama.cpp-omni marks each decode slice as end_of_turn. The + # actual duplex turn boundary is the later __IS_LISTEN__ + # result, so keep the raw marker for diagnostics but aggregate + # model text until listening resumes. + "decode_end": bool(result.get("end_of_turn")), + "end_of_turn": is_listen, + "message_type": str(result.get("type") or "result"), + "audio_samples": audio_samples, + "audio_ms": round( + audio_samples * 1000.0 / SAMPLE_RATE_OUT, 1 + ), + } + ) + if text: + LOG.info("[MODEL] listen=%s text=%s", is_listen, text) + + async def close(self) -> None: + await self._clear_playback_state("close") + if self.active: + await self.active.stop(self.config.cleanup_mode) + self.active = None + + def health(self) -> dict[str, Any]: + return { + "generation": self.gate.generation, + "current_skill": self.gate.current_skill, + "current_slots": dict(self.gate.current_slots or {}), + "speech_hold_active": self.gate.speech_hold_active, + "restart_in_progress": self.gate.restart_in_progress, + "dropped_old_text": self.gate.dropped_old_text, + "dropped_old_audio": self.gate.dropped_old_audio, + "session_id": self.active.session_id if self.active else None, + "gateway_status": self.active.status if self.active else "disconnected", + "gateway_error": self.active.last_error if self.active else "", + } + + +class HarnessClient: + def __init__( + self, + url: str, + client_id: str, + manager: GatewaySessionManager, + reconnect_s: float, + ): + separator = "&" if "?" in url else "?" + self.url = f"{url}{separator}client_id={client_id}" + self.manager = manager + self.reconnect_s = reconnect_s + self.connected = False + self.ws: aiohttp.ClientWebSocketResponse | None = None + self._stop = asyncio.Event() + self._send_lock = asyncio.Lock() + self._control_tasks: set[asyncio.Task[Any]] = set() + + async def run(self) -> None: + while not self._stop.is_set(): + try: + async with aiohttp.ClientSession() as client: + async with client.ws_connect( + self.url, heartbeat=30, max_msg_size=0 + ) as ws: + self.ws = ws + self.connected = True + LOG.info("Harness connected: %s", self.url) + async for message in ws: + if message.type != aiohttp.WSMsgType.TEXT: + if message.type in ( + aiohttp.WSMsgType.CLOSED, + aiohttp.WSMsgType.ERROR, + ): + break + continue + payload = json.loads(message.data) + message_type = payload.get("type") + if message_type == "harness.ready": + await self.manager.emit_recovery_sync() + elif message_type == "control.intent": + task = asyncio.create_task( + self.manager.handle_control(payload) + ) + self._control_tasks.add(task) + task.add_done_callback(self._control_tasks.discard) + except asyncio.CancelledError: + raise + except Exception as exc: + if not self._stop.is_set(): + LOG.warning("Harness connection failed: %s", exc) + finally: + self.connected = False + self.ws = None + if not self._stop.is_set(): + await asyncio.sleep(self.reconnect_s) + + async def send(self, payload: dict[str, Any]) -> None: + async with self._send_lock: + if self.ws is not None and not self.ws.closed: + await self.ws.send_json(payload) + + async def send_audio(self, samples: np.ndarray) -> None: + if samples.size == 0: + return + await self.send( + { + "type": "audio.mirror", + "started_at_ms": now_ms() - samples.size * 1000.0 / SAMPLE_RATE_IN, + "sample_rate": SAMPLE_RATE_IN, + "audio_b64": float32_to_base64(samples), + } + ) + + async def send_frame(self, jpeg: bytes, sequence: int, timestamp_ms: float) -> None: + await self.send( + { + "type": "frame.shadow", + "frame_id": f"rokid_{sequence}_{int(timestamp_ms)}", + "timestamp_ms": timestamp_ms, + "jpeg_b64": base64.b64encode(jpeg).decode("ascii"), + } + ) + + async def close(self) -> None: + self._stop.set() + if self.ws is not None and not self.ws.closed: + await self.ws.close() + for task in tuple(self._control_tasks): + task.cancel() + await asyncio.gather(*self._control_tasks, return_exceptions=True) + + +@dataclass(slots=True) +class RuntimeStats: + started_mono: float = field(default_factory=time.monotonic) + audio_packets: int = 0 + audio_bytes: int = 0 + audio_clients: int = 0 + image_count: int = 0 + image_bytes: int = 0 + last_audio_ms: float = 0.0 + audio_rms: float = 0.0 + audio_peak: float = 0.0 + non_silent_audio_packets: int = 0 + + +class PhaseBRokidRuntime: + """Rokid sensor ingress + frozen Harness control + Gateway session adapter.""" + + def __init__( + self, + config: RokidRuntimeConfig, + *, + speaker: SpeakerSink | None = None, + session_factory: SessionFactory = GatewayDuplexSession, + ): + self.config = config + self.registry = SkillRegistry(config.skills_config) + self.audio_queue = DropOldestAudioQueue(config.audio_queue_packets) + self.audio_mirror = AudioMirrorChunker() + self.latest_frame = LatestFrame() + self.stats = RuntimeStats(started_mono=time.monotonic()) + self.speaker = speaker or (PCSpeaker() if config.play_audio else NullSpeaker()) + self.manager = GatewaySessionManager( + config, + self.registry, + self.audio_queue, + self.latest_frame, + self.speaker, + session_factory=session_factory, + ) + self.harness = HarnessClient( + config.harness_url, + config.harness_client_id, + self.manager, + config.reconnect_s, + ) + self.manager.harness = self.harness + self._tasks: list[asyncio.Task[Any]] = [] + self._close_lock = asyncio.Lock() + self._closed = False + self._no_input_warning_emitted = False + + def health(self) -> dict[str, Any]: + return { + "ok": True, + "phase": "B", + "device": "rokid", + "uptime_s": round(time.monotonic() - self.stats.started_mono, 3), + "harness_connected": self.harness.connected, + "audio_packets_in": self.stats.audio_packets, + "audio_bytes_in": self.stats.audio_bytes, + "audio_clients": self.stats.audio_clients, + "latest_audio_age_ms": ( + round(now_ms() - self.stats.last_audio_ms, 1) + if self.stats.last_audio_ms + else None + ), + "audio_queue_size": self.audio_queue.size, + "dropped_audio_packets": self.audio_queue.dropped_packets, + "audio_rms": round(self.stats.audio_rms, 6), + "audio_peak": round(self.stats.audio_peak, 6), + "input_gain": self.config.input_gain, + "effective_audio_rms": round( + min(1.0, self.stats.audio_rms * self.config.input_gain), 6 + ), + "non_silent_audio_packets": self.stats.non_silent_audio_packets, + "image_count": self.stats.image_count, + "device_input_ready": bool( + self.stats.audio_packets or self.stats.image_count + ), + "latest_image_age_ms": ( + round(now_ms() - self.latest_frame.timestamp_ms, 1) + if self.latest_frame.timestamp_ms + else None + ), + **self.manager.health(), + } + + async def _initial_session_loop(self) -> None: + while True: + try: + await self.manager.start_initial() + return + except asyncio.CancelledError: + raise + except Exception as exc: + LOG.warning("initial Gateway session failed: %s", exc) + await asyncio.sleep(self.config.reconnect_s) + + async def _stats_loop(self) -> None: + previous_audio = 0 + previous_image = 0 + while True: + await asyncio.sleep(5.0) + if ( + not self._no_input_warning_emitted + and time.monotonic() - self.stats.started_mono >= 10.0 + and self.stats.audio_packets == 0 + and self.stats.image_count == 0 + ): + self._no_input_warning_emitted = True + LOG.warning( + "[ROKID][NO_INPUT] no PCM/JPEG has reached this process. " + "After the PC runtime is listening, restart OpenGlass " + "Sensor Mode on the glasses (STOP -> RUN) and verify that " + "the APK targets the PC WLAN address on port %d", + self.config.port, + ) + LOG.info( + "[STATS] audio=%.1fpps images=%d(+%d) audio_clients=%d " + "queue=%d drops=%d rms=%.4f peak=%.4f harness=%s gateway=%s " + "skill=%s generation=%d", + (self.stats.audio_packets - previous_audio) / 5.0, + self.stats.image_count, + self.stats.image_count - previous_image, + self.stats.audio_clients, + self.audio_queue.size, + self.audio_queue.dropped_packets, + self.stats.audio_rms, + self.stats.audio_peak, + self.harness.connected, + self.manager.health()["gateway_status"], + self.manager.gate.current_skill, + self.manager.gate.generation, + ) + previous_audio = self.stats.audio_packets + previous_image = self.stats.image_count + + async def start(self) -> None: + await self.speaker.start() + self._tasks = [ + asyncio.create_task(self.harness.run()), + asyncio.create_task(self._initial_session_loop()), + asyncio.create_task(self._stats_loop()), + ] + + async def close(self) -> None: + async with self._close_lock: + if self._closed: + return + LOG.info("Rokid runtime shutdown started") + + async def bounded(label: str, awaitable: Awaitable[None]) -> None: + try: + await asyncio.wait_for( + awaitable, + timeout=max(0.5, self.config.close_timeout_s), + ) + except asyncio.TimeoutError: + LOG.warning( + "Rokid shutdown step timed out: %s (continuing)", label + ) + except Exception as exc: + LOG.warning( + "Rokid shutdown step failed: %s: %s", label, exc + ) + + await bounded("harness", self.harness.close()) + for task in self._tasks: + task.cancel() + if self._tasks: + await bounded( + "background_tasks", + asyncio.gather(*self._tasks, return_exceptions=True), + ) + self._tasks.clear() + await bounded("gateway_session", self.manager.close()) + await bounded("pc_speaker", self.speaker.close()) + self.audio_queue.clear() + self.audio_mirror.clear() + self._closed = True + LOG.info("Rokid runtime shutdown complete; port may be reused") + + def create_app(self) -> web.Application: + app = web.Application(client_max_size=8 * 1024 * 1024) + app["runtime"] = self + + async def health(_request: web.Request) -> web.Response: + return web.json_response(self.health()) + + async def capture(_request: web.Request) -> web.Response: + if not self.latest_frame.jpeg: + return web.Response(status=404, text="no image") + return web.Response(body=self.latest_frame.jpeg, content_type="image/jpeg") + + async def rokid_image(request: web.Request) -> web.Response: + raw = await request.read() + if not (raw.startswith(b"\xff\xd8") and raw.endswith(b"\xff\xd9")): + return web.Response(status=400, text="expected JPEG bytes") + try: + jpeg = await asyncio.to_thread( + rotate_jpeg_clockwise, + raw, + self.config.image_rotate_cw, + self.config.image_jpeg_quality, + ) + except Exception as exc: + return web.Response(status=400, text=str(exc)) + timestamp = now_ms() + self.latest_frame.set(jpeg, timestamp) + self.stats.image_count += 1 + self.stats.image_bytes += len(raw) + if self.stats.image_count == 1: + LOG.info("[ROKID] first JPEG received from %s", request.remote) + asyncio.create_task( + self.harness.send_frame( + jpeg, self.latest_frame.sequence, timestamp + ) + ) + return web.json_response( + {"ok": True, "image_count": self.stats.image_count, "bytes": len(raw)} + ) + + async def rokid_audio(request: web.Request) -> web.WebSocketResponse: + ws = web.WebSocketResponse(heartbeat=30, max_msg_size=0) + await ws.prepare(request) + self.stats.audio_clients += 1 + LOG.info("[ROKID] audio connected from %s", request.remote) + try: + async for message in ws: + if message.type == aiohttp.WSMsgType.BINARY: + raw = bytes(message.data) + if len(raw) < PCM16_WIDTH: + continue + if len(raw) % PCM16_WIDTH: + raw = raw[:-1] + self.stats.audio_packets += 1 + self.stats.audio_bytes += len(raw) + self.stats.last_audio_ms = now_ms() + if self.stats.audio_packets == 1: + LOG.info("[ROKID] first PCM packet received") + samples = pcm16le_to_float32(raw) + packet_rms = float(np.sqrt(np.mean(np.square(samples)))) + packet_peak = float(np.max(np.abs(samples))) + self.stats.audio_rms = ( + 0.8 * self.stats.audio_rms + 0.2 * packet_rms + ) + self.stats.audio_peak = packet_peak + if packet_rms >= 0.01: + self.stats.non_silent_audio_packets += 1 + if time.monotonic() >= self.manager.gate.local_cue_mute_until_mono: + amplified = apply_pcm16_gain(raw, self.config.input_gain) + self.audio_queue.put_nowait(amplified) + for frame in self.audio_mirror.feed(amplified): + await self.harness.send_audio(frame) + elif message.type == aiohttp.WSMsgType.ERROR: + break + finally: + self.stats.audio_clients = max(0, self.stats.audio_clients - 1) + LOG.info("[ROKID] audio disconnected from %s", request.remote) + return ws + + async def on_startup(_app: web.Application) -> None: + await self.start() + + async def on_cleanup(_app: web.Application) -> None: + await self.close() + + app.router.add_get("/", health) + app.router.add_get("/health", health) + app.router.add_get("/capture", capture) + app.router.add_post("/rokid/image", rokid_image) + app.router.add_get("/rokid/audio", rokid_audio) + app.on_startup.append(on_startup) + app.on_cleanup.append(on_cleanup) + return app + + +def default_skills_config() -> str: + return str( + Path(__file__).resolve().parents[1] / "config" / "skills.example.yaml" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Rokid Phase B adapter: sensors + Harness + MiniCPM Gateway" + ) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=18_080) + parser.add_argument("--gateway", default="localhost:8040") + parser.add_argument("--gateway-tls", action="store_true", default=False) + parser.add_argument( + "--no-gateway-tls", dest="gateway_tls", action="store_false" + ) + parser.add_argument( + "--harness-url", default="ws://127.0.0.1:8021/ws/control" + ) + parser.add_argument("--client-id", default="rokid-phase-b") + parser.add_argument("--skills-config", default=default_skills_config()) + parser.add_argument("--cleanup-mode", choices=("light", "full"), default="light") + parser.add_argument("--image-rotate-cw", type=int, default=270) + parser.add_argument("--image-jpeg-quality", type=int, default=95) + parser.add_argument("--audio-queue-packets", type=int, default=96) + parser.add_argument( + "--input-gain", + type=float, + default=12.0, + help="Rokid PCM gain before Harness/Gateway; clipped to PCM16", + ) + parser.add_argument("--chunk-ms", type=int, default=1_000) + parser.add_argument("--force-listen-count", type=int, default=3) + parser.add_argument("--no-play", action="store_true") + parser.add_argument( + "--no-session-ready-chime", + action="store_true", + help="disable the local restart_complete notification cue", + ) + parser.add_argument( + "--session-ready-chime-volume", + type=float, + default=0.32, + help="restart cue amplitude in [0, 1] (default: 0.32)", + ) + parser.add_argument( + "--playback-echo-tail-s", + type=float, + default=0.80, + help="keep EchoGuard active after the PC speaker queue drains", + ) + parser.add_argument("--log-level", default="INFO") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if not 0.0 <= args.session_ready_chime_volume <= 1.0: + raise SystemExit("--session-ready-chime-volume must be between 0 and 1") + if args.playback_echo_tail_s < 0.0: + raise SystemExit("--playback-echo-tail-s must be non-negative") + logging.basicConfig( + level=getattr(logging, args.log_level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + force=True, + ) + config = RokidRuntimeConfig( + host=args.host, + port=args.port, + gateway=args.gateway, + gateway_tls=args.gateway_tls, + harness_url=args.harness_url, + harness_client_id=args.client_id, + skills_config=args.skills_config, + cleanup_mode=args.cleanup_mode, + chunk_ms=args.chunk_ms, + force_listen_count=args.force_listen_count, + audio_queue_packets=args.audio_queue_packets, + input_gain=args.input_gain, + image_rotate_cw=args.image_rotate_cw, + image_jpeg_quality=args.image_jpeg_quality, + play_audio=not args.no_play, + session_ready_chime=not args.no_session_ready_chime, + session_ready_chime_volume=args.session_ready_chime_volume, + playback_echo_tail_s=args.playback_echo_tail_s, + ) + runtime = PhaseBRokidRuntime(config) + watchdog = CtrlCExitWatchdog() + shutdown_signals = [signal.SIGINT] + if hasattr(signal, "SIGBREAK"): + shutdown_signals.append(signal.SIGBREAK) + previous_handlers = { + signum: signal.getsignal(signum) for signum in shutdown_signals + } + + def handle_console_shutdown(signum: int, frame: Any) -> None: + watchdog.arm() + previous_handler = previous_handlers[signum] + if callable(previous_handler): + previous_handler(signum, frame) + else: + raise KeyboardInterrupt + + for signum in shutdown_signals: + signal.signal(signum, handle_console_shutdown) + LOG.info("Rokid Phase B input: http://%s:%d", config.host, config.port) + LOG.info("Harness: %s", config.harness_url) + LOG.info("Gateway: %s://%s", "wss" if config.gateway_tls else "ws", config.gateway) + try: + web.run_app( + runtime.create_app(), + host=config.host, + port=config.port, + access_log=None, + shutdown_timeout=max(0.5, config.close_timeout_s), + handler_cancellation=True, + ) + except OSError as exc: + if getattr(exc, "winerror", None) == 10048: + LOG.error( + "Port %d is already occupied by another process. A runtime " + "started with this patched version exits within %.0fs after " + "Ctrl+C.", + config.port, + CTRL_C_HARD_EXIT_S, + ) + raise SystemExit(2) from None + raise + finally: + for signum, previous_handler in previous_handlers.items(): + signal.signal(signum, previous_handler) + if watchdog.armed: + LOG.info("Graceful shutdown returned; exiting Rokid process") + + +if __name__ == "__main__": + main() diff --git a/extensions/assistive_harness/prompts/README.md b/extensions/assistive_harness/prompts/README.md new file mode 100644 index 0000000..b818ff4 --- /dev/null +++ b/extensions/assistive_harness/prompts/README.md @@ -0,0 +1,39 @@ +# User-editable Skill prompts + +These files are the system prompts used when the Voice Skill Harness creates a +new MiniCPM-o Duplex Session. Prompt text is read again on every activation, so +editing an existing `.txt` file takes effect on the next Skill switch without +restarting the Harness service. + +## Shipped prompts + +| Skill ID | Prompt file | Default | +|---|---|---| +| `idle_chat` | `idle_chat_zh.txt` | enabled | +| `find_object` | `find_object_zh.txt` | enabled | +| `read_text` | `read_text_zh.txt` | enabled | +| `describe_scene` | `describe_scene_zh.txt` | enabled | +| `obstacle_avoidance` | `obstacle_avoidance_zh.txt` | enabled / experimental | + +The find/read/obstacle task bodies were derived from the project's frozen +AAAI_SI C1 prompts. `find_object_zh.txt` additionally contains `{{target}}`, +because the new Session must receive the target extracted from the command +that closed the old Session. No external absolute path is required at runtime. + +## Editing an existing prompt + +1. Back up the target `.txt` file. +2. Replace its contents with a UTF-8 system prompt. +3. Keep every declared template variable, such as `{{target}}`. +4. Switch to chat/another Skill and activate this Skill again. (The same + Skill/same slots command is intentionally deduplicated.) The new hot Session + will read the new text and record its rendered SHA-256 in telemetry. + +## Adding a Skill + +Create another `.txt` file and add a matching entry under `skills:` in +`../config/skills.example.yaml`. Configuration changes require restarting the +8021 Harness service; prompt-only changes do not. + +Obstacle avoidance has not passed a safety evaluation. Use it only for +stationary, supervised tests; never treat the Demo as a mobility safety device. diff --git a/extensions/assistive_harness/prompts/describe_scene_zh.txt b/extensions/assistive_harness/prompts/describe_scene_zh.txt new file mode 100644 index 0000000..b19376b --- /dev/null +++ b/extensions/assistive_harness/prompts/describe_scene_zh.txt @@ -0,0 +1,8 @@ +当前技能：场景描述。 + +规则： +1. 用一到三句话概括当前画面的主要物体、位置关系和明显状态。 +2. 优先描述门口、桌面、道路、人物、显著文字和可能影响用户行动的大物体。 +3. 只描述当前可见证据；看不清时直接说画面不清楚。 +4. 不给出未经验证的距离、路线或安全保证。 +5. 避免反复描述没有变化的内容。 diff --git a/extensions/assistive_harness/prompts/find_object_zh.txt b/extensions/assistive_harness/prompts/find_object_zh.txt new file mode 100644 index 0000000..affe3a2 --- /dev/null +++ b/extensions/assistive_harness/prompts/find_object_zh.txt @@ -0,0 +1,9 @@ +你是智能眼镜找物助手。你能看到用户当前第一视角画面，并听到用户要找的目标。 + +当前寻找目标：{{target}} + +上述目标已经由外部 Harness 从用户语音中识别。新会话启动后，直接寻找该目标，不要再次询问目标是什么。 +用户问完后，必须直接根据当前画面回答，不要先说“好的”“我来找”“稍等”。 +如果目标在画面里可见，用一句话说清它的位置、方向和附近锚点，例如“复印机在左前方，靠近窗户”。 +如果目标不可见，就说“没看到”，并给一个具体下一步建议，例如转向、靠近、抬头或调整角度。 +只描述画面中确实可见的内容，不要凭常识猜测。回答尽量短。 diff --git a/extensions/assistive_harness/prompts/idle_chat_zh.txt b/extensions/assistive_harness/prompts/idle_chat_zh.txt new file mode 100644 index 0000000..a558053 --- /dev/null +++ b/extensions/assistive_harness/prompts/idle_chat_zh.txt @@ -0,0 +1,8 @@ +你是一副面向视障用户的本地 AI 眼镜助手。 + +规则： +1. 只根据当前音频、视频和明确上下文回答，不要编造看不清或没听清的内容。 +2. 回答简洁、自然，优先一到三句话。 +3. 不要主动给出未经验证的距离、安全或行动指令。 +4. 用户的“停一下、重新开始、找物、识字”等控制命令由外部 Harness 处理；不要争夺控制权，也不要重复解释系统机制。 +5. 没有足够证据时，直接说明不确定，并建议用户停稳或调整视角。 diff --git a/extensions/assistive_harness/prompts/obstacle_avoidance_zh.txt b/extensions/assistive_harness/prompts/obstacle_avoidance_zh.txt new file mode 100644 index 0000000..82cb340 --- /dev/null +++ b/extensions/assistive_harness/prompts/obstacle_avoidance_zh.txt @@ -0,0 +1,9 @@ +你是智能眼镜避障助手。你能看到用户当前第一视角画面，并听到用户询问前方障碍。 + +用户问完后，必须直接根据当前画面回答，不要只说“好的”“我会留意”。 +如果能确认障碍，用一句短话说明障碍是什么、位于左前方/正前方/右前方或地面哪里，并给出立即可执行的安全建议，例如停下、向左绕行或向右绕行。 +只根据画面中确实可见的障碍和可通行空间判断，不要猜测，不要把普通物体误报为障碍。 +如果摄像头没有朝向行进方向、地面不可见、画面被遮挡或证据不足，明确回答“当前画面不足以判断前方是否安全”，建议用户先停下并把镜头转向前方或地面。 +在无法确认通行空间时，不得声称“前方安全”或直接给出通行方向。 +如果障碍随后进入画面或画面质量改善，应立即根据新画面更新判断。 +回答尽量简短。 diff --git a/extensions/assistive_harness/prompts/read_text_zh.txt b/extensions/assistive_harness/prompts/read_text_zh.txt new file mode 100644 index 0000000..5d104b6 --- /dev/null +++ b/extensions/assistive_harness/prompts/read_text_zh.txt @@ -0,0 +1,7 @@ +你是智能眼镜读字助手。你能看到用户当前第一视角画面，并听到用户要求读字。 + +用户一开口要求读字，你必须回答，不能保持沉默。 +用户问完后，直接读出画面里清晰可见的文字，不要先说“好的”“我来”。 +只输出你看清的文字、数字或符号；看不清的字不要猜，不要根据物品常识补全。 +如果只能看清部分文字，可以只读出看清的部分。 +如果整体看不清，也必须回答“看不清”，并建议靠近、正对文字或重新对焦。 diff --git a/extensions/assistive_harness/registry.py b/extensions/assistive_harness/registry.py new file mode 100644 index 0000000..35b2fc1 --- /dev/null +++ b/extensions/assistive_harness/registry.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + + +class RegistryError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class RenderedPrompt: + skill_id: str + text: str + path: str + sha256: str + slots: dict[str, Any] + + +class SkillRegistry: + def __init__(self, config_path: str | Path): + self.config_path = Path(config_path).resolve() + raw = yaml.safe_load(self.config_path.read_text(encoding="utf-8")) or {} + self.version = int(raw.get("version", 1)) + self.default_skill = str(raw.get("default_skill", "idle_chat")) + self.control = dict(raw.get("control") or {}) + self.skills: dict[str, dict[str, Any]] = dict(raw.get("skills") or {}) + if self.default_skill not in self.skills: + raise RegistryError(f"default skill is missing: {self.default_skill}") + self._validate_prompt_files() + + def _prompt_path(self, skill_id: str) -> Path: + spec = self.get(skill_id) + path = Path(str(spec.get("prompt_file") or "")) + if not path.is_absolute(): + path = self.config_path.parent / path + return path.resolve() + + def prompt_path(self, skill_id: str) -> Path: + """Return the user-editable prompt path exposed by the registry.""" + return self._prompt_path(skill_id) + + def _validate_prompt_files(self) -> None: + for skill_id, spec in self.skills.items(): + if not isinstance(spec, dict): + raise RegistryError(f"invalid skill spec: {skill_id}") + path = self._prompt_path(skill_id) + if not path.is_file(): + raise RegistryError(f"prompt file is missing for {skill_id}: {path}") + + def get(self, skill_id: str) -> dict[str, Any]: + spec = self.skills.get(skill_id) + if spec is None: + raise RegistryError(f"unknown skill: {skill_id}") + if not isinstance(spec, dict): + raise RegistryError(f"invalid skill spec: {skill_id}") + return spec + + def is_enabled(self, skill_id: str) -> bool: + return bool(self.get(skill_id).get("enabled", False)) + + def cooldown_ms(self, skill_id: str) -> int: + return int(self.get(skill_id).get("cooldown_ms", 0)) + + def task_trigger( + self, skill_id: str, slots: dict[str, Any] | None = None + ) -> str: + """Render the optional one-shot command sent to a fresh Skill Session.""" + text = str(self.get(skill_id).get("task_trigger") or "").strip() + slots = dict(slots or {}) + variables = set(re.findall(r"\{\{\s*([a-zA-Z_][\w]*)\s*\}\}", text)) + missing = sorted(name for name in variables if not str(slots.get(name, "")).strip()) + if missing: + raise RegistryError(f"missing task trigger variables for {skill_id}: {missing}") + for name in variables: + text = re.sub( + r"\{\{\s*" + re.escape(name) + r"\s*\}\}", + str(slots[name]), + text, + ) + return text + + def render(self, skill_id: str, slots: dict[str, Any] | None = None) -> RenderedPrompt: + if not self.is_enabled(skill_id): + raise RegistryError(f"skill is disabled: {skill_id}") + slots = dict(slots or {}) + spec = self.get(skill_id) + schema = dict(spec.get("slot_schema") or {}) + for name, slot_spec in schema.items(): + if bool((slot_spec or {}).get("required")) and not str(slots.get(name, "")).strip(): + raise RegistryError(f"missing required slot '{name}' for {skill_id}") + + path = self._prompt_path(skill_id) + text = path.read_text(encoding="utf-8") + variables = set(re.findall(r"\{\{\s*([a-zA-Z_][\w]*)\s*\}\}", text)) + missing = sorted(name for name in variables if name not in slots) + if missing: + raise RegistryError(f"missing prompt variables for {skill_id}: {missing}") + for name in variables: + text = re.sub( + r"\{\{\s*" + re.escape(name) + r"\s*\}\}", + str(slots[name]), + text, + ) + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + return RenderedPrompt( + skill_id=skill_id, + text=text, + path=str(path), + sha256=digest, + slots=slots, + ) diff --git a/extensions/assistive_harness/requirements.txt b/extensions/assistive_harness/requirements.txt new file mode 100644 index 0000000..a7d2d87 --- /dev/null +++ b/extensions/assistive_harness/requirements.txt @@ -0,0 +1,8 @@ +# Assistive Harness Core and local FunASR sidecar. +# Model weights are downloaded separately and are never committed to Git. +fastapi>=0.110,<1 +uvicorn[standard]>=0.29,<1 +numpy>=1.24 +PyYAML>=6.0 +funasr==1.3.1 +modelscope>=1.20 diff --git a/extensions/assistive_harness/router.py b/extensions/assistive_harness/router.py new file mode 100644 index 0000000..43038f5 --- /dev/null +++ b/extensions/assistive_harness/router.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import re +import time +from dataclasses import dataclass + +from .registry import SkillRegistry +from .schemas import ControlEvent, ControlIntent + + +_PUNCTUATION = "，,。！？!?；;：:、\"'“”‘’" +_SPEECH_PARTICLES = ("嗯", "啊", "呀", "吧", "呢", "啦", "哦") + + +def normalize_text(text: str) -> str: + compact = re.sub(r"\s+", "", text or "").lower() + return compact.strip(_PUNCTUATION) + + +def _strip_speech_particles(text: str) -> str: + cleaned = text + while cleaned and any(cleaned.endswith(item) for item in _SPEECH_PARTICLES): + cleaned = cleaned[:-1] + return cleaned + + +def _collapse_repeated_tail(text: str, max_width: int = 4) -> str: + """Collapse a short duplicated ASR tail such as ``手机手机`` or ``母母母``.""" + + cleaned = text + while cleaned: + collapsed = False + for width in range(min(max_width, len(cleaned) // 2), 0, -1): + if cleaned[-width:] == cleaned[-2 * width : -width]: + cleaned = cleaned[:-width] + collapsed = True + break + if not collapsed: + return cleaned + return cleaned + + +def _clean_slot_value(value: str) -> str: + cleaned = _strip_speech_particles(value.strip(_PUNCTUATION)) + cleaned = _collapse_repeated_tail(cleaned) + return _strip_speech_particles(cleaned) + + +@dataclass(slots=True) +class RouteResult: + intent: ControlIntent + skill_id: str | None = None + slots: dict[str, str] | None = None + reason: str = "" + confidence: float = 1.0 + + +class RuleIntentRouter: + """Small deterministic Chinese intent router with exact control commands.""" + + def __init__(self, registry: SkillRegistry): + self.registry = registry + control = registry.control + self.stop_phrases = { + normalize_text(item) + for item in (control.get("stop_speech") or {}).get("phrases", []) + } + self.stop_embedded_phrases = { + normalize_text(item) + for item in (control.get("stop_speech") or {}).get("embedded_phrases", []) + } + self.resume_embedded_phrases = { + normalize_text(item) + for item in (control.get("resume_speech") or {}).get("embedded_phrases", []) + } + self.reset_phrases = { + normalize_text(item) + for item in (control.get("reset_session") or {}).get("phrases", []) + } + self.reset_embedded_phrases = { + normalize_text(item) + for item in (control.get("reset_session") or {}).get("embedded_phrases", []) + } + self.return_phrases = { + normalize_text(item) + for item in (control.get("return_to_chat") or {}).get("phrases", []) + } + self.cancel_phrases = { + normalize_text(item) + for item in (control.get("cancel_skill") or {}).get("phrases", []) + } + + @staticmethod + def _is_explicit_command(compact: str, phrases: set[str]) -> bool: + polite_prefixes = ("麻烦你先", "请", "麻烦", "你先", "请你", "可以", "能不能") + polite_suffixes = ("一下", "好吗", "可以吗", "啊", "呀", "吧", "呢", "啦", "哦") + + # FunASR commonly appends a sentence particle ("停一下啊") or repeats + # the final syllable ("停一下下"). Build a small, bounded closure of + # command-only variants instead of doing substring matching, so a + # sentence such as "我刚才没有说停一下" still cannot trigger STOP. + candidates: set[str] = set() + pending = [compact] + while pending: + candidate = pending.pop() + if not candidate or candidate in candidates: + continue + candidates.add(candidate) + + for prefix in polite_prefixes: + if candidate.startswith(prefix) and len(candidate) > len(prefix): + pending.append(candidate[len(prefix) :]) + for suffix in polite_suffixes: + if candidate.endswith(suffix) and len(candidate) > len(suffix): + pending.append(candidate[: -len(suffix)]) + if len(candidate) >= 2 and candidate[-1] == candidate[-2]: + pending.append(candidate[:-1]) + + return any(candidate in phrases for candidate in candidates) + + @staticmethod + def _contains_command(compact: str, phrases: set[str]) -> bool: + """Use a literal command-anchor protocol; do not infer sentence meaning.""" + return any(phrase and phrase in compact for phrase in phrases) + + @staticmethod + def _skill_variants(compact: str) -> list[str]: + """Build bounded FunASR variants while retaining an explicit Skill verb.""" + + candidates: set[str] = set() + pending = [compact] + while pending: + candidate = pending.pop() + if not candidate or candidate in candidates: + continue + candidates.add(candidate) + + without_particles = _strip_speech_particles(candidate) + if without_particles and without_particles != candidate: + pending.append(without_particles) + + # Observed FunASR variants include ``一下下`` and omission of the + # pronoun in ``帮我找/读``. Restore only that pronoun; never invent + # a missing Skill verb such as ``找`` or ``读``. + collapsed_action = re.sub(r"一下下+", "一下", candidate) + if collapsed_action != candidate: + pending.append(collapsed_action) + if candidate.startswith("帮") and not candidate.startswith("帮我"): + pending.append("帮我" + candidate[1:]) + + for prefix in ("请你", "麻烦你", "麻烦", "请", "能不能"): + if candidate.startswith(prefix) and len(candidate) > len(prefix): + pending.append(candidate[len(prefix) :]) + polite_prefixes = ("请你", "麻烦你", "麻烦", "请", "能不能") + + def noise_rank(candidate: str) -> tuple[int, int, int, int, int, str]: + return ( + int(bool(re.search(r"一下下+", candidate))), + int(_strip_speech_particles(candidate) != candidate), + int(candidate.startswith("帮") and not candidate.startswith("帮我")), + int(candidate.startswith(polite_prefixes)), + len(candidate), + candidate, + ) + + # Regexes can match both the raw and corrected ASR text. Prefer the + # bounded, lower-noise candidate so slot extraction is deterministic. + return sorted(candidates, key=noise_rank) + + def route(self, utterance: str) -> RouteResult: + compact = normalize_text(utterance) + if not compact: + return RouteResult(ControlIntent.NONE, reason="empty") + + variants = self._skill_variants(compact) + + if self._is_explicit_command(compact, self.stop_phrases): + return RouteResult(ControlIntent.STOP_SPEECH, reason="explicit stop command") + # STOP wins if a transcript contains both anchors. + if self._contains_command(compact, self.stop_embedded_phrases): + return RouteResult(ControlIntent.STOP_SPEECH, reason="stop anchor contained") + # RESET wins over RESUME and all skills. The protocol is deliberately + # literal: callers can add a product wake name to the same sentence. + if ( + self._contains_command(compact, self.reset_embedded_phrases) + or self._is_explicit_command(compact, self.reset_phrases) + ): + return RouteResult( + ControlIntent.RESET_SESSION, + skill_id=self.registry.default_skill, + slots={}, + reason=( + "reset anchor contained" + if self._contains_command(compact, self.reset_embedded_phrases) + else "explicit reset command" + ), + ) + if self._contains_command(compact, self.resume_embedded_phrases): + return RouteResult(ControlIntent.RESUME_SPEECH, reason="resume anchor contained") + if self._is_explicit_command(compact, self.cancel_phrases): + return RouteResult(ControlIntent.CANCEL_SKILL, reason="explicit cancel command") + if self._is_explicit_command(compact, self.return_phrases): + return RouteResult( + ControlIntent.RETURN_TO_CHAT, + skill_id=self.registry.default_skill, + slots={}, + reason="explicit return-to-chat command", + ) + + for skill_id, spec in self.registry.skills.items(): + if not bool(spec.get("enabled", False)) or skill_id == self.registry.default_skill: + continue + for pattern in spec.get("activation_patterns") or []: + for candidate in variants: + match = re.fullmatch(str(pattern), candidate) + if match: + slots = { + key: _clean_slot_value(value) + for key, value in match.groupdict().items() + if value + } + return RouteResult( + ControlIntent.ACTIVATE_SKILL, + skill_id=skill_id, + slots=slots, + reason=f"matched {skill_id} pattern", + ) + phrases = [normalize_text(item) for item in spec.get("activation_phrases") or []] + if any( + candidate == phrase or candidate.startswith(phrase) + for candidate in variants + for phrase in phrases + ): + return RouteResult( + ControlIntent.ACTIVATE_SKILL, + skill_id=skill_id, + slots={}, + reason=f"matched {skill_id} phrase", + ) + + return RouteResult(ControlIntent.NONE, reason="ordinary chat") + + def make_event( + self, + route: RouteResult, + *, + event_id: int, + asr_event_id: int, + utterance: str, + created_at_ms: float | None = None, + ) -> ControlEvent: + return ControlEvent( + event_id=event_id, + intent=route.intent, + skill_id=route.skill_id, + slots=dict(route.slots or {}), + utterance=utterance, + confidence=route.confidence, + asr_event_id=asr_event_id, + created_at_ms=created_at_ms if created_at_ms is not None else time.time() * 1000, + reason=route.reason, + ) diff --git a/extensions/assistive_harness/schemas.py b/extensions/assistive_harness/schemas.py new file mode 100644 index 0000000..edad067 --- /dev/null +++ b/extensions/assistive_harness/schemas.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from enum import Enum +from typing import Any + + +class ControlIntent(str, Enum): + STOP_SPEECH = "stop_speech" + RESUME_SPEECH = "resume_speech" + RESET_SESSION = "reset_session" + CANCEL_SKILL = "cancel_skill" + RETURN_TO_CHAT = "return_to_chat" + ACTIVATE_SKILL = "activate_skill" + NONE = "none" + + +@dataclass(slots=True) +class ASREvent: + event_id: int + utterance: str + started_at_ms: float + ended_at_ms: float + final_at_ms: float + model: str + device: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(slots=True) +class ControlEvent: + event_id: int + intent: ControlIntent + utterance: str + confidence: float + asr_event_id: int + created_at_ms: float + skill_id: str | None = None + slots: dict[str, Any] = field(default_factory=dict) + reason: str = "" + system_prompt: str | None = None + prompt_path: str | None = None + prompt_sha256: str | None = None + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["type"] = "control.intent" + payload["intent"] = self.intent.value + return payload + + +@dataclass(slots=True) +class HarnessState: + current_skill: str = "idle_chat" + current_slots: dict[str, Any] = field(default_factory=dict) + session_generation: int = 0 + control_event_id: int = 0 + asr_event_id: int = 0 + restart_in_progress: bool = False + drop_output_until_listen: bool = False + speech_hold_active: bool = False + pending_skill: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) diff --git a/extensions/assistive_harness/server.py b/extensions/assistive_harness/server.py new file mode 100644 index 0000000..f9255e8 --- /dev/null +++ b/extensions/assistive_harness/server.py @@ -0,0 +1,571 @@ +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import time +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import numpy as np +import uvicorn +import yaml +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.responses import JSONResponse + +from .asr.base import ASREngine +from .asr.energy_vad import EnergyVAD, UtteranceAudio +from .asr.funasr_engine import FunASREngine +from .cv.base import CVObservation, FrameEnvelope +from .cv.pipeline import CVPipeline +from .cv.registry import CVProviderRegistry, build_provider_registry +from .echo_guard import EchoGuard +from .model_log import ModelTurnAccumulator +from .registry import SkillRegistry +from .router import RuleIntentRouter +from .schemas import ASREvent, ControlIntent +from .state_machine import HarnessController +from .telemetry import TelemetryWriter + + +def now_ms() -> float: + return time.time() * 1000.0 + + +def _decode_float32(value: str) -> np.ndarray: + raw = base64.b64decode(value, validate=True) + if len(raw) % 4: + raise ValueError("float32 payload length is not divisible by four") + return np.frombuffer(raw, dtype=" int: + self.event_counter += 1 + return self.event_counter + + def refresh_echo_speaking(self) -> None: + self.echo.set_ai_speaking(self.model_speaking or self.playback_active) + + def record_model_turn(self, turn: dict[str, Any]) -> None: + self.telemetry.write("model", turn) + self.telemetry.append_model_transcript(turn) + print( + "[AssistiveHarness][MODEL] " + f"turn={turn.get('turn_index')} role={turn.get('role')} " + f"generation={turn.get('generation')} skill={turn.get('skill_id')} " + f"audio_ms={turn.get('audio_ms')} text={turn.get('text')!r}", + flush=True, + ) + + async def recognize( + self, utterance: UtteranceAudio + ) -> dict[str, Any] | list[dict[str, Any]]: + async with self.asr_lock: + started = now_ms() + result = await asyncio.to_thread( + self.engine.transcribe, utterance.audio, self.vad.sample_rate + ) + return await self.route_transcript( + result.text, + confidence=result.confidence, + model=result.model, + device=result.device, + started_at_ms=utterance.started_at_ms, + ended_at_ms=utterance.ended_at_ms, + final_at_ms=now_ms(), + inference_started_at_ms=started, + ) + + def recognize_in_background(self, utterance: UtteranceAudio) -> None: + """Keep receiving microphone frames while a completed utterance is decoded.""" + + async def run() -> None: + try: + await self.outbound.put(await self.recognize(utterance)) + except asyncio.CancelledError: + raise + except Exception as exc: + await self.outbound.put( + { + "type": "error", + "code": "asr_failed", + "message": str(exc), + } + ) + + task = asyncio.create_task(run()) + self.background_tasks.add(task) + task.add_done_callback(self.background_tasks.discard) + + async def route_transcript( + self, + text: str, + *, + confidence: float, + model: str, + device: str, + started_at_ms: float | None = None, + ended_at_ms: float | None = None, + final_at_ms: float | None = None, + inference_started_at_ms: float | None = None, + ) -> dict[str, Any] | list[dict[str, Any]]: + finished = final_at_ms if final_at_ms is not None else now_ms() + asr_id = self.next_event_id() + asr_event = ASREvent( + event_id=asr_id, + utterance=text.strip(), + started_at_ms=started_at_ms if started_at_ms is not None else finished, + ended_at_ms=ended_at_ms if ended_at_ms is not None else finished, + final_at_ms=finished, + model=model, + device=device, + ) + self.telemetry.write("asr", asr_event.to_dict()) + transcript_payload: dict[str, Any] = { + "type": "asr.transcript", + "asr_event_id": asr_id, + "utterance": asr_event.utterance, + "confidence": confidence, + "final_at_ms": finished, + } + if inference_started_at_ms is not None: + self.telemetry.metric("asr_inference_ms", finished - inference_started_at_ms, asr_id) + + route = self.router.route(asr_event.utterance) + echo = self.echo.evaluate(asr_event.utterance, route.intent, at_ms=finished) + print( + "[AssistiveHarness][ASR] " + f"text={asr_event.utterance!r} intent={route.intent.value} " + f"echo={'allow' if echo.allow else 'drop'} reason={echo.reason}", + flush=True, + ) + self.telemetry.write( + "echo", + { + "asr_event_id": asr_id, + "allow": echo.allow, + "reason": echo.reason, + "similarity": echo.similarity, + }, + ) + if not echo.allow or route.intent is ControlIntent.NONE: + transcript_payload["suppressed"] = not echo.allow + transcript_payload["reason"] = echo.reason if not echo.allow else route.reason + return transcript_payload + + control_id = self.next_event_id() + event = self.router.make_event( + route, + event_id=control_id, + asr_event_id=asr_id, + utterance=asr_event.utterance, + created_at_ms=finished, + ) + decision = self.controller.process(event, now_ms=finished) + print( + "[AssistiveHarness][CONTROL] " + f"intent={route.intent.value} accepted={decision.accepted} " + f"action={decision.action} event={control_id}", + flush=True, + ) + payload = { + **decision.payload, + "accepted": decision.accepted, + "action": decision.action, + "decision_reason": decision.reason, + "client_id": self.client_id, + } + self.telemetry.write("control", payload) + if route.intent in { + ControlIntent.RESET_SESSION, + ControlIntent.CANCEL_SKILL, + ControlIntent.RETURN_TO_CHAT, + ControlIntent.ACTIVATE_SKILL, + }: + self.telemetry.write( + "skill", + { + "control_event_id": control_id, + "intent": route.intent.value, + "skill_id": event.skill_id, + "slots": event.slots, + "prompt_path": event.prompt_path, + "prompt_sha256": event.prompt_sha256, + "accepted": decision.accepted, + "action": decision.action, + }, + ) + if decision.accepted: + return [transcript_payload, payload] + transcript_payload["suppressed"] = True + transcript_payload["reason"] = decision.reason + return transcript_payload + + +class AssistiveHarnessService: + def __init__( + self, + config_path: str | Path, + *, + enabled: bool, + model_path: str | None = None, + allow_test_injection: bool = False, + engine: ASREngine | None = None, + cv_providers: CVProviderRegistry | None = None, + ): + self.config_path = Path(config_path).resolve() + self.config = yaml.safe_load(self.config_path.read_text(encoding="utf-8")) or {} + self.registry = SkillRegistry(self.config_path) + self.cv_config = dict(self.config.get("cv") or {}) + self.cv_providers = cv_providers or build_provider_registry( + self.config, config_dir=self.config_path.parent + ) + self.enabled = bool(enabled) + self.allow_test_injection = bool(allow_test_injection) + asr_cfg = dict(self.config.get("asr") or {}) + resolved_model_path = model_path or str(asr_cfg.get("model_path") or "") + if engine is not None: + self.engine = engine + elif self.enabled: + if not resolved_model_path: + raise ValueError("--model-path is required when the Harness is enabled") + self.engine = FunASREngine( + resolved_model_path, + device=str(asr_cfg.get("device") or "cpu"), + ) + else: + self.engine = None + run_id = time.strftime("%Y%m%d_%H%M%S") + "_" + uuid.uuid4().hex[:8] + telemetry_root = Path(__file__).resolve().parent / "runs" + self.telemetry = TelemetryWriter(telemetry_root, run_id, self.config) + self.clients: dict[str, ClientRuntime] = {} + self.started_at_ms = now_ms() + + def make_runtime(self, client_id: str) -> ClientRuntime: + if self.engine is None: + raise RuntimeError("Harness is disabled") + asr_cfg = dict(self.config.get("asr") or {}) + echo_cfg = dict(self.config.get("echo_guard") or {}) + return ClientRuntime( + client_id=client_id, + registry=self.registry, + router=RuleIntentRouter(self.registry), + engine=self.engine, + telemetry=self.telemetry, + vad=EnergyVAD( + rms_threshold=float(asr_cfg.get("rms_threshold", 0.012)), + min_speech_ms=int(asr_cfg.get("min_speech_ms", 180)), + end_silence_ms=int(asr_cfg.get("end_silence_ms", 450)), + max_utterance_ms=int(asr_cfg.get("max_utterance_ms", 8000)), + preroll_ms=int(asr_cfg.get("preroll_ms", 200)), + ), + echo=EchoGuard( + window_ms=int(echo_cfg.get("window_ms", 20000)), + similarity_threshold=float(echo_cfg.get("similarity_threshold", 0.86)), + ), + controller=HarnessController(self.registry), + allow_test_injection=self.allow_test_injection, + cv=CVPipeline( + self.cv_providers, + on_observation=lambda observation: self.telemetry.write( + "cv", observation.to_dict() + ), + on_metric=lambda name, value: self.telemetry.metric(name, value), + queue_size=int(self.cv_config.get("queue_size", 1)), + inference_timeout_ms=float( + self.cv_config.get("inference_timeout_ms", 2000) + ), + worker_name=f"assistive-cv-{client_id[:8]}", + ), + ) + + def create_app(self) -> FastAPI: + app = FastAPI(title="Assistive Voice Skill Harness", version="0.1.0") + + @app.get("/health") + async def health() -> JSONResponse: + return JSONResponse( + { + "ok": True, + "enabled": self.enabled, + "clients": len(self.clients), + "uptime_ms": now_ms() - self.started_at_ms, + "asr_loaded": bool(getattr(self.engine, "loaded", self.engine is not None)), + "config": self.config_path.name, + "cv_providers": sorted(self.cv_providers.providers), + } + ) + + @app.get("/skills") + async def skills() -> JSONResponse: + return JSONResponse( + { + "ok": True, + "default_skill": self.registry.default_skill, + "config_path": str(self.config_path), + "skills": { + skill_id: { + "enabled": self.registry.is_enabled(skill_id), + "description": str(spec.get("description") or ""), + "prompt_path": str(self.registry.prompt_path(skill_id)), + "requires_session_restart": bool( + spec.get("requires_session_restart", True) + ), + "cv_mode": str(spec.get("cv_mode") or "disabled"), + "cv_provider": str(spec.get("cv_provider") or "noop"), + } + for skill_id, spec in self.registry.skills.items() + }, + } + ) + + @app.on_event("shutdown") + async def write_shutdown_summary() -> None: + self.telemetry.write_summary( + { + "enabled": self.enabled, + "uptime_ms": now_ms() - self.started_at_ms, + "connected_clients_at_shutdown": len(self.clients), + "asr_loaded": bool(getattr(self.engine, "loaded", self.engine is not None)), + } + ) + + @app.websocket("/ws/control") + async def control_socket(websocket: WebSocket) -> None: + if not self.enabled: + await websocket.close(code=1013, reason="Harness disabled") + return + await websocket.accept() + client_id = websocket.query_params.get("client_id") or uuid.uuid4().hex + runtime = self.make_runtime(client_id) + self.clients[client_id] = runtime + + async def send_outbound() -> None: + while True: + response = await runtime.outbound.get() + if isinstance(response, list): + for item in response: + await websocket.send_json(item) + elif response is not None: + await websocket.send_json(response) + + sender = asyncio.create_task(send_outbound()) + await runtime.outbound.put( + { + "type": "harness.ready", + "client_id": client_id, + "state": runtime.controller.state.to_dict(), + } + ) + try: + while True: + message = await websocket.receive_json() + response = await self._handle_message(runtime, message) + if response is not None: + await runtime.outbound.put(response) + except WebSocketDisconnect: + runtime.controller.mark_disconnected() + finally: + pending_turn = runtime.model_turns.flush() + if pending_turn is not None: + runtime.record_model_turn(pending_turn) + sender.cancel() + for task in tuple(runtime.background_tasks): + task.cancel() + await runtime.cv.close() + await asyncio.gather(sender, *runtime.background_tasks, return_exceptions=True) + self.clients.pop(client_id, None) + + return app + + async def _handle_message( + self, runtime: ClientRuntime, message: dict[str, Any] + ) -> dict[str, Any] | list[dict[str, Any]] | None: + message_type = str(message.get("type") or "") + if message_type == "ping": + return {"type": "pong", "at_ms": now_ms()} + if message_type == "audio.mirror": + audio = _decode_float32(str(message.get("audio_b64") or "")) + utterance = runtime.vad.feed(audio, float(message.get("started_at_ms") or now_ms())) + if utterance is None: + return None + runtime.recognize_in_background(utterance) + return None + if message_type == "asr.inject": + if not runtime.allow_test_injection: + return {"type": "error", "code": "test_injection_disabled"} + return await runtime.route_transcript( + str(message.get("text") or ""), + confidence=1.0, + model="injected-test-only", + device="none", + ) + if message_type == "model.state": + event = dict(message) + runtime.telemetry.write("model", event) + speaking = str(message.get("state") or "") == "speak" + runtime.model_speaking = speaking + runtime.refresh_echo_speaking() + text = str(message.get("text") or "") + if speaking and text: + runtime.echo.note_model_text(text) + for turn in runtime.model_turns.feed(event): + runtime.record_model_turn(turn) + return None + if message_type == "playback.state": + event = dict(message) + runtime.telemetry.write("model", event) + runtime.playback_active = bool(message.get("active")) + runtime.refresh_echo_speaking() + return None + if message_type == "session.state": + event = dict(message) + runtime.telemetry.write("session", event) + phase = str(message.get("phase") or "") + if phase == "listen": + runtime.controller.mark_listen_fence() + elif phase == "restart_complete": + runtime.controller.mark_restart_complete( + str(message.get("skill_id") or runtime.registry.default_skill), + dict(message.get("slots") or {}), + int(message.get("generation") or 0), + ) + return None + if message_type == "control.ack": + if bool(message.get("ok")) and str(message.get("intent") or "") == "stop_speech": + # Browser STOP has already flushed/blocked playback. Do not + # leave EchoGuard in a stale speaking state while the user + # immediately issues the next explicit command. + runtime.model_speaking = False + runtime.playback_active = False + runtime.refresh_echo_speaking() + runtime.telemetry.write("session", dict(message)) + print( + "[AssistiveHarness][ACK] " + f"intent={message.get('intent')} ok={message.get('ok')} " + f"generation={message.get('generation')} " + f"session={message.get('old_session_id') or '-'}" + f"->{message.get('new_session_id') or '-'} " + f"cleanup={message.get('cleanup_mode') or '-'} " + f"restart_ms={message.get('restart_latency_ms') or '-'} " + f"stale_text={message.get('dropped_old_text') or 0} " + f"stale_audio={message.get('dropped_old_audio') or 0}", + flush=True, + ) + metric = message.get("restart_latency_ms") + if isinstance(metric, (int, float)): + runtime.telemetry.metric( + "restart_latency_ms", float(metric), int(message.get("event_id") or 0) + ) + return None + if message_type == "frame.shadow": + at_ms = float(message.get("timestamp_ms") or now_ms()) + max_fps = max(0.0, float(self.cv_config.get("max_fps", 1))) + if max_fps <= 0: + return None + if at_ms - runtime.last_frame_ms < 1000.0 / max_fps: + return None + skill_id = runtime.controller.state.current_skill + skill_spec = runtime.registry.get(skill_id) + mode = str(skill_spec.get("cv_mode") or self.cv_config.get("mode") or "disabled") + if mode == "disabled": + return None + provider_id = str(skill_spec.get("cv_provider") or "noop") + frame_b64 = str(message.get("jpeg_b64") or "") + frame_id = str(message.get("frame_id") or uuid.uuid4().hex) + try: + frame = base64.b64decode(frame_b64, validate=True) if frame_b64 else None + except Exception as exc: + runtime.telemetry.write( + "cv", + CVObservation( + frame_id=frame_id, + timestamp_ms=at_ms, + skill_id=skill_id, + provider=f"{mode}:{provider_id}", + values={"status": "error"}, + error=f"InvalidFramePayload: {exc}", + ).to_dict(), + ) + return None + runtime.last_frame_ms = at_ms + runtime.cv.submit( + FrameEnvelope( + frame=frame, + frame_id=frame_id, + timestamp_ms=at_ms, + skill_id=skill_id, + slots=dict(runtime.controller.state.current_slots), + mode=mode, + provider_id=provider_id, + ) + ) + return None + return {"type": "error", "code": "unknown_message", "message_type": message_type} + + +def build_parser() -> argparse.ArgumentParser: + default_config = Path(__file__).resolve().parent / "config" / "skills.example.yaml" + parser = argparse.ArgumentParser(description="Optional local assistive voice-skill Harness") + parser.add_argument("--config", default=str(default_config)) + parser.add_argument("--enabled", action="store_true", help="explicit opt-in; default is off") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8021) + parser.add_argument("--model-path", default=None) + parser.add_argument("--allow-test-injection", action="store_true") + parser.add_argument("--certfile", default=None) + parser.add_argument("--keyfile", default=None) + return parser + + +def main() -> None: + args = build_parser().parse_args() + service = AssistiveHarnessService( + args.config, + enabled=args.enabled, + model_path=args.model_path, + allow_test_injection=args.allow_test_injection, + ) + if isinstance(service.engine, FunASREngine): + warmup_started = time.perf_counter() + print("[AssistiveHarness] loading and warming FunASR before accepting clients...") + service.engine.warm_up() + print( + "[AssistiveHarness] FunASR ready " + f"({time.perf_counter() - warmup_started:.2f}s)" + ) + uvicorn.run( + service.create_app(), + host=args.host, + port=args.port, + ssl_certfile=args.certfile, + ssl_keyfile=args.keyfile, + ) + + +if __name__ == "__main__": + main() diff --git a/extensions/assistive_harness/state_machine.py b/extensions/assistive_harness/state_machine.py new file mode 100644 index 0000000..e0cd9ce --- /dev/null +++ b/extensions/assistive_harness/state_machine.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any + +from .registry import RegistryError, SkillRegistry +from .schemas import ControlEvent, ControlIntent, HarnessState + + +@dataclass(slots=True) +class ControlDecision: + accepted: bool + action: str + reason: str + payload: dict[str, Any] + + +class HarnessController: + def __init__(self, registry: SkillRegistry): + self.registry = registry + self.state = HarnessState(current_skill=registry.default_skill) + self._seen_asr_ids: set[int] = set() + self._last_skill_activation_ms: dict[tuple[str, tuple[tuple[str, str], ...]], float] = {} + + @staticmethod + def _skill_key(skill_id: str, slots: dict[str, Any]) -> tuple[str, tuple[tuple[str, str], ...]]: + return skill_id, tuple(sorted((str(key), str(value)) for key, value in slots.items())) + + def process(self, event: ControlEvent, now_ms: float | None = None) -> ControlDecision: + now = now_ms if now_ms is not None else time.time() * 1000 + if event.asr_event_id in self._seen_asr_ids: + return ControlDecision(False, "ignore", "duplicate_asr_event", {}) + self._seen_asr_ids.add(event.asr_event_id) + self.state.asr_event_id = max(self.state.asr_event_id, event.asr_event_id) + self.state.control_event_id = max(self.state.control_event_id, event.event_id) + + if event.intent is ControlIntent.STOP_SPEECH: + self.state.speech_hold_active = True + self.state.drop_output_until_listen = True + return ControlDecision(True, "stop_speech", "highest_priority", event.to_dict()) + + if event.intent is ControlIntent.RESUME_SPEECH: + self.state.speech_hold_active = False + self.state.drop_output_until_listen = False + return ControlDecision(True, "resume_speech", "explicit_resume", event.to_dict()) + + if event.intent is ControlIntent.RESET_SESSION: + rendered = self.registry.render(self.registry.default_skill, {}) + event.skill_id = self.registry.default_skill + event.slots = {} + event.system_prompt = rendered.text + event.prompt_path = rendered.path + event.prompt_sha256 = rendered.sha256 + self.state.restart_in_progress = True + self.state.pending_skill = None + return ControlDecision(True, "restart_session", "explicit_reset", event.to_dict()) + + if event.intent in {ControlIntent.CANCEL_SKILL, ControlIntent.RETURN_TO_CHAT}: + event.skill_id = self.registry.default_skill + event.slots = {} + + if event.intent in { + ControlIntent.CANCEL_SKILL, + ControlIntent.RETURN_TO_CHAT, + ControlIntent.ACTIVATE_SKILL, + }: + skill_id = event.skill_id or self.registry.default_skill + slots = dict(event.slots or {}) + try: + rendered = self.registry.render(skill_id, slots) + except RegistryError as exc: + return ControlDecision(False, "clarify", str(exc), event.to_dict()) + + if self.state.restart_in_progress: + self.state.pending_skill = {"skill_id": skill_id, "slots": slots} + return ControlDecision(True, "queue_skill", "restart_in_progress_last_write_wins", event.to_dict()) + + key = self._skill_key(skill_id, slots) + last = self._last_skill_activation_ms.get(key) + cooldown = self.registry.cooldown_ms(skill_id) + if skill_id == self.state.current_skill and slots == self.state.current_slots: + return ControlDecision(False, "ignore", "same_skill_same_slots", event.to_dict()) + if last is not None and now - last < cooldown: + return ControlDecision(False, "ignore", "skill_cooldown", event.to_dict()) + + self._last_skill_activation_ms[key] = now + event.system_prompt = rendered.text + event.prompt_path = rendered.path + event.prompt_sha256 = rendered.sha256 + self.state.restart_in_progress = True + return ControlDecision(True, "activate_skill", "skill_switch", event.to_dict()) + + return ControlDecision(False, "pass_chat", "ordinary_chat", event.to_dict()) + + def mark_restart_complete(self, skill_id: str, slots: dict[str, Any], generation: int) -> dict[str, Any] | None: + self.state.current_skill = skill_id + self.state.current_slots = dict(slots) + self.state.session_generation = int(generation) + self.state.restart_in_progress = False + self.state.speech_hold_active = False + self.state.drop_output_until_listen = False + pending = self.state.pending_skill + self.state.pending_skill = None + return pending + + def mark_listen_fence(self) -> None: + if not self.state.speech_hold_active: + self.state.drop_output_until_listen = False + + def mark_disconnected(self) -> None: + self.state.restart_in_progress = False + self.state.pending_skill = None + self.state.speech_hold_active = False + self.state.drop_output_until_listen = False diff --git a/extensions/assistive_harness/summarize_reset_run.py b/extensions/assistive_harness/summarize_reset_run.py new file mode 100644 index 0000000..1e8ce19 --- /dev/null +++ b/extensions/assistive_harness/summarize_reset_run.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + + +def nearest_rank(values: list[float], quantile: float) -> float | None: + if not values: + return None + ordered = sorted(values) + index = max(0, math.ceil(len(ordered) * quantile) - 1) + return ordered[index] + + +def load_events(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + events: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + continue + return events + + +def resolve_run(value: str | None) -> Path: + runs_root = Path(__file__).resolve().parent / "runs" + if value: + candidate = Path(value).expanduser().resolve() + if not candidate.is_dir(): + raise SystemExit(f"Run directory does not exist: {candidate}") + return candidate + candidates = [path for path in runs_root.iterdir() if path.is_dir()] + if not candidates: + raise SystemExit(f"No run directories found under {runs_root}") + return max(candidates, key=lambda path: path.stat().st_mtime) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Summarize real browser RESET telemetry") + parser.add_argument("--run", help="specific run directory; defaults to the newest run") + args = parser.parse_args() + + run_dir = resolve_run(args.run) + events = load_events(run_dir / "session_events.jsonl") + resets = [ + event + for event in events + if event.get("type") == "control.ack" + and event.get("intent") == "reset_session" + ] + successes = [event for event in resets if event.get("ok") is True] + latencies = [ + float(event["restart_latency_ms"]) + for event in successes + if isinstance(event.get("restart_latency_ms"), (int, float)) + ] + changed_ids = [ + event + for event in successes + if event.get("old_session_id") + and event.get("new_session_id") + and event.get("old_session_id") != event.get("new_session_id") + ] + + print(f"Run: {run_dir}") + print(f"RESET success: {len(successes)}/{len(resets)}") + print(f"Session ID changed: {len(changed_ids)}/{len(successes)}") + if latencies: + print(f"Restart P50: {nearest_rank(latencies, 0.50):.1f} ms") + print(f"Restart P90: {nearest_rank(latencies, 0.90):.1f} ms") + print(f"Restart max: {max(latencies):.1f} ms") + else: + print("Restart P50/P90: no RESET latency samples") + print( + "Dropped stale callbacks (cumulative): " + f"text={max((int(event.get('dropped_old_text') or 0) for event in successes), default=0)}, " + f"audio={max((int(event.get('dropped_old_audio') or 0) for event in successes), default=0)}" + ) + print("Audible old-output pollution: manual observation required") + for index, event in enumerate(successes, start=1): + print( + f"{index:02d}. generation={event.get('generation')} " + f"session={event.get('old_session_id') or '-'}->{event.get('new_session_id') or '-'} " + f"cleanup={event.get('cleanup_mode') or '-'} " + f"latency={float(event.get('restart_latency_ms') or 0):.1f} ms" + ) + + +if __name__ == "__main__": + main() diff --git a/extensions/assistive_harness/telemetry.py b/extensions/assistive_harness/telemetry.py new file mode 100644 index 0000000..2e3aa18 --- /dev/null +++ b/extensions/assistive_harness/telemetry.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import csv +import json +import threading +import time +from pathlib import Path +from typing import Any + +import yaml + + +class TelemetryWriter: + FILES = { + "asr": "asr_events.jsonl", + "control": "control_events.jsonl", + "session": "session_events.jsonl", + "skill": "skill_events.jsonl", + "cv": "cv_events.jsonl", + "echo": "echo_events.jsonl", + "model": "model_events.jsonl", + } + + def __init__(self, root: str | Path, run_id: str, config: dict[str, Any]): + self.run_id = run_id + self.run_dir = Path(root) / run_id + self.run_dir.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + (self.run_dir / "config_snapshot.yaml").write_text( + yaml.safe_dump(config, allow_unicode=True, sort_keys=True), + encoding="utf-8", + ) + self._metrics_path = self.run_dir / "metrics.csv" + with self._metrics_path.open("w", newline="", encoding="utf-8") as handle: + csv.writer(handle).writerow(["timestamp_ms", "metric", "value", "event_id"]) + + def write(self, stream: str, payload: dict[str, Any]) -> None: + filename = self.FILES[stream] + record = {"logged_at_ms": time.time() * 1000, **payload} + line = json.dumps(record, ensure_ascii=False, sort_keys=True) + with self._lock: + with (self.run_dir / filename).open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + + def metric(self, name: str, value: float, event_id: int | None = None) -> None: + with self._lock: + with self._metrics_path.open("a", newline="", encoding="utf-8") as handle: + csv.writer(handle).writerow([time.time() * 1000, name, value, event_id or ""]) + + def write_summary(self, summary: dict[str, Any]) -> None: + (self.run_dir / "summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + + def append_model_transcript(self, turn: dict[str, Any]) -> None: + text = str(turn.get("text") or "").replace("\r", " ").replace("\n", " ") + line = ( + f"turn={turn.get('turn_index')} role={turn.get('role')} " + f"generation={turn.get('generation')} skill={turn.get('skill_id')} " + f"audio_ms={turn.get('audio_ms')} text={text}\n" + ) + with self._lock: + with (self.run_dir / "model_transcript.txt").open( + "a", encoding="utf-8" + ) as handle: + handle.write(line) diff --git a/extensions/assistive_harness/tests/__init__.py b/extensions/assistive_harness/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/extensions/assistive_harness/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/extensions/assistive_harness/tests/test_core.py b/extensions/assistive_harness/tests/test_core.py new file mode 100644 index 0000000..83066c9 --- /dev/null +++ b/extensions/assistive_harness/tests/test_core.py @@ -0,0 +1,402 @@ +from __future__ import annotations + +import asyncio +import tempfile +import time +import unittest +from pathlib import Path + +import numpy as np + +from extensions.assistive_harness.asr.energy_vad import EnergyVAD +from extensions.assistive_harness.cv.noop import ShadowCVProvider +from extensions.assistive_harness.cv.base import CVObservation, FrameEnvelope +from extensions.assistive_harness.cv.pipeline import CVPipeline +from extensions.assistive_harness.cv.registry import CVProviderRegistry +from extensions.assistive_harness.echo_guard import EchoGuard +from extensions.assistive_harness.model_log import ModelTurnAccumulator +from extensions.assistive_harness.registry import RegistryError, SkillRegistry +from extensions.assistive_harness.router import RuleIntentRouter +from extensions.assistive_harness.schemas import ControlIntent +from extensions.assistive_harness.state_machine import HarnessController + + +CONFIG = Path(__file__).resolve().parents[1] / "config" / "skills.example.yaml" + + +class RouterCorpusTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.registry = SkillRegistry(CONFIG) + cls.router = RuleIntentRouter(cls.registry) + + def test_control_and_skill_corpus_over_fifty_utterances(self) -> None: + corpus = [ + ("停一下", ControlIntent.STOP_SPEECH, None), + ("停一下啊", ControlIntent.STOP_SPEECH, None), + ("停一下下", ControlIntent.STOP_SPEECH, None), + ("同一下", ControlIntent.STOP_SPEECH, None), + ("同一下下", ControlIntent.STOP_SPEECH, None), + ("等一下", ControlIntent.STOP_SPEECH, None), + ("等一一下", ControlIntent.STOP_SPEECH, None), + ("请停一下啊", ControlIntent.STOP_SPEECH, None), + ("请停一下", ControlIntent.STOP_SPEECH, None), + ("麻烦你先停一下", ControlIntent.STOP_SPEECH, None), + ("别说了", ControlIntent.STOP_SPEECH, None), + ("请你别说了好吗", ControlIntent.STOP_SPEECH, None), + ("闭嘴", ControlIntent.STOP_SPEECH, None), + ("停止播报", ControlIntent.STOP_SPEECH, None), + ("安静", ControlIntent.STOP_SPEECH, None), + ("你先停一下，我有个问题", ControlIntent.STOP_SPEECH, None), + ("说得有点长了麻烦停一下吧", ControlIntent.STOP_SPEECH, None), + ("现在可以先停一下然后听我说吗", ControlIntent.STOP_SPEECH, None), + ("不停停一下继续介绍", ControlIntent.STOP_SPEECH, None), + ("我刚才没有说停一下", ControlIntent.STOP_SPEECH, None), + ("请不要停一下，继续介绍", ControlIntent.STOP_SPEECH, None), + ("停一下是什么意思", ControlIntent.STOP_SPEECH, None), + ("恢复对话", ControlIntent.RESUME_SPEECH, None), + ("好了现在恢复对话吧", ControlIntent.RESUME_SPEECH, None), + ("乐奇恢复对话", ControlIntent.RESUME_SPEECH, None), + ("停一下然后恢复对话", ControlIntent.STOP_SPEECH, None), + ("重新开始", ControlIntent.RESET_SESSION, "idle_chat"), + ("请重新开始", ControlIntent.RESET_SESSION, "idle_chat"), + ("重置会话", ControlIntent.NONE, None), + ("乐奇请重新开始", ControlIntent.RESET_SESSION, "idle_chat"), + ("你有点卡了请重新开始会话", ControlIntent.RESET_SESSION, "idle_chat"), + ("请现在重置会话然后听我说", ControlIntent.NONE, None), + ("停一下然后重新开始", ControlIntent.STOP_SPEECH, None), + ("恢复对话然后重新开始", ControlIntent.RESET_SESSION, "idle_chat"), + ("新建会话", ControlIntent.NONE, None), + ("清空上下文", ControlIntent.NONE, None), + ("回到聊天", ControlIntent.RETURN_TO_CHAT, "idle_chat"), + ("请回到聊天", ControlIntent.RETURN_TO_CHAT, "idle_chat"), + ("回到普通聊天", ControlIntent.RETURN_TO_CHAT, "idle_chat"), + ("回到普通聊天天", ControlIntent.RETURN_TO_CHAT, "idle_chat"), + ("退出技能", ControlIntent.RETURN_TO_CHAT, "idle_chat"), + ("普通聊天", ControlIntent.RETURN_TO_CHAT, "idle_chat"), + ("取消任务", ControlIntent.CANCEL_SKILL, None), + ("不找了", ControlIntent.CANCEL_SKILL, None), + ("不读了", ControlIntent.CANCEL_SKILL, None), + ("帮我找手机", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("帮我找一下我的手机", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("帮我找一下手机", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("帮我找一下下我的手机嗯", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("帮找一下手机手机", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("请帮我找钥匙", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("水杯在哪", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("书在哪里", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("看到门卡了吗", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("有没有雨伞", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("麻烦帮我找眼镜", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("请问钱包在哪", ControlIntent.ACTIVATE_SKILL, "find_object"), + ("读一下", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("读一下这行字", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("请读一下", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("帮我读一下这个杯子上的字母母母", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("帮读一下这个水杯上面的字字个", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("帮我识字", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("上面写了什么", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("这是什么字", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("读文字", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("请你读文字", ControlIntent.ACTIVATE_SKILL, "read_text"), + ("描述一下", ControlIntent.ACTIVATE_SKILL, "describe_scene"), + ("帮我描述一下", ControlIntent.ACTIVATE_SKILL, "describe_scene"), + ("看看周围", ControlIntent.ACTIVATE_SKILL, "describe_scene"), + ("帮我避障", ControlIntent.ACTIVATE_SKILL, "obstacle_avoidance"), + ("前面有障碍吗", ControlIntent.ACTIVATE_SKILL, "obstacle_avoidance"), + ("今天天气怎么样", ControlIntent.NONE, None), + ("停止是一个动词", ControlIntent.NONE, None), + ("他说让我闭嘴但我没同意", ControlIntent.NONE, None), + ("重新开始这个词怎么翻译", ControlIntent.RESET_SESSION, "idle_chat"), + ("我在读一本书", ControlIntent.NONE, None), + ("你觉得这杯水怎么样", ControlIntent.NONE, None), + ("普通聊天机器人是什么", ControlIntent.NONE, None), + ("请介绍一下上海", ControlIntent.NONE, None), + ("我不想清空上下文因为还有用", ControlIntent.NONE, None), + ("帮我分析这段话", ControlIntent.NONE, None), + ("帮我一下手机手机", ControlIntent.NONE, None), + ("能听见我吗", ControlIntent.NONE, None), + ("今天周几", ControlIntent.NONE, None), + ("为什么会这样", ControlIntent.NONE, None), + ("继续说", ControlIntent.NONE, None), + ("接着说", ControlIntent.NONE, None), + ("你继续说话吧", ControlIntent.NONE, None), + ("谢谢", ControlIntent.NONE, None), + ("你好", ControlIntent.NONE, None), + ("左边有什么", ControlIntent.NONE, None), + ("给我讲个笑话", ControlIntent.NONE, None), + ("", ControlIntent.NONE, None), + ("。", ControlIntent.NONE, None), + ("安静是一种状态", ControlIntent.NONE, None), + ("取消任务是不是一个按钮", ControlIntent.NONE, None), + ("有人说不读了然后离开", ControlIntent.NONE, None), + ] + self.assertGreaterEqual(len(corpus), 50) + for utterance, expected_intent, expected_skill in corpus: + with self.subTest(utterance=utterance): + result = self.router.route(utterance) + self.assertEqual(result.intent, expected_intent) + self.assertEqual(result.skill_id, expected_skill) + + def test_find_target_slot(self) -> None: + result = self.router.route("请帮我找深绿色的书") + self.assertEqual(result.slots, {"target": "深绿色的书"}) + + natural = self.router.route("帮我找一下我的手机") + self.assertEqual(natural.slots, {"target": "手机"}) + + repeated = self.router.route("帮我找一下下我的手机嗯") + self.assertEqual(repeated.slots, {"target": "手机"}) + + missing_pronoun = self.router.route("帮找一下手机手机") + self.assertEqual(missing_pronoun.slots, {"target": "手机"}) + + +class RegistryAndStateTests(unittest.TestCase): + def setUp(self) -> None: + self.registry = SkillRegistry(CONFIG) + self.router = RuleIntentRouter(self.registry) + self.controller = HarnessController(self.registry) + + def event(self, text: str, event_id: int, asr_id: int): + return self.router.make_event( + self.router.route(text), event_id=event_id, asr_event_id=asr_id, + utterance=text, created_at_ms=float(event_id * 1000), + ) + + def test_prompt_render_and_hash(self) -> None: + prompt = self.registry.render("find_object", {"target": "手机"}) + self.assertIn("当前寻找目标：手机", prompt.text) + self.assertIn("智能眼镜找物助手", prompt.text) + self.assertEqual(len(prompt.sha256), 64) + self.assertIn("手机", self.registry.task_trigger("find_object", {"target": "手机"})) + + def test_aaai_skill_prompt_files_are_user_editable_and_registered(self) -> None: + find_prompt = self.registry.render("find_object", {"target": "水杯"}) + read_prompt = self.registry.render("read_text", {}) + self.assertTrue(Path(find_prompt.path).is_file()) + self.assertTrue(Path(read_prompt.path).is_file()) + self.assertIn("当前寻找目标：水杯", find_prompt.text) + self.assertIn("智能眼镜读字助手", read_prompt.text) + self.assertTrue(self.registry.is_enabled("describe_scene")) + self.assertTrue(self.registry.is_enabled("obstacle_avoidance")) + self.assertTrue( + Path(self.registry.get("obstacle_avoidance")["prompt_file"]).name + == "obstacle_avoidance_zh.txt" + ) + + def test_required_slot_validation(self) -> None: + with self.assertRaises(RegistryError): + self.registry.render("find_object", {}) + + def test_unknown_skill_is_rejected(self) -> None: + with self.assertRaises(RegistryError): + self.registry.render("not_registered", {}) + + def test_stop_hold_requires_explicit_resume(self) -> None: + decision = self.controller.process(self.event("停一下", 1, 1)) + self.assertEqual(decision.action, "stop_speech") + self.assertTrue(self.controller.state.speech_hold_active) + self.assertTrue(self.controller.state.drop_output_until_listen) + self.controller.mark_listen_fence() + self.assertTrue(self.controller.state.speech_hold_active) + self.assertTrue(self.controller.state.drop_output_until_listen) + resumed = self.controller.process(self.event("恢复对话", 2, 2)) + self.assertEqual(resumed.action, "resume_speech") + self.assertFalse(self.controller.state.speech_hold_active) + self.assertFalse(self.controller.state.drop_output_until_listen) + + def test_reset_always_supplies_idle_prompt(self) -> None: + decision = self.controller.process(self.event("重新开始", 1, 1)) + self.assertEqual(decision.action, "restart_session") + self.assertIn("AI 眼镜助手", decision.payload["system_prompt"]) + self.assertTrue(decision.payload["prompt_path"].endswith("idle_chat_zh.txt")) + + def test_skill_activation_supplies_rendered_prompt_path_and_hash(self) -> None: + decision = self.controller.process(self.event("帮我找手机", 1, 1)) + self.assertEqual(decision.action, "activate_skill") + self.assertIn("当前寻找目标：手机", decision.payload["system_prompt"]) + self.assertTrue(decision.payload["prompt_path"].endswith("find_object_zh.txt")) + self.assertEqual(len(decision.payload["prompt_sha256"]), 64) + + def test_duplicate_asr_event_is_ignored(self) -> None: + first = self.controller.process(self.event("停一下", 1, 9)) + second = self.controller.process(self.event("停一下", 2, 9)) + self.assertTrue(first.accepted) + self.assertEqual(second.reason, "duplicate_asr_event") + + def test_same_skill_same_slots_does_not_restart(self) -> None: + first = self.controller.process(self.event("帮我找手机", 1, 1)) + self.assertTrue(first.accepted) + self.controller.mark_restart_complete("find_object", {"target": "手机"}, 1) + second = self.controller.process(self.event("帮我找手机", 2, 2), now_ms=9000) + self.assertFalse(second.accepted) + self.assertEqual(second.reason, "same_skill_same_slots") + + def test_restart_pending_is_last_write_wins(self) -> None: + self.controller.process(self.event("帮我找手机", 1, 1)) + queued_one = self.controller.process(self.event("帮我找钥匙", 2, 2)) + queued_two = self.controller.process(self.event("读一下", 3, 3)) + self.assertEqual(queued_one.action, "queue_skill") + self.assertEqual(queued_two.action, "queue_skill") + pending = self.controller.mark_restart_complete("find_object", {"target": "手机"}, 1) + self.assertEqual(pending, {"skill_id": "read_text", "slots": {}}) + + def test_disconnect_clears_restart_and_pending(self) -> None: + self.controller.process(self.event("帮我找手机", 1, 1)) + self.controller.process(self.event("读一下", 2, 2)) + self.controller.mark_disconnected() + self.assertFalse(self.controller.state.restart_in_progress) + self.assertIsNone(self.controller.state.pending_skill) + + +class EchoAndVADTests(unittest.TestCase): + def test_recent_model_echo_is_blocked_but_stop_is_allowed(self) -> None: + guard = EchoGuard(window_ms=20_000, similarity_threshold=0.8) + guard.note_model_text("请告诉我还需要什么帮助", at_ms=1000) + blocked = guard.evaluate("请告诉我还需要什么帮助", ControlIntent.NONE, at_ms=1100) + self.assertFalse(blocked.allow) + guard.set_ai_speaking(True) + allowed = guard.evaluate("停一下", ControlIntent.STOP_SPEECH, at_ms=1200) + self.assertTrue(allowed.allow) + resumed = guard.evaluate("恢复对话", ControlIntent.RESUME_SPEECH, at_ms=1300) + self.assertTrue(resumed.allow) + + skill = guard.evaluate("帮我找一下我的手机", ControlIntent.ACTIVATE_SKILL, at_ms=1400) + self.assertTrue(skill.allow) + returned = guard.evaluate("回到普通聊天", ControlIntent.RETURN_TO_CHAT, at_ms=1450) + self.assertTrue(returned.allow) + cancelled = guard.evaluate("取消任务", ControlIntent.CANCEL_SKILL, at_ms=1475) + self.assertTrue(cancelled.allow) + ordinary = guard.evaluate("今天天气怎么样", ControlIntent.NONE, at_ms=1500) + self.assertFalse(ordinary.allow) + self.assertEqual(ordinary.reason, "ordinary_skill_suppressed_while_ai_speaking") + + def test_vad_emits_one_utterance(self) -> None: + vad = EnergyVAD(rms_threshold=0.01, min_speech_ms=100, end_silence_ms=200) + frames = [np.zeros(1600, np.float32)] + frames += [np.full(1600, 0.1, np.float32) for _ in range(3)] + frames += [np.zeros(1600, np.float32) for _ in range(3)] + utterances = [] + for index, frame in enumerate(frames): + result = vad.feed(frame, index * 100.0) + if result is not None: + utterances.append(result) + self.assertEqual(len(utterances), 1) + self.assertGreater(utterances[0].audio.size, 0) + + +class ModelTurnLogTests(unittest.TestCase): + def test_streamed_fragments_are_aggregated_with_audio_duration(self) -> None: + turns = ModelTurnAccumulator() + first = turns.feed( + { + "state": "speak", + "session_id": "s1", + "generation": 2, + "skill_id": "read_text", + "text": "上海", + "audio_ms": 120, + } + ) + self.assertEqual(first, []) + second = turns.feed( + { + "state": "speak", + "session_id": "s1", + "generation": 2, + "skill_id": "read_text", + "text": "电力", + "audio_ms": 180, + "decode_end": True, + } + ) + self.assertEqual(second, []) + completed = turns.feed( + { + "state": "listen", + "session_id": "s1", + "generation": 2, + "skill_id": "read_text", + "end_of_turn": True, + } + ) + self.assertEqual(completed[0]["text"], "上海电力") + self.assertEqual(completed[0]["audio_ms"], 300.0) + + +class CVIsolationTests(unittest.IsolatedAsyncioTestCase): + async def test_shadow_provider_contains_failures(self) -> None: + class Broken: + def analyze(self, *args, **kwargs): + raise RuntimeError("synthetic CV failure") + + result = ShadowCVProvider(Broken(), "broken").analyze( + None, "f1", 1.0, "find_object", {} + ) + self.assertEqual(result.provider, "shadow:broken") + self.assertIn("synthetic CV failure", result.error or "") + + async def test_pipeline_is_non_blocking_and_latest_frame_wins(self) -> None: + observations: list[CVObservation] = [] + + class Slow: + def analyze(self, frame, frame_id, timestamp_ms, skill_id, slots): + time.sleep(0.08) + return CVObservation( + frame_id=frame_id, + timestamp_ms=timestamp_ms, + skill_id=skill_id, + provider="slow", + ) + + pipeline = CVPipeline( + CVProviderRegistry({"slow": Slow()}), + on_observation=observations.append, + inference_timeout_ms=500, + ) + pipeline.submit(FrameEnvelope(b"1", "f1", 1.0, "find_object", {}, "shadow", "slow")) + # submit() must return before the synchronous provider completes. A + # wall-clock threshold is flaky on a loaded Windows workstation. + self.assertEqual(observations, []) + self.assertIsNotNone(pipeline.worker_task) + await asyncio.sleep(0.01) + pipeline.submit(FrameEnvelope(b"2", "f2", 2.0, "find_object", {}, "shadow", "slow")) + pipeline.submit(FrameEnvelope(b"3", "f3", 3.0, "find_object", {}, "shadow", "slow")) + for _ in range(100): + if len(observations) >= 2: + break + await asyncio.sleep(0.01) + snapshot = pipeline.snapshot() + await pipeline.close() + self.assertEqual([item.frame_id for item in observations], ["f1", "f3"]) + self.assertEqual(snapshot["dropped_frames"], 1) + + async def test_pipeline_timeout_is_observed_without_escaping(self) -> None: + observations: list[CVObservation] = [] + + class TooSlow: + def analyze(self, frame, frame_id, timestamp_ms, skill_id, slots): + time.sleep(0.08) + return CVObservation(frame_id, timestamp_ms, skill_id, "too_slow") + + pipeline = CVPipeline( + CVProviderRegistry({"too_slow": TooSlow()}), + on_observation=observations.append, + inference_timeout_ms=10, + ) + pipeline.submit( + FrameEnvelope(b"x", "timeout", 1.0, "find_object", {}, "shadow", "too_slow") + ) + for _ in range(50): + if observations: + break + await asyncio.sleep(0.005) + snapshot = pipeline.snapshot() + await pipeline.close() + self.assertIn("TimeoutError", observations[0].error or "") + self.assertEqual(snapshot["timeout_count"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/assistive_harness/tests/test_cv_yolo.py b/extensions/assistive_harness/tests/test_cv_yolo.py new file mode 100644 index 0000000..62c231a --- /dev/null +++ b/extensions/assistive_harness/tests/test_cv_yolo.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +import cv2 +import numpy as np + +from extensions.assistive_harness.cv.yolo_onnx import YoloOnnxProvider + + +MODEL = Path(__file__).resolve().parents[3] / "models" / "yolo26n.onnx" + + +@unittest.skipUnless(MODEL.is_file(), "local YOLO reference weights are unavailable") +class YoloOnnxSmokeTests(unittest.TestCase): + def test_blank_jpeg_runs_real_onnx_inference(self) -> None: + ok, encoded = cv2.imencode(".jpg", np.zeros((480, 640, 3), dtype=np.uint8)) + self.assertTrue(ok) + observation = YoloOnnxProvider(MODEL).analyze( + encoded.tobytes(), + "blank", + 1.0, + "find_object", + {"target": "手机"}, + ) + self.assertEqual(observation.provider, "yolo_onnx") + self.assertEqual(observation.values["status"], "ok") + self.assertEqual(observation.values["canonical_label"], "cell phone") + self.assertEqual(observation.values["model"], "yolo26n.onnx") + self.assertIsInstance(observation.values["detections"], list) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/assistive_harness/tests/test_phase_b_rokid.py b/extensions/assistive_harness/tests/test_phase_b_rokid.py new file mode 100644 index 0000000..1787b69 --- /dev/null +++ b/extensions/assistive_harness/tests/test_phase_b_rokid.py @@ -0,0 +1,352 @@ +from __future__ import annotations + +import asyncio +import base64 +import unittest +from pathlib import Path +from typing import Any + +import numpy as np + +from extensions.assistive_harness.phase_b.rokid_runtime import ( + AudioMirrorChunker, + DropOldestAudioQueue, + GatewaySessionManager, + LatestFrame, + PCSpeaker, + PhaseBRokidRuntime, + RokidRuntimeConfig, + SessionSpec, + apply_pcm16_gain, + make_session_ready_chime, + pcm16le_to_float32, +) +from extensions.assistive_harness.registry import SkillRegistry + + +CONFIG = Path(__file__).resolve().parents[1] / "config" / "skills.example.yaml" + + +class FakeSpeaker: + def __init__(self) -> None: + self.blocked = False + self.flush_count = 0 + self.resume_count = 0 + self.enqueued: list[tuple[np.ndarray, int]] = [] + + async def start(self) -> None: + pass + + async def enqueue(self, samples: np.ndarray, generation: int) -> None: + if not self.blocked: + self.enqueued.append((samples.copy(), generation)) + + async def block_and_flush(self) -> None: + self.blocked = True + self.flush_count += 1 + + async def resume(self) -> None: + self.blocked = False + self.resume_count += 1 + + async def close(self) -> None: + pass + + def pending_ms(self) -> float: + return 0.0 + + +class FakeTelemetry: + connected = True + + def __init__(self) -> None: + self.messages: list[dict[str, Any]] = [] + + async def send(self, payload: dict[str, Any]) -> None: + self.messages.append(dict(payload)) + + +class FakeSession: + next_id = 0 + + def __init__( + self, + _config: RokidRuntimeConfig, + spec: SessionSpec, + _audio_queue: DropOldestAudioQueue, + _latest_frame: LatestFrame, + _gate: object, + on_result: object, + ) -> None: + type(self).next_id += 1 + self.spec = spec + self.on_result = on_result + self.session_id = f"fake-{type(self).next_id}" + self.status = "created" + self.last_error = "" + self.started = False + self.stopped_with: str | None = None + self.injected_tasks: list[str] = [] + + async def start(self) -> None: + self.started = True + self.status = "running" + + async def stop(self, cleanup_mode: str) -> None: + self.stopped_with = cleanup_mode + self.status = "stopped" + + async def inject_task(self, text: str) -> bool: + self.injected_tasks.append(text) + return True + + +class RokidAudioBoundaryTests(unittest.IsolatedAsyncioTestCase): + def test_health_marks_device_input_not_ready_before_first_packet(self) -> None: + runtime = PhaseBRokidRuntime( + RokidRuntimeConfig(skills_config=str(CONFIG), play_audio=False), + speaker=FakeSpeaker(), + session_factory=FakeSession, + ) + self.assertFalse(runtime.health()["device_input_ready"]) + runtime.stats.audio_packets = 1 + self.assertTrue(runtime.health()["device_input_ready"]) + + def test_pcm16_conversion_and_mirror_chunking(self) -> None: + raw = np.array([-32768, 0, 32767], dtype=" None: + raw = np.array([-4_000, 1_000, 4_000], dtype=" None: + chime = make_session_ready_chime() + self.assertEqual(chime.dtype, np.float32) + self.assertGreater(chime.size, 24_000 * 0.15) + self.assertLess(chime.size, 24_000 * 0.25) + self.assertLessEqual(float(np.max(np.abs(chime))), 0.321) + self.assertGreater(float(np.max(np.abs(chime))), 0.30) + + async def test_drop_oldest_queue_preserves_newest_packets(self) -> None: + queue = DropOldestAudioQueue(max_packets=2) + queue.put_nowait(b"old") + queue.put_nowait(b"middle") + queue.put_nowait(b"new") + self.assertEqual(queue.dropped_packets, 1) + self.assertEqual(await queue.get(0.01), b"middle") + self.assertEqual(await queue.get(0.01), b"new") + + async def test_pc_speaker_stop_closes_and_resume_reopens_stream(self) -> None: + class FakeOutputStream: + def __init__(self) -> None: + self.abort_count = 0 + self.close_count = 0 + + def abort(self) -> None: + self.abort_count += 1 + + def close(self) -> None: + self.close_count += 1 + + speaker = PCSpeaker() + old_stream = FakeOutputStream() + new_stream = FakeOutputStream() + speaker._stream = old_stream + + await speaker.block_and_flush() + self.assertTrue(speaker.blocked) + self.assertIsNone(speaker._stream) + self.assertEqual(old_stream.abort_count, 1) + self.assertEqual(old_stream.close_count, 1) + + speaker._open_stream = lambda: setattr(speaker, "_stream", new_stream) + await speaker.resume() + self.assertFalse(speaker.blocked) + self.assertIs(speaker._stream, new_stream) + + +class PhaseBControlContractTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + FakeSession.next_id = 0 + self.registry = SkillRegistry(CONFIG) + self.speaker = FakeSpeaker() + self.telemetry = FakeTelemetry() + self.sessions: list[FakeSession] = [] + + def factory(*args: Any) -> FakeSession: + session = FakeSession(*args) + self.sessions.append(session) + return session + + self.manager = GatewaySessionManager( + RokidRuntimeConfig( + skills_config=str(CONFIG), + play_audio=False, + playback_echo_tail_s=0.0, + ), + self.registry, + DropOldestAudioQueue(max_packets=4), + LatestFrame(), + self.speaker, + session_factory=factory, + ) + self.manager.harness = self.telemetry + + async def test_stop_and_resume_flush_pc_output_and_ack(self) -> None: + await self.manager.start_initial() + stop_ack = await self.manager.handle_control( + { + "type": "control.intent", + "event_id": 1, + "intent": "stop_speech", + "accepted": True, + } + ) + self.assertTrue(stop_ack["ok"]) + self.assertTrue(self.manager.gate.speech_hold_active) + self.assertTrue(self.speaker.blocked) + self.assertEqual(self.speaker.flush_count, 1) + + resume_ack = await self.manager.handle_control( + { + "type": "control.intent", + "event_id": 2, + "intent": "resume_speech", + "accepted": True, + } + ) + self.assertTrue(resume_ack["ok"]) + self.assertFalse(self.manager.gate.speech_hold_active) + self.assertFalse(self.speaker.blocked) + self.assertEqual( + [message["intent"] for message in self.telemetry.messages if message.get("type") == "control.ack"], + ["stop_speech", "resume_speech"], + ) + + async def test_skill_switch_replaces_session_and_fences_old_output(self) -> None: + await self.manager.start_initial() + old_session = self.sessions[0] + ack = await self.manager.handle_control( + { + "type": "control.intent", + "event_id": 3, + "intent": "activate_skill", + "accepted": True, + "skill_id": "read_text", + "slots": {}, + } + ) + new_session = self.sessions[1] + self.assertTrue(ack["ok"]) + self.assertEqual(old_session.stopped_with, "light") + self.assertTrue(new_session.started) + self.assertNotEqual(old_session.session_id, new_session.session_id) + self.assertEqual(self.manager.gate.generation, 1) + self.assertEqual(self.manager.gate.current_skill, "read_text") + self.assertEqual( + new_session.injected_tasks, + ["请立即读取当前画面中最明显的文字，只读看到的内容。"], + ) + self.assertTrue(ack["task_trigger_sent"]) + self.assertFalse(self.speaker.blocked) + self.assertTrue( + any( + message.get("type") == "session.state" + and message.get("phase") == "restart_complete" + for message in self.telemetry.messages + ) + ) + + audio = base64.b64encode(np.ones(10, dtype=np.float32).tobytes()).decode() + await self.manager.handle_result( + old_session, + {"type": "result", "text": "stale", "audio_data": audio}, + ) + self.assertEqual(self.manager.gate.dropped_old_text, 1) + self.assertEqual(self.manager.gate.dropped_old_audio, 1) + self.assertEqual(self.speaker.enqueued, []) + + await self.manager.handle_result( + new_session, + {"type": "result", "text": "current", "audio_data": audio}, + ) + self.assertEqual(len(self.speaker.enqueued), 1) + self.assertEqual(self.speaker.enqueued[0][1], 1) + + async def test_restart_complete_plays_one_local_ready_chime(self) -> None: + self.manager.config.play_audio = True + await self.manager.start_initial() + self.assertEqual(self.speaker.enqueued, []) + + ack = await self.manager.handle_control( + { + "type": "control.intent", + "event_id": 4, + "intent": "reset_session", + "accepted": True, + "skill_id": "idle_chat", + "slots": {}, + } + ) + + self.assertTrue(ack["ok"]) + self.assertEqual(len(self.speaker.enqueued), 1) + cue, generation = self.speaker.enqueued[0] + self.assertEqual(generation, 1) + self.assertGreater(cue.size, 0) + self.assertGreater(self.manager.gate.local_cue_mute_until_mono, 0.0) + + async def test_obstacle_switch_injects_one_shot_visual_task(self) -> None: + await self.manager.start_initial() + ack = await self.manager.handle_control( + { + "type": "control.intent", + "event_id": 5, + "intent": "activate_skill", + "accepted": True, + "skill_id": "obstacle_avoidance", + "slots": {}, + } + ) + self.assertTrue(ack["task_trigger_sent"]) + self.assertIn("判断当前画面", self.sessions[1].injected_tasks[0]) + + async def test_model_log_turn_ends_on_listen_not_decode_slice(self) -> None: + await self.manager.start_initial() + session = self.sessions[0] + await self.manager.handle_result( + session, + {"type": "result", "text": "上海", "end_of_turn": True}, + ) + await self.manager.handle_result( + session, + {"type": "result", "text": "电力", "end_of_turn": True}, + ) + await self.manager.handle_result( + session, + {"type": "result", "is_listen": True}, + ) + states = [ + item for item in self.telemetry.messages + if item.get("type") == "model.state" + ] + self.assertEqual([item["end_of_turn"] for item in states], [False, False, True]) + self.assertEqual([item["decode_end"] for item in states], [True, True, False]) + + +if __name__ == "__main__": + unittest.main() diff --git a/integrations/minicpm_browser/README_zh.md b/integrations/minicpm_browser/README_zh.md new file mode 100644 index 0000000..82877a2 --- /dev/null +++ b/integrations/minicpm_browser/README_zh.md @@ -0,0 +1,20 @@ +# MiniCPM-o 浏览器 Phase A 集成资产 + +本目录保存已经验证过的浏览器薄适配层；Harness Core 位于仓库根目录的 +`extensions/assistive_harness/`。这些文件不是一个独立网页，也不会替代外部 +MiniCPM-o-Demo 后端。 + +接入已有 MiniCPM-o-Demo 时，将 `static/assistive_harness/` 整个复制到目标 +仓库的同名 `static/assistive_harness/`，并在目标版本的 +`static/omni/omni-app.js` 中接入以下边界： + +1. 导入并创建 `createAssistiveHarnessIntegration()`； +2. 把浏览器麦克风的 Web Audio source 镜像给 `attachAudioMirror()`； +3. 以不高于约 1 fps 把 JPEG 传给 `mirrorFrame()`； +4. 每次创建 Duplex Session 后调用 `bindSession(session)`； +5. 页面销毁时调用 `close()`。 + +不要直接用本目录覆盖不同版本的完整 `omni-app.js`。MiniCPM-o-Demo 的 +Duplex Session 私有接口可能随上游版本改变；浏览器 Hook 应在它自己的 +分支中审阅和回归。本 OpenGlass 分支中的 Phase B 设备链路无需这些网页 +文件即可运行。 diff --git a/integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.html b/integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.html new file mode 100644 index 0000000..452c74d --- /dev/null +++ b/integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.html @@ -0,0 +1,17 @@ + + + + + Assistive Harness Browser Acceptance Testbed + + + +

Assistive Harness Browser Acceptance Testbed

+

Browser adapter only; no model, microphone, camera, or private data.

+ 
RUNNING
+ + + diff --git a/integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.js b/integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.js new file mode 100644 index 0000000..2768401 --- /dev/null +++ b/integrations/minicpm_browser/static/assistive_harness/acceptance-testbed.js @@ -0,0 +1,115 @@ +import { BrowserSessionAdapter } from './browser-session-adapter.js'; + +class FakeSocket { + constructor(session) { this.readyState = 1; this.session = session; this.listeners = new Set(); } + addEventListener(type, listener) { if (type === 'message') this.listeners.add(listener); } + removeEventListener(type, listener) { if (type === 'message') this.listeners.delete(listener); } + send(value) { + if (JSON.parse(value).type !== 'stop') return; + queueMicrotask(() => { + this.session.cleanup(); + for (const listener of [...this.listeners]) { + listener({ data: JSON.stringify({ type: 'stopped' }) }); + } + }); + } +} + +function percentile(values, ratio) { + const sorted = [...values].sort((a, b) => a - b); + if (!sorted.length) return null; + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)]; +} + +async function run() { + let active = null; + const originalResults = []; + const telemetry = []; + const makeSession = () => { + const session = { + forceListenActive: false, + audioPlayer: { turnActive: true, stopAll() {}, endTurn() { this.turnActive = false; } }, + onForceListenChange() {}, + _handleResult(result) { originalResults.push(result); }, + cleanup() { this.ws.readyState = 3; if (active === this) active = null; }, + }; + session.ws = new FakeSocket(session); + return session; + }; + const adapter = new BrowserSessionAdapter({ + getSession: () => active, + startSession: async () => { active = makeSession(); return active; }, + setSystemPrompt() {}, + sendTelemetry: (event) => telemetry.push(event), + closeTimeoutMs: 50, + }); + active = makeSession(); + adapter.bindSession(active); + + let stopPassed = 0; + for (let index = 1; index <= 20; index += 1) { + const before = originalResults.length; + const ack = adapter.stopSpeech({ event_id: index }); + active._handleResult({ is_listen: false, text: 'old', audio_data: 'old' }); + const fenced = originalResults.length === before; + active._handleResult({ is_listen: true }); + const resume = adapter.resumeSpeech({ event_id: 1000 + index }); + if (ack.ok && resume.ok && fenced && originalResults.length === before + 1) stopPassed += 1; + } + + let resetPassed = 0; + let pollutedOldOutputs = 0; + for (let index = 1; index <= 20; index += 1) { + const old = active; + const before = originalResults.length; + const ack = await adapter.handleControl({ + accepted: true, intent: 'reset_session', event_id: 100 + index, + skill_id: 'idle_chat', slots: {}, system_prompt: `idle-${index}`, + }); + old._handleResult({ is_listen: false, text: 'late', audio_data: 'late' }); + if (originalResults.length !== before) pollutedOldOutputs += 1; + if (ack.ok && ack.generation === index) resetPassed += 1; + } + + let roundTripPassed = 0; + for (let index = 1; index <= 20; index += 1) { + const find = index % 2 === 1; + const skill = find ? 'find_object' : 'read_text'; + const slots = find ? { target: `物体${index}` } : {}; + const ack = await adapter.handleControl({ + accepted: true, intent: 'activate_skill', event_id: 200 + index, + skill_id: skill, slots, system_prompt: `${skill}-${index}`, + }); + const before = originalResults.length; + active._handleResult({ is_listen: false, text: `fresh-${index}` }); + if (ack.ok && originalResults.length === before + 1) roundTripPassed += 1; + } + + const latencies = adapter.snapshot().restart_latencies_ms; + return { + kind: 'BROWSER_ADAPTER_SIMULATION', + stop: { passed: stopPassed, total: 20 }, + reset: { passed: resetPassed, total: 20 }, + find_read_roundtrip: { passed: roundTripPassed, total: 20 }, + restart_latency_ms: { + p50: percentile(latencies, 0.50), + p90: percentile(latencies, 0.90), + samples: latencies.length, + }, + old_output_pollution_count: pollutedOldOutputs, + dropped_old_text: adapter.snapshot().dropped_old_text, + dropped_old_audio: adapter.snapshot().dropped_old_audio, + telemetry_events: telemetry.length, + note: 'Synthetic browser adapter test; not a microphone/model manual result.', + }; +} + +try { + const result = await run(); + document.getElementById('result').textContent = JSON.stringify(result, null, 2); + document.body.dataset.status = 'PASS'; + window.__acceptanceResult = result; +} catch (error) { + document.getElementById('result').textContent = String(error?.stack || error); + document.body.dataset.status = 'FAIL'; +} diff --git a/integrations/minicpm_browser/static/assistive_harness/audio-mirror-processor.js b/integrations/minicpm_browser/static/assistive_harness/audio-mirror-processor.js new file mode 100644 index 0000000..1b3017a --- /dev/null +++ b/integrations/minicpm_browser/static/assistive_harness/audio-mirror-processor.js @@ -0,0 +1,36 @@ +class AssistiveAudioMirrorProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + this.frameSize = options?.processorOptions?.frameSize || 1600; + this.buffer = new Float32Array(this.frameSize); + this.offset = 0; + this.active = true; + this.port.onmessage = (event) => { + if (event.data?.command === 'stop') this.active = false; + }; + } + + process(inputs, outputs) { + const output = outputs[0]?.[0]; + if (output) output.fill(0); + if (!this.active) return true; + const input = inputs[0]?.[0]; + if (!input) return true; + let sourceOffset = 0; + while (sourceOffset < input.length) { + const count = Math.min(input.length - sourceOffset, this.frameSize - this.offset); + this.buffer.set(input.subarray(sourceOffset, sourceOffset + count), this.offset); + sourceOffset += count; + this.offset += count; + if (this.offset === this.frameSize) { + const frame = this.buffer; + this.port.postMessage({ type: 'audio.frame', audio: frame }, [frame.buffer]); + this.buffer = new Float32Array(this.frameSize); + this.offset = 0; + } + } + return true; + } +} + +registerProcessor('assistive-audio-mirror', AssistiveAudioMirrorProcessor); diff --git a/integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.js b/integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.js new file mode 100644 index 0000000..387935b --- /dev/null +++ b/integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.js @@ -0,0 +1,307 @@ +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function sameSlots(left = {}, right = {}) { + const leftKeys = Object.keys(left).sort(); + const rightKeys = Object.keys(right).sort(); + return leftKeys.length === rightKeys.length + && leftKeys.every((key, index) => key === rightKeys[index] && String(left[key]) === String(right[key])); +} + +function sessionIdentity(session) { + const value = session?.sessionId || session?.recordingSessionId || ''; + return value ? String(value) : null; +} + +export class BrowserSessionAdapter { + constructor({ + getSession, + startSession, + setSystemPrompt, + addLog = () => {}, + sendTelemetry = () => {}, + onRestartingChange = () => {}, + closeTimeoutMs = 2000, + restartCleanupMode = 'light', + now = () => performance.now(), + }) { + this.getSession = getSession; + this.startSession = startSession; + this.setSystemPrompt = setSystemPrompt; + this.addLog = addLog; + this.sendTelemetry = sendTelemetry; + this.onRestartingChange = onRestartingChange; + this.closeTimeoutMs = closeTimeoutMs; + this.restartCleanupMode = restartCleanupMode === 'full' ? 'full' : 'light'; + this.now = now; + this.generation = 0; + this.currentSkill = 'idle_chat'; + this.currentSlots = {}; + this.dropOutputUntilListen = false; + this.speechHoldActive = false; + this.restartInProgress = false; + this.pendingRestart = null; + this.restartPromise = null; + this.droppedOldText = 0; + this.droppedOldAudio = 0; + this.stopCount = 0; + this.restartLatencies = []; + this._boundSessions = new WeakSet(); + } + + bindSession(session) { + if (!session || this._boundSessions.has(session)) return; + this._boundSessions.add(session); + const boundGeneration = this.generation; + const originalHandleResult = session._handleResult.bind(session); + session._handleResult = (result) => { + const stale = boundGeneration !== this.generation || session !== this.getSession(); + if (stale || (this.dropOutputUntilListen && !result.is_listen)) { + if (result.text) this.droppedOldText += 1; + if (result.audio_data) this.droppedOldAudio += 1; + return; + } + if (result.is_listen && this.dropOutputUntilListen && !this.speechHoldActive) { + this.dropOutputUntilListen = false; + session.forceListenActive = false; + session.audioPlayer?.setOutputBlocked?.(false); + session.onForceListenChange(false); + } + this.sendTelemetry({ + type: 'model.state', + state: result.is_listen ? 'listen' : 'speak', + text: result.text || '', + generation: boundGeneration, + }); + originalHandleResult(result); + }; + this.sendTelemetry({ + type: 'session.state', + phase: 'bound', + generation: boundGeneration, + skill_id: this.currentSkill, + slots: this.currentSlots, + }); + } + + stopSpeech(event = {}, { emitAck = true } = {}) { + const session = this.getSession(); + this.stopCount += 1; + this.speechHoldActive = true; + this.dropOutputUntilListen = true; + if (session) { + session.audioPlayer?.setOutputBlocked?.(true); + session.audioPlayer?.stopAll?.(); + if (session.audioPlayer?.turnActive) session.audioPlayer.endTurn(); + session.forceListenActive = true; + session.onForceListenChange?.(true); + } + this.addLog(`Harness STOP (event ${event.event_id ?? '?'})`); + const ack = { + type: 'control.ack', + event_id: event.event_id, + intent: 'stop_speech', + ok: true, + generation: this.generation, + dropped_old_text: this.droppedOldText, + dropped_old_audio: this.droppedOldAudio, + }; + if (emitAck) this.sendTelemetry(ack); + return ack; + } + + resumeSpeech(event = {}, { emitAck = true } = {}) { + const session = this.getSession(); + this.speechHoldActive = false; + this.dropOutputUntilListen = false; + if (session) { + session.forceListenActive = false; + session.audioPlayer?.setOutputBlocked?.(false); + session.onForceListenChange?.(false); + } + this.addLog(`Harness RESUME (event ${event.event_id ?? '?'})`); + const ack = { + type: 'control.ack', + event_id: event.event_id, + intent: 'resume_speech', + ok: true, + generation: this.generation, + dropped_old_text: this.droppedOldText, + dropped_old_audio: this.droppedOldAudio, + }; + if (emitAck) this.sendTelemetry(ack); + return ack; + } + + async handleControl(event) { + if (!event?.accepted) return { ok: false, ignored: true }; + if (event.intent === 'stop_speech') return this.stopSpeech(event); + if (event.intent === 'resume_speech') return this.resumeSpeech(event); + if (['reset_session', 'activate_skill', 'cancel_skill', 'return_to_chat'].includes(event.intent)) { + return this.requestRestart(event); + } + return { ok: false, ignored: true }; + } + + restartSession({ skillId = 'idle_chat', systemPrompt = '', reason = 'reset', eventId = 0 } = {}) { + return this.requestRestart({ + accepted: true, intent: 'reset_session', event_id: eventId, + skill_id: skillId, slots: {}, system_prompt: systemPrompt, reason, + }); + } + + activateSkill({ skillId, slots = {}, systemPrompt = '', reason = 'activate', eventId = 0 } = {}) { + return this.requestRestart({ + accepted: true, intent: 'activate_skill', event_id: eventId, + skill_id: skillId, slots, system_prompt: systemPrompt, reason, + }); + } + + returnToChat(reason = 'return_to_chat', eventId = 0, systemPrompt = '') { + return this.requestRestart({ + accepted: true, intent: 'return_to_chat', event_id: eventId, + skill_id: 'idle_chat', slots: {}, system_prompt: systemPrompt, reason, + }); + } + + getState() { return this.snapshot(); } + + requestRestart(event) { + const requestedSkill = event.skill_id || 'idle_chat'; + const requestedSlots = event.slots || {}; + if ( + event.intent !== 'reset_session' + && !this.restartInProgress + && requestedSkill === this.currentSkill + && sameSlots(requestedSlots, this.currentSlots) + ) { + const ack = { + type: 'control.ack', event_id: event.event_id, intent: event.intent, + ok: true, no_restart: true, generation: this.generation, + }; + this.sendTelemetry(ack); + return Promise.resolve(ack); + } + this.pendingRestart = event; + if (!this.restartPromise) { + this.restartPromise = this._restartLoop().finally(() => { + this.restartPromise = null; + }); + } + return this.restartPromise; + } + + async _restartLoop() { + let result = null; + while (this.pendingRestart) { + const event = this.pendingRestart; + this.pendingRestart = null; + result = await this._restartOnce(event); + } + return result; + } + + async _restartOnce(event) { + const started = this.now(); + this.restartInProgress = true; + this.onRestartingChange(true); + this.stopSpeech({ event_id: event.event_id }, { emitAck: false }); + const oldSession = this.getSession(); + const oldSessionId = sessionIdentity(oldSession); + this.generation += 1; + this.sendTelemetry({ + type: 'session.state', phase: 'restart_started', generation: this.generation, + event_id: event.event_id, old_session_id: oldSessionId, + }); + try { + await this._closeOldSession(oldSession, this.restartCleanupMode); + if (event.system_prompt) this.setSystemPrompt(event.system_prompt); + this.currentSkill = event.skill_id || 'idle_chat'; + this.currentSlots = event.slots || {}; + await this.startSession(); + const replacement = this.getSession(); + if (!replacement) throw new Error('replacement session was not created'); + this.bindSession(replacement); + const newSessionId = sessionIdentity(replacement); + this.speechHoldActive = false; + this.dropOutputUntilListen = false; + replacement.forceListenActive = false; + replacement.audioPlayer?.setOutputBlocked?.(false); + replacement.onForceListenChange?.(false); + const latency = this.now() - started; + this.restartLatencies.push(latency); + const ack = { + type: 'control.ack', event_id: event.event_id, intent: event.intent, + ok: true, generation: this.generation, restart_latency_ms: latency, + skill_id: this.currentSkill, slots: this.currentSlots, + old_session_id: oldSessionId, new_session_id: newSessionId, + cleanup_mode: this.restartCleanupMode, + dropped_old_text: this.droppedOldText, dropped_old_audio: this.droppedOldAudio, + }; + this.sendTelemetry({ + type: 'session.state', phase: 'restart_complete', generation: this.generation, + skill_id: this.currentSkill, slots: this.currentSlots, + old_session_id: oldSessionId, new_session_id: newSessionId, + cleanup_mode: this.restartCleanupMode, + }); + this.sendTelemetry(ack); + this.addLog(`Harness ${this.currentSkill} ready (${Math.round(latency)} ms)`); + return ack; + } catch (error) { + const ack = { + type: 'control.ack', event_id: event.event_id, intent: event.intent, + ok: false, generation: this.generation, error: String(error?.message || error), + }; + this.sendTelemetry(ack); + this.addLog(`Harness restart failed: ${ack.error}`); + return ack; + } finally { + this.restartInProgress = false; + this.onRestartingChange(false); + } + } + + async _closeOldSession(session, cleanupMode = 'light') { + if (!session) return; + const ws = session.ws; + if (!ws || ws.readyState !== 1) { + session.cleanup?.(); + return; + } + let settled = false; + await Promise.race([ + new Promise((resolve) => { + const listener = (message) => { + try { + if (JSON.parse(message.data)?.type === 'stopped') { + settled = true; + ws.removeEventListener('message', listener); + resolve(); + } + } catch (_) {} + }; + ws.addEventListener('message', listener); + ws.send(JSON.stringify({ type: 'stop', cleanup_mode: cleanupMode })); + }), + sleep(this.closeTimeoutMs), + ]); + if (!settled) session.cleanup?.(); + } + + snapshot() { + return { + generation: this.generation, + current_skill: this.currentSkill, + current_slots: { ...this.currentSlots }, + drop_output_until_listen: this.dropOutputUntilListen, + speech_hold_active: this.speechHoldActive, + restart_in_progress: this.restartInProgress, + pending_restart: Boolean(this.pendingRestart), + active_session_id: sessionIdentity(this.getSession()), + restart_cleanup_mode: this.restartCleanupMode, + dropped_old_text: this.droppedOldText, + dropped_old_audio: this.droppedOldAudio, + stop_count: this.stopCount, + restart_latencies_ms: [...this.restartLatencies], + }; + } +} diff --git a/integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.test.mjs b/integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.test.mjs new file mode 100644 index 0000000..d147004 --- /dev/null +++ b/integrations/minicpm_browser/static/assistive_harness/browser-session-adapter.test.mjs @@ -0,0 +1,202 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { BrowserSessionAdapter } from './browser-session-adapter.js'; + +let fakeSessionSequence = 0; + +class FakeSocket { + constructor(session) { + this.readyState = 1; + this.session = session; + this.listeners = new Set(); + } + addEventListener(type, listener) { if (type === 'message') this.listeners.add(listener); } + removeEventListener(type, listener) { if (type === 'message') this.listeners.delete(listener); } + send(value) { + const message = JSON.parse(value); + if (message.type !== 'stop') return; + this.lastStopMessage = message; + queueMicrotask(() => { + this.session.cleanup(); + for (const listener of [...this.listeners]) listener({ data: JSON.stringify({ type: 'stopped' }) }); + }); + } +} + +function fakeSession(onCleanup = () => {}) { + const counters = { stopAll: 0, endTurn: 0, originalResults: 0, forceChanges: [], outputBlocks: [] }; + const session = { + counters, + sessionId: `fake_${++fakeSessionSequence}`, + forceListenActive: false, + audioPlayer: { + turnActive: true, + stopAll() { counters.stopAll += 1; }, + setOutputBlocked(blocked) { counters.outputBlocks.push(Boolean(blocked)); }, + endTurn() { counters.endTurn += 1; this.turnActive = false; }, + }, + onForceListenChange(active) { counters.forceChanges.push(active); }, + _handleResult() { counters.originalResults += 1; }, + cleanup() { this.ws.readyState = 3; onCleanup(this); }, + }; + session.ws = new FakeSocket(session); + return session; +} + +function harnessFixture() { + let active = null; + let prompt = ''; + const telemetry = []; + const adapter = new BrowserSessionAdapter({ + getSession: () => active, + startSession: async () => { + active = fakeSession((session) => { if (active === session) active = null; }); + return active; + }, + setSystemPrompt: (value) => { prompt = value; }, + sendTelemetry: (value) => telemetry.push(value), + closeTimeoutMs: 20, + }); + const createInitial = () => { + active = fakeSession((session) => { if (active === session) active = null; }); + adapter.bindSession(active); + return active; + }; + return { adapter, createInitial, get active() { return active; }, get prompt() { return prompt; }, telemetry }; +} + +test('STOP holds output and RESUME releases it 20/20', () => { + const fixture = harnessFixture(); + const session = fixture.createInitial(); + for (let index = 1; index <= 20; index += 1) { + const ack = fixture.adapter.stopSpeech({ event_id: index }); + assert.equal(ack.ok, true); + session._handleResult({ is_listen: false, text: `stale-${index}`, audio_data: 'old' }); + assert.equal(session.counters.originalResults, index - 1); + session._handleResult({ is_listen: true }); + assert.equal(session.counters.originalResults, index); + assert.equal(fixture.adapter.snapshot().speech_hold_active, true); + assert.equal(fixture.adapter.snapshot().drop_output_until_listen, true); + const resumed = fixture.adapter.resumeSpeech({ event_id: 100 + index }); + assert.equal(resumed.ok, true); + assert.equal(fixture.adapter.snapshot().speech_hold_active, false); + assert.equal(fixture.adapter.snapshot().drop_output_until_listen, false); + } + assert.equal(session.counters.stopAll, 20); + assert.equal(session.counters.outputBlocks.filter(Boolean).length, 20); + assert.equal(session.counters.outputBlocks.filter((value) => !value).length, 20); + assert.equal(fixture.adapter.snapshot().stop_count, 20); + assert.equal(fixture.adapter.snapshot().dropped_old_text, 20); + assert.equal(fixture.adapter.snapshot().dropped_old_audio, 20); +}); + +test('handleControl routes the resume_speech intent', async () => { + const fixture = harnessFixture(); + const session = fixture.createInitial(); + fixture.adapter.stopSpeech({ event_id: 1 }); + const ack = await fixture.adapter.handleControl({ + accepted: true, intent: 'resume_speech', event_id: 2, + }); + assert.equal(ack.ok, true); + assert.equal(ack.intent, 'resume_speech'); + assert.equal(session.forceListenActive, false); + assert.equal(fixture.adapter.snapshot().speech_hold_active, false); +}); + +test('RESET completes 20/20 with new generations and drops old output', async () => { + const fixture = harnessFixture(); + fixture.createInitial(); + for (let index = 1; index <= 20; index += 1) { + const old = fixture.active; + const ack = await fixture.adapter.handleControl({ + accepted: true, intent: 'reset_session', event_id: index, + skill_id: 'idle_chat', slots: {}, system_prompt: `idle-${index}`, + }); + assert.equal(ack.ok, true); + assert.equal(ack.generation, index); + assert.equal(ack.old_session_id, old.sessionId); + assert.equal(ack.new_session_id, fixture.active.sessionId); + assert.notEqual(ack.old_session_id, ack.new_session_id); + assert.equal(ack.cleanup_mode, 'light'); + assert.equal(old.ws.lastStopMessage.cleanup_mode, 'light'); + assert.equal(fixture.adapter.snapshot().active_session_id, ack.new_session_id); + old._handleResult({ is_listen: false, text: 'late', audio_data: 'late-audio' }); + assert.equal(old.counters.originalResults, 0); + } + assert.equal(fixture.adapter.snapshot().generation, 20); + assert.equal(fixture.prompt, 'idle-20'); + assert.equal(fixture.adapter.snapshot().dropped_old_text, 20); + assert.equal(fixture.adapter.snapshot().dropped_old_audio, 20); +}); + +test('find/read skill round trips complete 20/20 without duplicate native results', async () => { + const fixture = harnessFixture(); + fixture.createInitial(); + for (let index = 1; index <= 20; index += 1) { + const find = index % 2 === 1; + const skill = find ? 'find_object' : 'read_text'; + const slots = find ? { target: `物体${index}` } : {}; + const ack = await fixture.adapter.handleControl({ + accepted: true, intent: 'activate_skill', event_id: 100 + index, + skill_id: skill, slots, system_prompt: `${skill}-${index}`, + }); + assert.equal(ack.ok, true); + assert.equal(ack.cleanup_mode, 'light'); + assert.equal(fixture.prompt, `${skill}-${index}`); + fixture.active._handleResult({ is_listen: false, text: `native-${index}` }); + assert.equal(fixture.active.counters.originalResults, 1); + } + assert.equal(fixture.adapter.snapshot().generation, 20); + assert.equal(fixture.adapter.snapshot().current_skill, 'read_text'); +}); + +test('same skill and same slots do not restart', async () => { + const fixture = harnessFixture(); + fixture.createInitial(); + await fixture.adapter.handleControl({ + accepted: true, intent: 'activate_skill', event_id: 1, + skill_id: 'find_object', slots: { target: '手机' }, system_prompt: 'phone', + }); + const generation = fixture.adapter.snapshot().generation; + const ack = await fixture.adapter.handleControl({ + accepted: true, intent: 'activate_skill', event_id: 2, + skill_id: 'find_object', slots: { target: '手机' }, system_prompt: 'phone', + }); + assert.equal(ack.no_restart, true); + assert.equal(fixture.adapter.snapshot().generation, generation); +}); + +test('RESET timeout force-cleans the old websocket and still recovers', async () => { + let active = fakeSession((session) => { if (active === session) active = null; }); + active.ws.send = () => {}; + const adapter = new BrowserSessionAdapter({ + getSession: () => active, + startSession: async () => { + active = fakeSession((session) => { if (active === session) active = null; }); + return active; + }, + setSystemPrompt() {}, closeTimeoutMs: 5, + }); + adapter.bindSession(active); + const ack = await adapter.restartSession({systemPrompt: 'idle', eventId: 9}); + assert.equal(ack.ok, true); + assert.equal(ack.generation, 1); +}); + +test('STOP remains immediate while a restart is waiting for close', async () => { + const fixture = harnessFixture(); + fixture.createInitial(); + fixture.active.ws.send = () => {}; + fixture.adapter.closeTimeoutMs = 25; + const restart = fixture.adapter.activateSkill({ + skillId: 'read_text', systemPrompt: 'read', eventId: 1, + }); + const stop = fixture.adapter.stopSpeech({event_id: 2}); + assert.equal(stop.ok, true); + assert.equal(fixture.adapter.snapshot().drop_output_until_listen, true); + const recovered = await restart; + assert.equal(recovered.ok, true); + assert.equal(fixture.adapter.snapshot().speech_hold_active, false); + assert.equal(fixture.adapter.snapshot().drop_output_until_listen, false); +}); diff --git a/integrations/minicpm_browser/static/assistive_harness/harness-client.js b/integrations/minicpm_browser/static/assistive_harness/harness-client.js new file mode 100644 index 0000000..f8bb396 --- /dev/null +++ b/integrations/minicpm_browser/static/assistive_harness/harness-client.js @@ -0,0 +1,212 @@ +import { BrowserSessionAdapter } from './browser-session-adapter.js?v=phase-a-voice-reset-hot-v5'; + +function enabledByQuery() { + return new URLSearchParams(window.location.search).get('assistive_harness') === '1'; +} + +function bytesToBase64(bytes) { + let output = ''; + const chunkSize = 0x8000; + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + output += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize)); + } + return btoa(output); +} + +function defaultControlUrl() { + const params = new URLSearchParams(window.location.search); + const override = params.get('assistive_harness_ws'); + if (override) return override; + const scheme = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + return `${scheme}//${window.location.hostname}:8021/ws/control`; +} + +function makeStatusPanel() { + const root = document.createElement('div'); + root.id = 'assistiveHarnessStatus'; + root.style.cssText = [ + 'position:fixed', 'right:12px', 'bottom:12px', 'z-index:9999', + 'padding:6px 9px', 'border-radius:6px', 'font:12px sans-serif', + 'color:#fff', 'background:#7a5b00', 'opacity:.9', 'max-width:340px', + ].join(';'); + const label = document.createElement('button'); + label.type = 'button'; + label.textContent = 'Harness: connecting ▸'; + label.style.cssText = 'border:0;background:transparent;color:inherit;font:inherit;cursor:pointer;padding:0'; + const details = document.createElement('pre'); + details.style.cssText = 'display:none;margin:6px 0 0;white-space:pre-wrap;font:11px/1.4 ui-monospace,monospace'; + details.textContent = 'Waiting for state…'; + label.addEventListener('click', () => { + const open = details.style.display !== 'none'; + details.style.display = open ? 'none' : 'block'; + label.textContent = label.textContent.replace(open ? '▾' : '▸', open ? '▸' : '▾'); + }); + root.append(label, details); + document.body.appendChild(root); + return { + root, label, details, + setConnection(text, color) { + const marker = details.style.display === 'none' ? '▸' : '▾'; + label.textContent = `${text} ${marker}`; + root.style.background = color; + }, + }; +} + +export function createAssistiveHarnessIntegration(options) { + if (!enabledByQuery()) { + return { + enabled: false, + bindSession() {}, + async attachAudioMirror() {}, + detachAudioMirror() {}, + mirrorFrame() {}, + close() {}, + }; + } + + const status = makeStatusPanel(); + const debugState = { + connected: false, latest_transcript: '', intent: '', current_skill: 'idle_chat', + target: '', generation: 0, restart_latency_ms: null, output_gate: false, cv_mode: 'shadow', + old_session_id: '', new_session_id: '', cleanup_mode: '', + }; + let socket = null; + let reconnectTimer = null; + let audioNode = null; + let zeroGain = null; + let lastFrameAt = 0; + let closed = false; + + const renderDebugState = () => { + status.details.textContent = [ + `connected: ${debugState.connected}`, + `ASR: ${debugState.latest_transcript || '—'}`, + `intent: ${debugState.intent || '—'}`, + `skill: ${debugState.current_skill}`, + `target: ${debugState.target || '—'}`, + `generation: ${debugState.generation}`, + `session: ${debugState.old_session_id || '—'} -> ${debugState.new_session_id || '—'}`, + `reset_mode: ${debugState.cleanup_mode === 'light' ? 'hot/light' : (debugState.cleanup_mode || '—')}`, + `restart_ms: ${debugState.restart_latency_ms ?? '—'}`, + `output_gate: ${debugState.output_gate}`, + `CV: ${debugState.cv_mode}`, + ].join('\n'); + }; + const rawSend = (payload) => { + if (socket?.readyState === WebSocket.OPEN) socket.send(JSON.stringify(payload)); + }; + const adapter = new BrowserSessionAdapter({ + ...options, + sendTelemetry: (payload) => { + if (payload.type === 'control.ack') { + debugState.generation = payload.generation ?? debugState.generation; + debugState.restart_latency_ms = payload.restart_latency_ms ?? debugState.restart_latency_ms; + debugState.old_session_id = payload.old_session_id ?? debugState.old_session_id; + debugState.new_session_id = payload.new_session_id ?? debugState.new_session_id; + debugState.cleanup_mode = payload.cleanup_mode ?? debugState.cleanup_mode; + const snapshot = adapter.snapshot(); + debugState.current_skill = snapshot.current_skill; + debugState.target = snapshot.current_slots?.target || ''; + debugState.output_gate = snapshot.drop_output_until_listen; + renderDebugState(); + } + rawSend(payload); + }, + }); + + const connect = () => { + if (closed) return; + status.setConnection('Harness: connecting', '#7a5b00'); + socket = new WebSocket(defaultControlUrl()); + socket.onopen = () => { + debugState.connected = true; + status.setConnection('Harness: ready', '#176b36'); + renderDebugState(); + }; + socket.onmessage = (message) => { + let payload; + try { payload = JSON.parse(message.data); } catch (_) { return; } + if (payload.type === 'asr.transcript') { + debugState.latest_transcript = payload.utterance || ''; + renderDebugState(); + } + if (payload.type === 'control.intent') { + debugState.latest_transcript = payload.utterance || debugState.latest_transcript; + debugState.intent = payload.intent || ''; + renderDebugState(); + void adapter.handleControl(payload); + } + }; + socket.onerror = () => { + status.setConnection('Harness: unavailable (native unaffected)', '#8b1e1e'); + }; + socket.onclose = () => { + socket = null; + debugState.connected = false; + if (!closed) { + status.setConnection('Harness: reconnecting', '#7a5b00'); + renderDebugState(); + reconnectTimer = setTimeout(connect, 1500); + } + }; + }; + connect(); + + const integration = { + enabled: true, + adapter, + bindSession(session) { adapter.bindSession(session); }, + async attachAudioMirror(context, source) { + if (audioNode) return; + await context.audioWorklet.addModule('/static/assistive_harness/audio-mirror-processor.js'); + audioNode = new AudioWorkletNode(context, 'assistive-audio-mirror', { + processorOptions: { frameSize: 1600 }, + }); + zeroGain = context.createGain(); + zeroGain.gain.value = 0; + source.connect(audioNode); + audioNode.connect(zeroGain); + zeroGain.connect(context.destination); + audioNode.port.onmessage = (event) => { + if (event.data?.type !== 'audio.frame') return; + const frame = event.data.audio; + rawSend({ + type: 'audio.mirror', + started_at_ms: Date.now() - (frame.length * 1000 / context.sampleRate), + sample_rate: context.sampleRate, + audio_b64: bytesToBase64(new Uint8Array(frame.buffer)), + }); + }; + }, + detachAudioMirror() { + try { audioNode?.port.postMessage({ command: 'stop' }); } catch (_) {} + try { audioNode?.disconnect(); } catch (_) {} + try { zeroGain?.disconnect(); } catch (_) {} + audioNode = null; + zeroGain = null; + }, + mirrorFrame(jpegBase64) { + const timestamp = Date.now(); + if (!jpegBase64 || timestamp - lastFrameAt < 1000) return; + lastFrameAt = timestamp; + rawSend({ + type: 'frame.shadow', frame_id: `browser_${timestamp}`, + timestamp_ms: timestamp, jpeg_b64: jpegBase64, + }); + }, + injectTranscript(text) { + rawSend({ type: 'asr.inject', text }); + }, + snapshot() { return adapter.snapshot(); }, + close() { + closed = true; + if (reconnectTimer) clearTimeout(reconnectTimer); + integration.detachAudioMirror(); + try { socket?.close(); } catch (_) {} + status.root.remove(); + }, + }; + window.__assistiveHarness = integration; + return integration; +} diff --git a/integrations/minicpm_browser/static/assistive_harness/package.json b/integrations/minicpm_browser/static/assistive_harness/package.json new file mode 100644 index 0000000..e986b24 --- /dev/null +++ b/integrations/minicpm_browser/static/assistive_harness/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..a1c0c17 --- /dev/null +++ b/models/README.md @@ -0,0 +1,6 @@ +# Local model directory + +Put optional local inference weights here, for example `yolo26n.onnx`. +Weights and accelerator artifacts are ignored by Git; this directory contains +only documentation. ModelScope ASR models normally live in the user's external +cache (`%USERPROFILE%\.cache\modelscope\hub`) and do not need to be copied here. diff --git a/runtime/openglass_omni/README.md b/runtime/openglass_omni/README.md index 286008b..30438e3 100644 --- a/runtime/openglass_omni/README.md +++ b/runtime/openglass_omni/README.md @@ -1,6 +1,10 @@ # OpenGlass Omni Runtime -This directory holds OpenGlass's own control panel, the ESP32 audio/video bridge, the Rokid link, and the local session recording / replay code. The MiniCPM-o-Demo and llama.cpp-omni projects stay **external** — nothing here is copied into an upstream directory, and this panel never downloads, builds, or rewrites upstream config. +This directory holds OpenGlass's control panel, ESP32 audio/video bridge, Rokid +link, and local recording/replay code. The shared voice-command Core is included +in this repository at `extensions/assistive_harness/`. MiniCPM-o-Demo, +llama.cpp-omni, their workers and large-model weights remain **external**; this +panel never downloads, builds, or rewrites their upstream configuration. This is an experimental research integration. It is not production-ready, not a certified navigation aid, and not validated for unbounded-length sessions. @@ -121,7 +125,9 @@ Watch the rerun via the bridge's own live view at `http://localhost:/`. ## Current boundaries - The panel starts and supervises processes and shows the first-person view; it does not own model weights, backend paths, or upstream configuration. -- `worker.py` / `gateway.py` and the model weights come from external upstream projects and are not vendored here. +- `worker.py` / `gateway.py` and large-model weights come from external upstream + projects and are not vendored here. The small Harness Core itself is vendored + under `extensions/assistive_harness/` and can be installed from this clone. - The Rokid link is included, but its gateway protocol may differ from the ESP32 link depending on your build; treat the ESP32 link as the primary supported path. - One-click rerun from within the panel is not implemented; rerun is the command-line workflow above. - Session output under `sessions/` may contain faces, surroundings, voices, and device addresses. Review it before sharing or publishing.