diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..cde9a881 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - voicelife-hil + - sparkbot + - pcb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 898ad437..8dfce642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,8 @@ jobs: --journey im-gateway-strong-reminder \ --profile host \ --artifact-dir artifacts/im-gateway-e2e \ - --timeout 60 + --timeout 60 \ + --retries 0 - name: 运行 Host 快速恢复 E2E run: | E2E_RECOVERY_SUITE=quick python3 scripts/run_e2e.py \ @@ -101,13 +102,36 @@ jobs: --artifact-dir artifacts/im-gateway-e2e \ --timeout 120 \ --retries 0 - - name: 上传 Host E2E 脱敏证据 + - name: 验证故意失败 evidence + if: always() + shell: bash + run: | + set +e + VOICELIFE_E2E_CONTRACT_FAILURE=1 python3 scripts/run_e2e.py \ + --layer host \ + --journey lifecycle-example \ + --profile host \ + --artifact-dir artifacts/im-gateway-e2e/contract-failure \ + --timeout 60 \ + --retries 0 + failure_status=$? + set -e + test "$failure_status" -eq 20 + - name: 校验 Host E2E 脱敏 evidence + id: validate-host-evidence if: always() + run: python3 scripts/check_e2e_artifacts.py artifacts/im-gateway-e2e + - name: 写入 Host E2E job summary + if: always() + run: python3 scripts/render_e2e_summary.py artifacts/im-gateway-e2e >> "$GITHUB_STEP_SUMMARY" + - name: 上传 Host E2E 脱敏证据 + if: always() && steps.validate-host-evidence.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: im-gateway-host-e2e-evidence path: artifacts/im-gateway-e2e if-no-files-found: ignore + retention-days: 14 coverage: name: 覆盖率（Codecov） diff --git a/.github/workflows/hil-nightly.yml b/.github/workflows/hil-nightly.yml new file mode 100644 index 00000000..df1240f0 --- /dev/null +++ b/.github/workflows/hil-nightly.yml @@ -0,0 +1,154 @@ +name: HIL Manual + +on: + workflow_dispatch: + inputs: + profile: + description: Device profile to run (the default runs both profiles) + required: false + type: choice + default: all + options: + - all + - sparkbot + - pcb + journey: + description: HIL journey + required: false + type: choice + default: im-pairing + options: + - im-pairing + - voice + tts_provider: + description: Host TTS fixture provider used by the voice journey + required: false + type: choice + default: dashscope + options: + - dashscope + - aliyun-nls + device: + description: Optional descriptor name; requires a single profile selection + required: false + type: string + default: "" + +permissions: + contents: read + +concurrency: + group: hil-nightly-${{ github.ref }}-${{ inputs.profile || 'all' }} + cancel-in-progress: false + +jobs: + hil: + name: HIL / ${{ matrix.profile }} + runs-on: [self-hosted, voicelife-hil, "${{ matrix.profile }}"] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + profile: ${{ fromJSON(inputs.profile == 'sparkbot' && '["sparkbot"]' || inputs.profile == 'pcb' && '["pcb"]' || '["sparkbot", "pcb"]') }} + env: + HIL_DEVICE_ROOT: ${{ vars.VOICELIFE_HIL_DEVICE_ROOT || '/opt/voicelife/hil/devices' }} + HIL_LEASE_ROOT: ${{ vars.VOICELIFE_HIL_LEASE_ROOT || '/opt/voicelife/hil/leases' }} + VOICELIFE_HIL_SERVER: ${{ secrets.VOICELIFE_HIL_SERVER }} + VOICELIFE_HIL_SERVER_DIR: ${{ secrets.VOICELIFE_HIL_SERVER_DIR }} + VOICELIFE_HIL_GATEWAY_ORIGIN: ${{ secrets.VOICELIFE_HIL_GATEWAY_ORIGIN }} + VOICELIFE_HIL_USER_ID: ${{ secrets.VOICELIFE_HIL_USER_ID }} + DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} + ALIYUN_NLS_APPKEY: ${{ secrets.ALIYUN_NLS_APPKEY }} + ALIYUN_NLS_TOKEN: ${{ secrets.ALIYUN_NLS_TOKEN }} + ALIYUN_NLS_URL: ${{ vars.ALIYUN_NLS_URL }} + HIL_DEVICE_NAME: ${{ inputs.device || '' }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: 检查受控 Runner 配置 + run: | + set -euo pipefail + device_name="$HIL_DEVICE_NAME" + if [[ -n "$device_name" ]]; then + [[ "${{ inputs.profile || 'all' }}" != "all" ]] + [[ "$device_name" =~ ^[a-z0-9][a-z0-9_-]{0,63}$ ]] + descriptor="$HIL_DEVICE_ROOT/$device_name.json" + else + descriptor="$HIL_DEVICE_ROOT/${{ matrix.profile }}.json" + fi + test -r "$descriptor" + test -n "${VOICELIFE_HIL_SERVER:-}" + test -n "${VOICELIFE_HIL_SERVER_DIR:-}" + test -n "${VOICELIFE_HIL_GATEWAY_ORIGIN:-}" + test -n "${VOICELIFE_HIL_USER_ID:-}" + python3 -c 'import serial, esptool' >/dev/null + if [[ "${{ inputs.journey || 'im-pairing' }}" == "voice" ]]; then + if [[ "${{ inputs.tts_provider || 'dashscope' }}" == "dashscope" ]]; then + test -n "${DASHSCOPE_API_KEY:-}" + python3 -c 'import dashscope' >/dev/null + else + test -n "${ALIYUN_NLS_APPKEY:-}" + test -n "${ALIYUN_NLS_TOKEN:-}" + python3 -c 'import nls' >/dev/null + fi + command -v ffmpeg >/dev/null + fi + - name: 运行 HIL journey + id: journey + run: | + set +e + mkdir -p "artifacts/hil-${{ matrix.profile }}" + device_name="$HIL_DEVICE_NAME" + if [[ -n "$device_name" ]]; then + device_path="$HIL_DEVICE_ROOT/$device_name.json" + else + device_path="$HIL_DEVICE_ROOT/${{ matrix.profile }}.json" + fi + run_args=( + python3 scripts/run_e2e.py \ + --layer hil \ + --journey "${{ inputs.journey || 'im-pairing' }}" \ + --profile "${{ matrix.profile }}" \ + --artifact-dir "artifacts/hil-${{ matrix.profile }}" \ + --timeout 900 \ + --retries 0 \ + --device "$device_path" \ + --lease-dir "$HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" + ) + if [[ "${{ inputs.journey || 'im-pairing' }}" == "voice" ]]; then + run_args+=( + --input-tts "${{ inputs.tts_provider || 'dashscope' }}" + --tts-model "${VOICELIFE_TTS_MODEL:-qwen-audio-3.0-tts-flash}" + --voice "${VOICELIFE_TTS_VOICE:-longanlingxi}" + --text '你好牛牛，请介绍一下你自己。' + --text '把刚才的回答再简短一点。' + --text '请用一句话总结我们刚才的对话。' + --expect-terminal + ) + fi + "${run_args[@]}" + run_status=$? + exit "$run_status" + - name: 校验 HIL 脱敏 evidence + id: validate-hil-evidence + if: always() + run: python3 scripts/check_e2e_artifacts.py "artifacts/hil-${{ matrix.profile }}" + - name: 写入 HIL job summary + if: always() + run: python3 scripts/render_e2e_summary.py "artifacts/hil-${{ matrix.profile }}" >> "$GITHUB_STEP_SUMMARY" + - name: 上传 HIL 脱敏 evidence + if: always() && steps.validate-hil-evidence.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: hil-${{ matrix.profile }}-evidence + path: artifacts/hil-${{ matrix.profile }} + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/im-recovery-nightly.yml b/.github/workflows/im-recovery-nightly.yml index b71e2bcb..edfa2ce4 100644 --- a/.github/workflows/im-recovery-nightly.yml +++ b/.github/workflows/im-recovery-nightly.yml @@ -59,10 +59,18 @@ jobs: --artifact-dir artifacts/im-gateway-recovery \ --timeout 300 \ --retries 0 - - name: 上传恢复矩阵脱敏证据 + - name: 校验恢复矩阵脱敏 evidence + id: validate-recovery-evidence + if: always() + run: python3 scripts/check_e2e_artifacts.py artifacts/im-gateway-recovery + - name: 写入恢复矩阵 job summary if: always() + run: python3 scripts/render_e2e_summary.py artifacts/im-gateway-recovery >> "$GITHUB_STEP_SUMMARY" + - name: 上传恢复矩阵脱敏证据 + if: always() && steps.validate-recovery-evidence.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: im-gateway-recovery-nightly-evidence path: artifacts/im-gateway-recovery if-no-files-found: ignore + retention-days: 14 diff --git a/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port.cc b/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port.cc index 27259732..7f9276b7 100644 --- a/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port.cc +++ b/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port.cc @@ -70,6 +70,8 @@ Status Esp32s3PcmAudioPorts::Impl::InputPort::StartCapture(voice::VoiceMode mode Status Esp32s3PcmAudioPorts::Impl::InputPort::StopCapture() { return owner_.StopCapture(); } +Status Esp32s3PcmAudioPorts::Impl::InputPort::DiscardPendingInput() { return owner_.DiscardPendingInput(); } + void Esp32s3PcmAudioPorts::Impl::InputPort::Close() { (void)owner_.CloseInput(); } Status Esp32s3PcmAudioPorts::Impl::OutputPort::Open(const voice::AudioFormat& format) { @@ -537,6 +539,19 @@ Status Esp32s3PcmAudioPorts::Impl::StopCapture() { #endif } +Status Esp32s3PcmAudioPorts::Impl::DiscardPendingInput() { +#ifndef ESP_PLATFORM + return detail::Unavailable("ESP32-S3 PCM Audio Port 只能在 ESP-IDF 目标运行"); +#else + std::lock_guard lock(mutex_); + const std::size_t queued = input_queue_.size(); + input_queue_.clear(); + if (assembler_) assembler_->Reset(); + ESP_LOGI(detail::kAudioRuntimeTag, "INPUT_BOUNDARY_RESET queued=%u", static_cast(queued)); + return Status::Ok(); +#endif +} + Status Esp32s3PcmAudioPorts::Impl::CloseInput() { const Status stop_status = StopCapture(); #ifdef ESP_PLATFORM diff --git a/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port_internal.h b/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port_internal.h index ef0542a7..39e10fbd 100644 --- a/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port_internal.h +++ b/components/voicelife_audio_esp/src/esp32s3_pcm_audio_port_internal.h @@ -60,6 +60,7 @@ class Esp32s3PcmAudioPorts::Impl final { Status Open(const voice::AudioFormat& format) override; Status StartCapture(voice::VoiceMode mode) override; Status StopCapture() override; + Status DiscardPendingInput() override; void Close() override; private: @@ -107,6 +108,7 @@ class Esp32s3PcmAudioPorts::Impl final { Status OpenOutput(const voice::AudioFormat& format); Status StartCapture(voice::VoiceMode mode); Status StopCapture(); + Status DiscardPendingInput(); Status CloseInput(); Status PushOutput(voice::AudioFrame frame); Status FlushOutput(); diff --git a/components/voicelife_linx/CMakeLists.txt b/components/voicelife_linx/CMakeLists.txt index 198b6213..958709b6 100644 --- a/components/voicelife_linx/CMakeLists.txt +++ b/components/voicelife_linx/CMakeLists.txt @@ -7,4 +7,5 @@ idf_component_register( "../../third_party/cjson/cJSON.c" INCLUDE_DIRS "include" "../../third_party/cjson" REQUIRES voicelife_contracts voicelife_voice + PRIV_REQUIRES pthread ) diff --git a/components/voicelife_linx/include/voicelife/linx/linx_speech_provider.h b/components/voicelife_linx/include/voicelife/linx/linx_speech_provider.h index 8a66d1ae..75445c59 100644 --- a/components/voicelife_linx/include/voicelife/linx/linx_speech_provider.h +++ b/components/voicelife_linx/include/voicelife/linx/linx_speech_provider.h @@ -96,7 +96,6 @@ class LinxSpeechProviderAdapter final : public voice::SpeechProviderAdapter { [[nodiscard]] voice::VoiceSessionConfig ActiveSessionConfig() const; Status Send(Result encoded); void Emit(voice::VoiceEvent event); - LinxTransportPort& transport_; LinxProtocolCodecPort& codec_; LinxConnectionConfig connection_; diff --git a/components/voicelife_linx/include/voicelife/linx/linx_types.h b/components/voicelife_linx/include/voicelife/linx/linx_types.h index 91a7a047..c69e51f3 100644 --- a/components/voicelife_linx/include/voicelife/linx/linx_types.h +++ b/components/voicelife_linx/include/voicelife/linx/linx_types.h @@ -13,10 +13,10 @@ namespace voicelife::linx { /** 保存 Linx WebSocket 连接所需的非敏感配置引用。 */ struct LinxConnectionConfig { - // Linx uses this value to choose its downstream send strategy. Keep it in - // the same latency budget as the board playback queue rather than deriving - // an unbounded duration from the negotiated packet size. - static constexpr uint32_t kDefaultPlaybackBufferDurationMs = 200; + // Linx uses this value to choose its downstream send strategy. The + // WebSocket contract documents 1000 ms as the PCM default; keeping the + // negotiated value aligned avoids a server-side strategy mismatch. + static constexpr uint32_t kDefaultPlaybackBufferDurationMs = 1000; std::string websocket_url; // A reference such as secret://linx/device-token. The resolved token is @@ -26,6 +26,11 @@ struct LinxConnectionConfig { std::string client_id; std::optional agent_id; uint32_t playback_buffer_duration_ms = kDefaultPlaybackBufferDurationMs; + // Physical audio remains PCM at the board boundary. When set, this is the + // wire format advertised in hello and handled by the Linx provider codec. + // Keeping the two formats separate lets the local wake detector and serial + // PCM fixture continue to operate without exposing encoded frames to them. + std::optional preferred_audio; /** * @brief 校验连接配置是否完整。 @@ -33,7 +38,7 @@ struct LinxConnectionConfig { */ [[nodiscard]] bool valid() const { return !websocket_url.empty() && !token_ref.empty() && !device_id.empty() && !client_id.empty() && - playback_buffer_duration_ms > 0; + playback_buffer_duration_ms > 0 && (!preferred_audio.has_value() || preferred_audio->valid()); } }; diff --git a/components/voicelife_linx/src/linx_json_codec.cc b/components/voicelife_linx/src/linx_json_codec.cc index e7cf7326..47a613d0 100644 --- a/components/voicelife_linx/src/linx_json_codec.cc +++ b/components/voicelife_linx/src/linx_json_codec.cc @@ -117,6 +117,10 @@ Result LinxJsonCodec::EncodeHello(const voice::VoiceSessionConfig& if (!config.audio.valid() || !connection.valid()) { return Result::Failure(ErrorCode::kInvalidArgument, "Linx hello 音频参数无效"); } + const voice::AudioFormat wire_audio = connection.preferred_audio.value_or(config.audio); + if (!wire_audio.valid()) { + return Result::Failure(ErrorCode::kInvalidArgument, "Linx hello 线上音频参数无效"); + } JsonPtr root(cJSON_CreateObject()); cJSON_AddStringToObject(root.get(), "type", "hello"); cJSON_AddNumberToObject(root.get(), "version", 1); @@ -125,15 +129,15 @@ Result LinxJsonCodec::EncodeHello(const voice::VoiceSessionConfig& cJSON_AddStringToObject(root.get(), "transport", "websocket"); cJSON* audio = cJSON_AddObjectToObject(root.get(), "audio_params"); - cJSON_AddStringToObject(audio, "format", CodecName(config.audio.codec)); - cJSON_AddNumberToObject(audio, "sample_rate", config.audio.sample_rate_hz); - cJSON_AddNumberToObject(audio, "channels", config.audio.channels); - cJSON_AddNumberToObject(audio, "bit_depth", config.audio.bits_per_sample); - cJSON_AddStringToObject(audio, "endianness", "little"); - cJSON_AddNumberToObject(audio, "frame_duration", config.audio.frame_duration_ms); - - if (config.audio.codec == voice::AudioCodec::kPcmS16Le) { - const uint32_t frame_size = config.audio.sample_rate_hz * config.audio.frame_duration_ms / 1000U; + cJSON_AddStringToObject(audio, "format", CodecName(wire_audio.codec)); + cJSON_AddNumberToObject(audio, "sample_rate", wire_audio.sample_rate_hz); + cJSON_AddNumberToObject(audio, "channels", wire_audio.channels); + cJSON_AddNumberToObject(audio, "frame_duration", wire_audio.frame_duration_ms); + + if (wire_audio.codec == voice::AudioCodec::kPcmS16Le) { + cJSON_AddNumberToObject(audio, "bit_depth", wire_audio.bits_per_sample); + cJSON_AddStringToObject(audio, "endianness", "little"); + const uint32_t frame_size = wire_audio.sample_rate_hz * wire_audio.frame_duration_ms / 1000U; cJSON_AddNumberToObject(audio, "frame_size", frame_size); cJSON_AddStringToObject(audio, "sample_format", "signed_int16"); cJSON_AddNumberToObject(audio, "play_buffer_duration", connection.playback_buffer_duration_ms); @@ -165,7 +169,8 @@ Result LinxJsonCodec::EncodeListenStop(const voice::VoiceSessionCon JsonPtr root(cJSON_CreateObject()); cJSON_AddStringToObject(root.get(), "type", "listen"); cJSON_AddStringToObject(root.get(), "state", "stop"); - cJSON_AddStringToObject(root.get(), "mode", ModeName(config.mode)); + // Linx defines mode on listen.start only. Keep stop to the documented + // shape used by the reference SparkBot client. if (!config.session_id.empty()) { cJSON_AddStringToObject(root.get(), "session_id", config.session_id.c_str()); } diff --git a/components/voicelife_linx/src/linx_ota.cc b/components/voicelife_linx/src/linx_ota.cc index 4b370987..d9673a49 100644 --- a/components/voicelife_linx/src/linx_ota.cc +++ b/components/voicelife_linx/src/linx_ota.cc @@ -221,7 +221,8 @@ Result BuildLinxConnectionConfig(const LinxOtaResponse& re .token_ref = std::string(token_reference), .device_id = std::string(device_id), .client_id = std::string(client_id), - .agent_id = std::nullopt}; + .agent_id = std::nullopt, + .preferred_audio = std::nullopt}; if (!config.valid()) { return Result::Failure(ErrorCode::kInvalidArgument, "Linx OTA 连接配置无效"); } diff --git a/components/voicelife_linx/src/linx_speech_provider.cc b/components/voicelife_linx/src/linx_speech_provider.cc index 3d107f59..7a5cdf71 100644 --- a/components/voicelife_linx/src/linx_speech_provider.cc +++ b/components/voicelife_linx/src/linx_speech_provider.cc @@ -1,8 +1,15 @@ #include "voicelife/linx/linx_speech_provider.h" +#include #include #include +#ifdef ESP_PLATFORM +#include "esp_heap_caps.h" +#include "esp_log.h" +#include "esp_pthread.h" +#endif + namespace voicelife::linx { namespace { @@ -24,6 +31,41 @@ bool SameAudioFormats(const voice::VoiceAudioFormats& left, const voice::VoiceAu return SameFormat(left.capture, right.capture) && SameFormat(left.playback, right.playback); } +#ifdef ESP_PLATFORM +const char* MessageKindName(LinxMessageKind kind) { + switch (kind) { + case LinxMessageKind::kHello: + return "hello"; + case LinxMessageKind::kStt: + return "stt"; + case LinxMessageKind::kTts: + return "tts"; + case LinxMessageKind::kMcp: + return "mcp"; + case LinxMessageKind::kError: + return "error"; + case LinxMessageKind::kGoodbye: + return "goodbye"; + case LinxMessageKind::kLlm: + return "llm"; + } + return "unknown"; +} + +const char* TtsStateName(const std::optional& state) { + if (!state.has_value()) return "-"; + switch (*state) { + case LinxTtsState::kStart: + return "start"; + case LinxTtsState::kSentenceStart: + return "sentence_start"; + case LinxTtsState::kStop: + return "stop"; + } + return "unknown"; +} +#endif + } // namespace LinxSpeechProviderAdapter::LinxSpeechProviderAdapter(LinxTransportPort& transport, LinxProtocolCodecPort& codec, @@ -225,6 +267,14 @@ void LinxSpeechProviderAdapter::OnTransportConnected() { remote_session_id_.reset(); hello_status_ = Status::Ok(); } +#ifdef ESP_PLATFORM + ESP_LOGI("voicelife_linx", + "LINX_HELLO_REQUEST format=%d sample_rate=%u channels=%u bits=%u frame_ms=%u play_buffer_ms=%u", + static_cast(config_.audio.codec), static_cast(config_.audio.sample_rate_hz), + static_cast(config_.audio.channels), static_cast(config_.audio.bits_per_sample), + static_cast(config_.audio.frame_duration_ms), + static_cast(connection_.playback_buffer_duration_ms)); +#endif const Status status = Send(codec_.EncodeHello(config_, connection_)); if (!status.ok()) { { @@ -294,6 +344,16 @@ void LinxSpeechProviderAdapter::OnText(std::string_view message) { return; } const LinxInboundMessage& inbound = *decoded.value; +#ifdef ESP_PLATFORM + // Record the server's control sequence without exposing credentials. STT + // and TTS text are intentionally visible on the authorized hardware log so + // a reset can be correlated with the last protocol event. + const std::string_view text = inbound.text; + ESP_LOGI("voicelife_linx", "LINX_RX kind=%s tts_state=%s session_present=%d session_len=%u text=%.*s", + MessageKindName(inbound.kind), TtsStateName(inbound.tts_state), inbound.session_id.has_value() ? 1 : 0, + inbound.session_id.has_value() ? static_cast(inbound.session_id->size()) : 0U, + static_cast(std::min(text.size(), 160U)), text.data()); +#endif // Linx assigns session_id in its hello response. Only that first hello // can establish the remote ID; all later messages must match it. if (inbound.kind != LinxMessageKind::kHello) { @@ -327,6 +387,15 @@ void LinxSpeechProviderAdapter::OnText(std::string_view message) { } { const LinxAudioParams& negotiated = *inbound.audio_params; +#ifdef ESP_PLATFORM + ESP_LOGI("voicelife_linx", + "LINX_HELLO_RESPONSE format=%d sample_rate=%u channels=%u bits=%u frame_ms=%u " + "session_present=%d session_len=%u", + static_cast(negotiated.codec), static_cast(negotiated.sample_rate_hz), + static_cast(negotiated.channels), static_cast(negotiated.bits_per_sample), + static_cast(negotiated.frame_duration_ms), inbound.session_id.has_value() ? 1 : 0, + inbound.session_id.has_value() ? static_cast(inbound.session_id->size()) : 0U); +#endif if (negotiated.codec != config_.audio.codec) { Emit(Event(voice::VoiceEventKind::kError, "Linx hello 改变音频编码，但当前未配置转码策略")); { @@ -442,6 +511,25 @@ void LinxSpeechProviderAdapter::StartMcpWorker() { std::lock_guard lock(mcp_mutex_); if (mcp_worker_.joinable()) return; mcp_stop_ = false; +#ifdef ESP_PLATFORM + // MCP handlers wait on the Runtime worker and may carry a large JSON-RPC + // response through std::function/condition_variable frames. The ESP-IDF + // pthread default is only 3072 bytes and is allocated from internal RAM; + // that is insufficient for the first initialize/tools/list exchange and + // canaries report it later as a "task pthread" overflow. Give this one + // worker a bounded PSRAM stack without changing other pthread users. + esp_pthread_cfg_t pthread_config = esp_pthread_get_default_config(); + pthread_config.stack_size = 16 * 1024; + pthread_config.stack_alloc_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; + pthread_config.thread_name = "voicelife_linx_mcp"; + pthread_config.inherit_cfg = false; + if (esp_pthread_set_cfg(&pthread_config) != ESP_OK) { + ESP_LOGW("voicelife_linx", "LINX_MCP_PTHREAD_CONFIG_FAILED=1"); + } else { + ESP_LOGI("voicelife_linx", "LINX_MCP_PTHREAD_CONFIG stack_bytes=%u caps=spiram", + static_cast(pthread_config.stack_size)); + } +#endif mcp_worker_ = std::thread([this]() { McpWorkerLoop(); }); } diff --git a/components/voicelife_linx_esp/include/voicelife/linx_esp/linx_tx_policy.h b/components/voicelife_linx_esp/include/voicelife/linx_esp/linx_tx_policy.h index 621f7901..70b9ae49 100644 --- a/components/voicelife_linx_esp/include/voicelife/linx_esp/linx_tx_policy.h +++ b/components/voicelife_linx_esp/include/voicelife/linx_esp/linx_tx_policy.h @@ -10,16 +10,17 @@ enum class LinxTextTxLane { kControl, kMediaOrdered }; /** * @brief 为 Linx 文本控制帧选择发送队列。 * - * listen.stop 是当前上行语音的结束边界，必须排在已采集 PCM 之后； - * abort 则需要立即抢占旧音频。因此只有前者进入媒体 FIFO。 + * listen.start/stop 是当前上行语音的起止边界，必须与 PCM 共享同一 FIFO； + * abort 则需要立即抢占旧音频，因此继续进入控制 FIFO。 * * @param message 已编码的 Linx 文本控制帧。 - * @return listen.stop 返回 kMediaOrdered，其他文本返回 kControl。 + * @return listen.start/stop 返回 kMediaOrdered，其他文本返回 kControl。 */ [[nodiscard]] inline LinxTextTxLane SelectLinxTextTxLane(std::string_view message) { const bool is_listen = message.find("\"type\":\"listen\"") != std::string_view::npos; - const bool is_listen_stop = is_listen && message.find("\"state\":\"stop\"") != std::string_view::npos; - return is_listen_stop ? LinxTextTxLane::kMediaOrdered : LinxTextTxLane::kControl; + const bool is_listen_boundary = is_listen && (message.find("\"state\":\"start\"") != std::string_view::npos || + message.find("\"state\":\"stop\"") != std::string_view::npos); + return is_listen_boundary ? LinxTextTxLane::kMediaOrdered : LinxTextTxLane::kControl; } } // namespace voicelife::linx_esp diff --git a/components/voicelife_linx_esp/src/esp_websocket_events.cc b/components/voicelife_linx_esp/src/esp_websocket_events.cc index 63880405..bd1c25bc 100644 --- a/components/voicelife_linx_esp/src/esp_websocket_events.cc +++ b/components/voicelife_linx_esp/src/esp_websocket_events.cc @@ -53,20 +53,54 @@ void EspWebSocketTransport::Impl::Enqueue(int32_t event_id, const esp_websocket_ // - TCP 有序 FIN（esp-tls 报 TCP_CLOSED_FIN） // 均映射为 kDisconnected（触发自动重连），其余才是真正故障（证书/握手/超时）。 const auto error_type = event_data != nullptr ? event_data->error_handle.error_type : WEBSOCKET_ERROR_TYPE_NONE; - const bool ordered_close = - error_type == WEBSOCKET_ERROR_TYPE_SERVER_CLOSE || - (event_data != nullptr && event_data->error_handle.esp_tls_last_esp_err == ESP_ERR_ESP_TLS_TCP_CLOSED_FIN); - if (ordered_close) { + const bool tcp_transport_error = error_type == WEBSOCKET_ERROR_TYPE_TCP_TRANSPORT; + // ESP-IDF leaves error_handle diagnostic members unspecified for some + // ERROR_TYPE_NONE callbacks (notably peer TCP RST/SSL read failure). + // Never interpret those bytes as a TLS failure: doing so turns a + // recoverable disconnect into provider_error and an error screen. + const bool diagnostics_valid = event_data != nullptr && error_type != WEBSOCKET_ERROR_TYPE_NONE; + const int handshake_status = diagnostics_valid ? event_data->error_handle.esp_ws_handshake_status_code : 0; + const int tls_last_error = diagnostics_valid ? event_data->error_handle.esp_tls_last_esp_err : 0; + const int tls_stack_error = diagnostics_valid ? event_data->error_handle.esp_tls_stack_err : 0; + const int tls_cert_flags = diagnostics_valid ? event_data->error_handle.esp_tls_cert_verify_flags : 0; + const int socket_errno = diagnostics_valid ? event_data->error_handle.esp_transport_sock_errno : 0; + const bool handshake_failed = handshake_status != 0; + const bool tls_failed = tls_last_error != 0 || tls_stack_error != 0 || tls_cert_flags != 0; + const bool ordered_close = error_type == WEBSOCKET_ERROR_TYPE_SERVER_CLOSE || + (tcp_transport_error && event_data != nullptr && + event_data->error_handle.esp_tls_last_esp_err == ESP_ERR_ESP_TLS_TCP_CLOSED_FIN); + // On ESP-IDF, a peer TCP RST can arrive as ERROR_TYPE_NONE with all + // diagnostic fields zero. It is still a lost WebSocket connection and + // must enter the reconnect path. Handshake/TLS failures retain the + // error path so invalid credentials and certificates are not retried + // as if the session had been cleanly disconnected. + const bool retryable_transport_loss = + ordered_close || + (event_data != nullptr && !handshake_failed && !tls_failed && + (tcp_transport_error || error_type == WEBSOCKET_ERROR_TYPE_NONE) && event_data->close_status_code == 0); + if (event_data != nullptr) { + ESP_LOGW(detail::kTag, + "LINX_WS_ERROR_EVENT event=ERROR classified=%s type=%u close=%d handshake=%d tls_valid=%d tls=%d " + "stack=%d cert_flags=%d errno=%d", + retryable_transport_loss ? "disconnect" : "error", static_cast(error_type), + event_data->close_status_code, handshake_status, diagnostics_valid ? 1 : 0, tls_last_error, + tls_stack_error, tls_cert_flags, socket_errno); + } else { + ESP_LOGW(detail::kTag, "LINX_WS_ERROR_EVENT event=ERROR classified=error type=%u close=0 event_data=null", + static_cast(error_type)); + } + if (retryable_transport_loss) { envelope.kind = detail::EventKind::kDisconnected; envelope.opcode = static_cast(error_type); } else { envelope.kind = detail::EventKind::kError; if (event_data != nullptr) { - envelope.tls_last_error = event_data->error_handle.esp_tls_last_esp_err; - envelope.tls_stack_error = event_data->error_handle.esp_tls_stack_err; - envelope.tls_cert_flags = event_data->error_handle.esp_tls_cert_verify_flags; - envelope.handshake_status = event_data->error_handle.esp_ws_handshake_status_code; - envelope.socket_errno = event_data->error_handle.esp_transport_sock_errno; + envelope.handshake_status = handshake_status; + envelope.close_status_code = event_data->close_status_code; + envelope.tls_last_error = tls_last_error; + envelope.tls_stack_error = tls_stack_error; + envelope.tls_cert_flags = tls_cert_flags; + envelope.socket_errno = socket_errno; envelope.opcode = static_cast(error_type); } } @@ -129,6 +163,9 @@ void EspWebSocketTransport::Impl::TxEntry(void* argument) { void EspWebSocketTransport::Impl::TxLoop() { // 唯一 TX 任务：按队列顺序发送文本/音频，TLS 只在本任务运行。 // 独立的短 TX 超时避免写阻塞拖垮采集；网络接收仍使用其正常预算。 + uint64_t last_audio_generation = 0; + uint64_t last_audio_sequence = 0; + bool have_audio_sequence = false; while (running_.load()) { detail::LinxTxItem* item = nullptr; // 控制命令优先；作为音频结束边界的 listen.stop 已进入媒体 FIFO， @@ -157,24 +194,19 @@ void EspWebSocketTransport::Impl::TxLoop() { pdMS_TO_TICKS(options_.tx_timeout_ms)); }); const size_t want = item->payload.size(); + const auto kind = item->kind; + const uint64_t generation = item->generation; + const uint64_t sequence = item->sequence; ReleaseTxItem(item); item = nullptr; if (!sent_current) { continue; } if (sent < 0 || static_cast(sent) != want) { - // 发送失败（写阻塞/短写/连接已断）：不能直接 esp_websocket_client_stop - // ——stop 会停止客户端，ESP 内建自动重连（disable_auto_reconnect=false） - // 随之失效，Session 永久卡在非 Ready（无法二次唤醒/说话）。 - // 正确做法：停止后立即重启 client，让内建自动重连继续负责重连 - // （单一重连执行者），随后断开事件会走 transport_disconnected 恢复。 - ESP_LOGW(detail::kTag, "LINX_TX_SEND_FAIL sent=%d want=%u, restart client for reconnect", sent, + // 发送失败时交给 ESP-IDF 客户端自己的自动重连状态机。TX 任务不能 + // 并发 stop/start，否则会与客户端重连任务竞争并丢失后续唤醒。 + ESP_LOGW(detail::kTag, "LINX_TX_SEND_FAIL sent=%d want=%u, await client auto-reconnect", sent, static_cast(want)); - if (client_ != nullptr && !closing_.load()) { - (void)esp_websocket_client_stop(client_); - // 重启以恢复内建自动重连；start 会重新进入连接流程并自动重连。 - (void)esp_websocket_client_start(client_); - } // 本次连接的媒体和控制命令都不能穿过重连边界。仅清理 PCM // 会让失效的 listen.start/abort 在新连接上被错误发送。 detail::LinxTxItem* remaining = nullptr; @@ -188,6 +220,27 @@ void EspWebSocketTransport::Impl::TxLoop() { } continue; } + if (kind == detail::LinxTxItem::Kind::kAudio) { + // sequence restarts at zero for every listen.start media round. A + // new round is a boundary, not a missing frame in the previous one. + const bool new_media_round = sequence == 0 || generation != last_audio_generation; + if (have_audio_sequence && !new_media_round && sequence != last_audio_sequence + 1) { + ESP_LOGW(detail::kTag, "LINX_TX_AUDIO_GAP previous=%llu current=%llu generation=%llu", + static_cast(last_audio_sequence), + static_cast(sequence), static_cast(generation)); + } + last_audio_generation = generation; + last_audio_sequence = sequence; + have_audio_sequence = true; + ++tx_audio_sent_; + if (tx_audio_sent_ <= 3 || tx_audio_sent_ % 20 == 0) { + ESP_LOGI(detail::kTag, "LINX_TX_AUDIO_SENT count=%llu sequence=%llu bytes=%u", + static_cast(tx_audio_sent_), static_cast(sequence), + static_cast(want)); + } + } else { + ESP_LOGI(detail::kTag, "LINX_TX_TEXT_SENT bytes=%u", static_cast(want)); + } } if (tx_stopped_ != nullptr) { xSemaphoreGive(tx_stopped_); @@ -217,9 +270,11 @@ void EspWebSocketTransport::Impl::HandleEnvelope(const detail::EventEnvelope& en } return; case detail::EventKind::kError: { - ESP_LOGW(detail::kTag, "LINX_WS_ERROR type=%u tls=%d stack=%d cert_flags=%d handshake=%d errno=%d", - static_cast(envelope.opcode), envelope.tls_last_error, envelope.tls_stack_error, - envelope.tls_cert_flags, envelope.handshake_status, envelope.socket_errno); + ESP_LOGW(detail::kTag, + "LINX_WS_ERROR event=worker type=%u close=%d tls=%d stack=%d cert_flags=%d handshake=%d errno=%d", + static_cast(envelope.opcode), envelope.close_status_code, envelope.tls_last_error, + envelope.tls_stack_error, envelope.tls_cert_flags, envelope.handshake_status, + envelope.socket_errno); std::lock_guard status_lock(status_mutex_); error_status_ = Status::Error(ErrorCode::kUnavailable, "ESP Linx WebSocket 收到错误事件"); } diff --git a/components/voicelife_linx_esp/src/esp_websocket_impl.cc b/components/voicelife_linx_esp/src/esp_websocket_impl.cc index ff6e2d90..45146fbb 100644 --- a/components/voicelife_linx_esp/src/esp_websocket_impl.cc +++ b/components/voicelife_linx_esp/src/esp_websocket_impl.cc @@ -159,12 +159,13 @@ Status EspWebSocketTransport::Impl::SendText(std::string_view message) { } // 脱敏诊断：仅记录控制消息的 type/state 字段，不输出 token、设备 ID 或完整消息。 const bool is_listen = message.find("\"type\":\"listen\"") != std::string_view::npos; - const bool is_listen_stop = SelectLinxTextTxLane(message) == LinxTextTxLane::kMediaOrdered; + const bool is_listen_boundary = SelectLinxTextTxLane(message) == LinxTextTxLane::kMediaOrdered; + const bool is_listen_stop = is_listen_boundary && message.find("\"state\":\"stop\"") != std::string_view::npos; const bool is_abort = message.find("\"type\":\"abort\"") != std::string_view::npos; - const bool is_control = (is_listen && !is_listen_stop) || is_abort; - // abort 可以抢占旧音频；listen.stop 则必须位于已采集 PCM 之后，否则 - // 服务端会先封口再收到尾音。stop 进入媒体 FIFO，而其他文本继续走控制 FIFO。 - QueueHandle_t target = is_listen_stop ? tx_queue_ : tx_control_queue_; + const bool is_control = (is_listen && !is_listen_boundary) || is_abort; + // abort 可以抢占旧音频；listen.start/stop 必须与 PCM 共享媒体 FIFO，避免 + // TX worker 在 start 尚未发送时先取出首个二进制帧。 + QueueHandle_t target = is_listen_boundary ? tx_queue_ : tx_control_queue_; if (target == nullptr) { return Status::Error(ErrorCode::kUnavailable, "ESP Linx TX 队列未就绪"); } @@ -172,26 +173,37 @@ Status EspWebSocketTransport::Impl::SendText(std::string_view message) { if (item == nullptr) return Status::Error(ErrorCode::kUnavailable, "ESP Linx TX item pool 已满"); item->kind = detail::LinxTxItem::Kind::kText; item->generation = generation_.load(); + item->sequence = 0; item->payload.assign(message.begin(), message.end()); // 非控制文本可短暂等待 TX 队列空位，避免工具结果因慢网络而打断交互。 - // listen.stop 是实时音频的结束边界，必须保持非阻塞并留在媒体 FIFO。 - const TickType_t wait_ticks = (is_control || is_listen_stop) ? 0 : pdMS_TO_TICKS(150); + // listen.stop 是实时音频的结束边界：在等待 FIFO 空位前就关闭闸门， + // 确保 stop 入队期间不会继续接收并发送迟到的 PCM。 + // A stop is an ordered media boundary. Wait for the FIFO to drain rather + // than evicting PCM, otherwise the server can observe a truncated stream + // or a stop followed by a late binary frame and reset the WebSocket. + if (is_listen_stop) { + media_tx_open_ = false; + } + const TickType_t wait_ticks = + is_listen_stop ? pdMS_TO_TICKS(options_.tx_timeout_ms) : (is_control ? 0 : pdMS_TO_TICKS(150)); if (xQueueSend(target, &item, wait_ticks) != pdTRUE) { if (!is_listen_stop) { ReleaseTxItem(item); return Status::Error(ErrorCode::kUnavailable, "ESP Linx TX 队列已满"); } - // 音频 FIFO 满载时为结束标记保留一个槽位。丢弃最旧 PCM 保留最近语音， - // 然后由同一 FIFO 在最后一帧之后发送 stop，既有界又不截断当前尾音。 - detail::LinxTxItem* stale = nullptr; - if (xQueueReceive(tx_queue_, &stale, 0) == pdTRUE) { - ReleaseTxItem(stale); - } - if (xQueueSend(tx_queue_, &item, 0) != pdTRUE) { - ReleaseTxItem(item); - return Status::Error(ErrorCode::kUnavailable, "ESP Linx 音频 TX 队列已满"); - } - ESP_LOGW(detail::kTag, "LINX_TX_STOP_EVICTED_OLDEST_PCM"); + ReleaseTxItem(item); + return Status::Error(ErrorCode::kUnavailable, "ESP Linx 音频 TX 队列等待 stop 超时"); + } + if (is_listen_stop) { + ESP_LOGI(detail::kTag, "LINX_TX_STOP_QUEUE enqueued_audio=%llu dropped_audio=%llu queued_media=%u", + static_cast(tx_audio_enqueued_.load(std::memory_order_relaxed)), + static_cast(tx_audio_dropped_.load(std::memory_order_relaxed)), + static_cast(uxQueueMessagesWaiting(tx_queue_))); + } + if (is_listen_stop || is_abort) { + media_tx_open_ = false; + } else if (is_listen && message.find("\"state\":\"start\"") != std::string_view::npos) { + media_tx_open_ = true; } // 入队成功后打印（此前在入队前打印，队列满时会误报“已发送”）。 if (is_listen) { @@ -199,7 +211,11 @@ Status EspWebSocketTransport::Impl::SendText(std::string_view message) { : message.find("\"state\":\"start\"") != std::string_view::npos ? "start" : message.find("\"state\":\"stop\"") != std::string_view::npos ? "stop" : "?"; - ESP_LOGI(detail::kTag, "LINX_SEND listen state=%s", state); + const char* mode = message.find("\"mode\":\"auto\"") != std::string_view::npos ? "auto" + : message.find("\"mode\":\"manual\"") != std::string_view::npos ? "manual" + : message.find("\"mode\":\"realtime\"") != std::string_view::npos ? "realtime" + : "-"; + ESP_LOGI(detail::kTag, "LINX_SEND listen state=%s mode=%s", state, mode); } else if (is_abort) { ESP_LOGI(detail::kTag, "LINX_SEND abort"); } @@ -212,6 +228,11 @@ Status EspWebSocketTransport::Impl::SendAudio(voice::AudioFrame frame) { frame.payload.size() > static_cast(INT_MAX)) { return Status::Error(ErrorCode::kUnavailable, "ESP Linx Transport 尚未连接"); } + if (!media_tx_open_) { + // Capture can race the state transition that enqueues listen.stop. + // The frame is valid locally but must never cross that protocol fence. + return Status::Ok(); + } // 统一 TX 队列：音频帧移入队后立即返回，网络写由 TxTask 执行， // 避免 esp_websocket_client_send_bin 同步阻塞 I2S 采集链导致大量丢帧。 if (tx_queue_ == nullptr) { @@ -221,19 +242,21 @@ Status EspWebSocketTransport::Impl::SendAudio(voice::AudioFrame frame) { if (item == nullptr) return Status::Error(ErrorCode::kUnavailable, "ESP Linx TX item pool 已满"); item->kind = detail::LinxTxItem::Kind::kAudio; item->generation = frame.generation; + item->sequence = frame.sequence; item->payload = std::move(frame.payload); - if (xQueueSend(tx_queue_, &item, 0) != pdTRUE) { - // Capture is a real-time producer. Preserve the newest audio instead - // of retaining stale speech that makes the interaction feel delayed. - detail::LinxTxItem* stale = nullptr; - if (xQueueReceive(tx_queue_, &stale, 0) == pdTRUE) { - ReleaseTxItem(stale); - } - if (xQueueSend(tx_queue_, &item, 0) != pdTRUE) { - ReleaseTxItem(item); - return Status::Error(ErrorCode::kUnavailable, "ESP Linx TX 队列已满"); - } - } + // PCM is an ordered stream. Waiting briefly for the writer preserves the + // sequence instead of evicting an older frame and making STT observe a + // discontinuity. The capture producer is already decoupled by the audio + // handoff queue, so this bounded wait cannot block the I2S read task. + if (xQueueSend(tx_queue_, &item, pdMS_TO_TICKS(50)) != pdTRUE) { + tx_audio_dropped_.fetch_add(1, std::memory_order_relaxed); + ESP_LOGW(detail::kTag, "LINX_TX_AUDIO_DROP sequence=%llu queue_depth=%u", + static_cast(frame.sequence), + static_cast(uxQueueMessagesWaiting(tx_queue_))); + ReleaseTxItem(item); + return Status::Error(ErrorCode::kUnavailable, "ESP Linx TX 音频队列等待超时"); + } + tx_audio_enqueued_.fetch_add(1, std::memory_order_relaxed); return Status::Ok(); } @@ -243,6 +266,7 @@ Status EspWebSocketTransport::Impl::Close() { closing_.store(true); running_.store(false); state_ = TransportState::kDisconnected; + media_tx_open_ = false; { std::lock_guard callback_lock(callback_mutex_); accepting_events_.store(false); @@ -309,6 +333,7 @@ void EspWebSocketTransport::Impl::SetGeneration(uint64_t generation) { // generation before waiting for an in-flight TX write, so an envelope // captured by the previous turn is rejected while this transition waits. generation_.store(generation, std::memory_order_release); + media_tx_open_ = false; // An item may already have been dequeued by TxLoop. Advance generation // only after its check-and-write critical section has completed; otherwise // that old item could cross an interrupt or reconnect boundary. @@ -342,6 +367,7 @@ detail::LinxTxItem* EspWebSocketTransport::Impl::TryAcquireTxItem() { auto* item = &tx_items_[index]; item->kind = detail::LinxTxItem::Kind::kText; item->generation = 0; + item->sequence = 0; item->payload = voice::AudioPayload{}; return item; } @@ -422,8 +448,11 @@ bool EspWebSocketTransport::Impl::PrepareWorker() { return false; } // 统一 TX 队列：文本/音频/barrier 由唯一 TxTask 顺序发送。 - // 低延迟音频队列：8 x 20 ms 约 160 ms；满载时 SendAudio 丢旧保新。 - constexpr int kTxQueueDepth = 8; + // 64 x 20 ms 约 1.28 s；音频满载时由 SendAudio 有界等待，不淘汰旧 PCM。 + // A long Chinese utterance can briefly outpace the TLS writer by more than + // the previous 640 ms media FIFO. Keep this finite at 1.28 s; the item pool + // is sized to cover this queue plus the control lane and in-flight write. + constexpr int kTxQueueDepth = 64; #if CONFIG_SPIRAM && (configSUPPORT_STATIC_ALLOCATION == 1) tx_queue_ = xQueueCreateWithCaps(kTxQueueDepth, sizeof(detail::LinxTxItem*), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); tx_queue_uses_caps_ = tx_queue_ != nullptr; diff --git a/components/voicelife_linx_esp/src/esp_websocket_impl.h b/components/voicelife_linx_esp/src/esp_websocket_impl.h index 3264c9c7..b28c65d1 100644 --- a/components/voicelife_linx_esp/src/esp_websocket_impl.h +++ b/components/voicelife_linx_esp/src/esp_websocket_impl.h @@ -25,8 +25,10 @@ namespace detail { constexpr char kTag[] = "voicelife_linx_esp"; constexpr size_t kMaxEventChunkBytes = 4096; -// 8 media + 16 control + one in-flight writer + one replacement/stop item. -constexpr size_t kTxItemPoolCapacity = 26; +// 64 media + 16 control + one in-flight writer + one stop item. Audio items +// hold pooled PCM leases, so this remains bounded while absorbing the measured +// sub-second TLS stalls without silently evicting an earlier speech frame. +constexpr size_t kTxItemPoolCapacity = 82; constexpr EventBits_t kConnectedBit = BIT0; constexpr EventBits_t kFailedBit = BIT1; @@ -38,6 +40,7 @@ struct LinxTxItem { Kind kind = Kind::kText; voice::AudioPayload payload; uint64_t generation = 0; + uint64_t sequence = 0; }; struct EventEnvelope { @@ -53,6 +56,7 @@ struct EventEnvelope { int tls_cert_flags = 0; int handshake_status = 0; int socket_errno = 0; + int close_status_code = 0; std::array data{}; }; @@ -104,6 +108,12 @@ class EspWebSocketTransport::Impl final { std::array tx_item_in_use_{}; std::mutex tx_item_mutex_; bool tx_queue_uses_caps_ = false; + // listen.stop closes the media lane for the current generation. Late + // capture frames are discarded instead of being sent after the stop frame. + bool media_tx_open_ = false; + uint64_t tx_audio_sent_ = 0; + std::atomic tx_audio_enqueued_{0}; + std::atomic tx_audio_dropped_{0}; TaskHandle_t tx_task_ = nullptr; // linx_ws_tx 任务栈常驻 PSRAM、TCB 在内部 RAM（一次性分配、跨连接复用，随 // Transport 生命周期）：交互（采集+音频流）期间内部 RAM 最大连续块常 <16KB， diff --git a/components/voicelife_mcp/test/mcp_server_test.cc b/components/voicelife_mcp/test/mcp_server_test.cc index 96c6bb8c..67dc94c5 100644 --- a/components/voicelife_mcp/test/mcp_server_test.cc +++ b/components/voicelife_mcp/test/mcp_server_test.cc @@ -471,6 +471,7 @@ void TestToolListing() { yyjson_val* level = yyjson_obj_get(properties, "level"); yyjson_val* label = yyjson_obj_get(properties, "label"); Check(yyjson_equals_str(yyjson_obj_get(configure, "name"), "self.device.configure") && + yyjson_obj_get(configure, "type") == nullptr && yyjson_equals_str(yyjson_obj_get(level, "type"), "integer") && yyjson_get_sint(yyjson_obj_get(level, "minimum")) == 0 && yyjson_get_sint(yyjson_obj_get(level, "maximum")) == 100, diff --git a/components/voicelife_runtime/src/linx_ota_bootstrap.cc b/components/voicelife_runtime/src/linx_ota_bootstrap.cc index 96fafa63..45c4117b 100644 --- a/components/voicelife_runtime/src/linx_ota_bootstrap.cc +++ b/components/voicelife_runtime/src/linx_ota_bootstrap.cc @@ -58,6 +58,8 @@ constexpr EventBits_t kWifiConnectedBit = BIT0; constexpr EventBits_t kWifiFailedBit = BIT1; constexpr int kWifiConnectTimeoutMs = 15000; constexpr int kOtaAttempts = 3; +constexpr int kOtaBootstrapAttempts = 5; +constexpr std::array kOtaRetryDelayMs = {1000, 2000, 4000, 8000}; constexpr char kTag[] = "VoiceLifeLinxOta"; struct WifiCredentials { @@ -561,7 +563,7 @@ Result BootstrapLinxOtaConfig(std::string_view board const WifiProvisioningStatusSink& provisioning_status_sink) { Result last_failure = Result::Failure(ErrorCode::kUnavailable, "Linx OTA 初始化失败"); - for (int attempt = 1; attempt <= kOtaAttempts; ++attempt) { + for (int attempt = 1; attempt <= kOtaBootstrapAttempts; ++attempt) { auto device = ReadOtaDeviceInfo(board_identity, provisioning_status_sink); if (!device.ok() || !device.value.has_value()) { last_failure = Result::Failure(device.status.code, device.status.message); @@ -629,9 +631,10 @@ Result BootstrapLinxOtaConfig(std::string_view board } } } - if (attempt < kOtaAttempts) { - ESP_LOGW(kTag, "LINX_OTA_RETRY attempt=%d", attempt + 1); - vTaskDelay(pdMS_TO_TICKS(1000)); + if (attempt < kOtaBootstrapAttempts) { + const int delay_ms = kOtaRetryDelayMs[static_cast(attempt - 1)]; + ESP_LOGW(kTag, "LINX_OTA_RETRY attempt=%d delay_ms=%d", attempt + 1, delay_ms); + vTaskDelay(pdMS_TO_TICKS(delay_ms)); } } return last_failure; diff --git a/components/voicelife_runtime/src/runtime.cc b/components/voicelife_runtime/src/runtime.cc index 7e4a8433..f9ebbe05 100644 --- a/components/voicelife_runtime/src/runtime.cc +++ b/components/voicelife_runtime/src/runtime.cc @@ -364,7 +364,10 @@ class Runtime final { voice::VoiceSessionConfig config; config.session_id = "voicelife-linx-session"; config.provider_id = "xrobot-websocket"; - config.mode = voice::VoiceMode::kRealtime; + // SparkBot has no playback reference channel or AEC. Match the + // Xiaozhi SparkBot default: server VAD closes an auto-stop turn after + // playback drains, while realtime is reserved for AEC-capable boards. + config.mode = voice::VoiceMode::kAuto; config.audio.codec = voice::AudioCodec::kPcmS16Le; config.audio.sample_rate_hz = 16000; config.audio.channels = 1; @@ -555,7 +558,28 @@ class Runtime final { return method != nullptr && method->IsString() && method->string == "tools/call"; } + static std::string McpMethod(std::string_view payload) { + JsonValue request; + if (!ParseJson(payload, request).ok() || !request.IsObject()) return "invalid"; + const JsonValue* method = request.Get("method"); + return method != nullptr && method->IsString() ? method->string : "missing"; + } + + static std::string McpRequestId(std::string_view payload) { + JsonValue request; + if (!ParseJson(payload, request).ok() || !request.IsObject()) return "invalid"; + const JsonValue* id = request.Get("id"); + if (id == nullptr) return "notification"; + if (id->IsString()) return id->string; + if (id->kind == JsonValue::Kind::kNumber) return std::to_string(static_cast(id->number)); + return "non_scalar"; + } + Result HandleMcpRequest(std::string_view payload, std::string_view session_id) { + const std::string method = McpMethod(payload); + const std::string request_id = McpRequestId(payload); + ESP_LOGI(kTag, "MCP_RX method=%s id=%s bytes=%u session_len=%u", method.c_str(), request_id.c_str(), + static_cast(payload.size()), static_cast(session_id.size())); auto request = std::make_shared(); request->payload.assign(payload); request->session_id.assign(session_id); @@ -567,7 +591,8 @@ class Runtime final { } mcp_queue_.push_back(request); } - ESP_LOGI(kTag, "MCP_REQUEST_QUEUED bytes=%u", static_cast(payload.size())); + ESP_LOGI(kTag, "MCP_REQUEST_QUEUED method=%s id=%s bytes=%u", method.c_str(), request_id.c_str(), + static_cast(payload.size())); mcp_cv_.notify_one(); std::unique_lock lock(request->mutex); @@ -603,6 +628,10 @@ class Runtime final { const LinxMcpToolOutcome outcome = InspectLinxMcpToolOutcome(request->payload, response); session_->ReportToolResult(TruncateUtf8(outcome.summary, 96), outcome.success); } + ESP_LOGI(kTag, "MCP_TX method=%s id=%s bytes=%u result=%d", McpMethod(request->payload).c_str(), + McpRequestId(request->payload).c_str(), + response.ok() && response.value.has_value() ? static_cast(response.value->size()) : 0U, + response.ok() ? 1 : 0); ESP_LOGI(kTag, "MCP_TOOL_EXECUTED tool_call=%d result=%d", tool_call ? 1 : 0, response.ok() ? 1 : 0); { std::lock_guard lock(request->mutex); @@ -740,10 +769,27 @@ class Runtime final { : Status::Error(ErrorCode::kUnavailable, "测试注入端口不可用"); }; callbacks.end_turn = [this]() { + auto* injection = assembly_ != nullptr ? assembly_->test_audio_injection() : nullptr; + if (injection == nullptr) return Status::Error(ErrorCode::kUnavailable, "测试注入端口不可用"); + const Status disabled = injection->SetTestInputEnabled(false); + if (!disabled.ok()) return disabled; return EnqueueBoardInput(BoardInputAction::kPressUp) ? Status::Ok() : Status::Error(ErrorCode::kUnavailable, "语音测试结束事件未进入状态机队列"); }; + callbacks.begin_wake = [this]() { + auto* injection = assembly_ != nullptr ? assembly_->test_audio_injection() : nullptr; + if (injection == nullptr) return Status::Error(ErrorCode::kUnavailable, "测试注入端口不可用"); + if (!assembly_->wake_gate().standby()) { + return Status::Error(ErrorCode::kConflict, "本地唤醒注入要求设备处于待机"); + } + return injection->SetTestInputEnabled(true); + }; + callbacks.end_wake = [this]() { + auto* injection = assembly_ != nullptr ? assembly_->test_audio_injection() : nullptr; + return injection != nullptr ? injection->SetTestInputEnabled(false) + : Status::Error(ErrorCode::kUnavailable, "测试注入端口不可用"); + }; serial_voice_test_ = std::make_unique(std::move(callbacks)); return serial_voice_test_->Start(); } @@ -1152,12 +1198,23 @@ class Runtime final { RestoreStandbyFromWakeTask(); continue; } - // SparkBot 目前没有 AEC。确认播报完成后才会由 kTtsStopped 进入 - // kOpeningCapture，避免把“收到！”录回云端，也避免假“聆听中”。 + // SparkBot 没有 AEC。普通唤醒先请求一次明确的“收到！”确认音， + // 再按 detect -> listen.start 顺序进入同一 Linx 会话；VoiceSession + // 会等确认 TTS 的 stop 或有界超时后才打开物理麦克风，避免自我介绍 + // 或确认音被采进首轮用户语音。 const Status acknowledge = session_->NotifyLocalWakeWord(request.wake_word, "收到！"); if (!acknowledge.ok()) { ESP_LOGW(kTag, "唤醒确认请求失败: %s", acknowledge.message.c_str()); - // 确认请求失败：回待机，不显示"出错了/牛牛走了"。 + (void)EnqueueEvent(voice::VoiceInteractionEvent::kStandbyReady); + continue; + } + // Linx requires detect -> listen.start in one ordered control + // sequence. Keep the physical input gated until the greeting TTS + // ends; VoiceSession will reuse this Provider lease when the + // interaction event loop later requests BeginCapture(). + const Status provider_capture = session_->BeginProviderCapture(); + if (!provider_capture.ok()) { + ESP_LOGW(kTag, "唤醒后 Provider 监听启动失败: %s", provider_capture.message.c_str()); (void)EnqueueEvent(voice::VoiceInteractionEvent::kStandbyReady); } } @@ -1489,8 +1546,13 @@ class Runtime final { wake_ack_tts_started_at_us_ = esp_timer_get_time(); ESP_LOGI(kTag, "WAKE_ACK_LATENCY stage=tts_started ms=%lld", static_cast(wake_latency_ms)); } - } else if (evidence.event == "tts_first_audio" && wake_ack_tts_started_at_us_ > 0) { + } else if (evidence.event == "tts_first_audio" && wake_ack_requested_at_us_ > 0) { + // Binary PCM can be delivered to the session before the event-loop + // item for tts.start is processed. The first actual audio is still + // the strongest proof that the greeting stream is alive, so it must + // cancel the bounded wake-greeting timer in either ordering. CancelListenTimer(); + if (wake_ack_tts_started_at_us_ == 0) wake_ack_tts_started_at_us_ = esp_timer_get_time(); const int64_t audio_latency_ms = (esp_timer_get_time() - wake_ack_requested_at_us_) / 1000; ESP_LOGI(kTag, "WAKE_ACK_LATENCY stage=first_audio ms=%lld", static_cast(audio_latency_ms)); } else if (evidence.event == "tts_stopped" && wake_ack_tts_started_at_us_ > 0) { @@ -1570,6 +1632,19 @@ class Runtime final { // 听到声音。首段 PCM 到达前保留 deadline，防止下行缓冲把首轮卡住。 if (wake_ack_requested_at_us_ == 0) CancelListenTimer(); (void)EnqueueEvent(voice::VoiceInteractionEvent::kTtsStarted); + } else if (evidence.event == "local_wake_detect_requested") { + // detect 已进入 TX FIFO。Linx 可能随后发送本地唤醒问候 TTS； + // 先确认协议顺序，再等待 tts.stop，超时才开启干净的用户采集。 + const auto mode = session_ != nullptr ? session_->config().mode : voice::VoiceMode::kManual; + const char* mode_name = "manual"; + if (mode == voice::VoiceMode::kAuto) { + mode_name = "auto"; + } else if (mode == voice::VoiceMode::kRealtime) { + mode_name = "realtime"; + } + ESP_LOGI(kTag, "LOCAL_WAKE_PROTOCOL_ACCEPTED action=await_greeting mode=%s", mode_name); + (void)EnqueueEvent(voice::VoiceInteractionEvent::kWakeDetectionAccepted); + StartListenTimer(kWakeAckFirstAudioTimeoutMs); } else if (evidence.event == "local_wake_ack_requested" || evidence.event == "interrupt_ack_requested") { // 本地唤醒/打断确认已提交给 Provider。直到首段 PCM 到达前保留 // deadline；超时后直接开始采集，不能无限等待远端音频。 @@ -1589,7 +1664,7 @@ class Runtime final { } // 服务端可能先送文本字幕，数秒后才送 PCM。确认阶段只有实际 // tts_first_audio 才能解除 deadline，不能把字幕当成已播放。 - if (wake_ack_tts_started_at_us_ == 0) CancelListenTimer(); + if (wake_ack_requested_at_us_ == 0) CancelListenTimer(); if (!evidence.detail.empty()) { // 事件化：文本经事件循环应用（唯一写者），门控仍在事件循环校验。 stt_display_text_ = evidence.detail; @@ -2195,10 +2270,18 @@ class Runtime final { continue; } if (item.listen_timeout) { - if (interaction_.state() == voice::VoiceInteractionState::kAcknowledging || - (interaction_.state() == voice::VoiceInteractionState::kSpeaking && wake_ack_requested_at_us_ > 0 && - wake_ack_tts_started_at_us_ > 0)) { - ESP_LOGW(kTag, "ACK_FIRST_AUDIO_TIMEOUT transition=acknowledging_or_speaking->opening_capture"); + if (interaction_.state() == voice::VoiceInteractionState::kAcknowledging) { + // The Provider listen.start was already sent immediately + // after detect. Timeout only opens the physical input; an + // abort/restart here would create a second protocol turn. + ESP_LOGW(kTag, "ACK_FIRST_AUDIO_TIMEOUT transition=acknowledging->opening_capture"); + wake_ack_requested_at_us_ = 0; + wake_ack_tts_started_at_us_ = 0; + wake_ack_until_us_ = 0; + (void)HandleInteractionEvent(voice::VoiceInteractionEvent::kAcknowledgementTimedOut); + } else if (interaction_.state() == voice::VoiceInteractionState::kSpeaking && + wake_ack_requested_at_us_ > 0) { + ESP_LOGW(kTag, "ACK_FIRST_AUDIO_TIMEOUT transition=speaking->opening_capture"); if (session_) (void)session_->Interrupt(); // Interrupt 使旧确认流失效，迟到的 tts.stop 会由会话层 // 丢弃；这里必须立即清除归因，避免下一轮正常回复被误算 @@ -2249,6 +2332,13 @@ class Runtime final { wake_ack_tts_started_at_us_ = 0; wake_ack_until_us_ = now + kWakeAckDisplayUs; } + if (item.event == voice::VoiceInteractionEvent::kWakeDetectionAccepted) { + // Keep the wake-greeting timing lease alive. A detect has been + // accepted by TX, but the microphone remains closed until the + // optional server greeting finishes or the bounded timer fires. + ESP_LOGI(kTag, "LOCAL_WAKE_GREETING_WAIT state=%d timeout_ms=%u", + static_cast(interaction_.state()), static_cast(kWakeAckFirstAudioTimeoutMs)); + } const Status wake_status = HandleInteractionEvent(item.event, item.wake_word); if (item.event != voice::VoiceInteractionEvent::kWakeDetected && !wake_status.ok()) { ESP_LOGW(kTag, "INTERACTION_REJECTED event=%d state=%d err=%s", static_cast(item.event), diff --git a/components/voicelife_runtime/src/serial_voice_protocol.h b/components/voicelife_runtime/src/serial_voice_protocol.h index 4c37f634..8b9008ed 100644 --- a/components/voicelife_runtime/src/serial_voice_protocol.h +++ b/components/voicelife_runtime/src/serial_voice_protocol.h @@ -11,6 +11,10 @@ inline constexpr uint8_t kSerialVoiceProtocolVersion = 1; inline constexpr uint8_t kSerialVoiceBegin = 1; inline constexpr uint8_t kSerialVoicePcm = 2; inline constexpr uint8_t kSerialVoiceEnd = 3; +// Test-only frames for feeding the local wake detector while the board stays +// in standby. They must never be interpreted as interaction press events. +inline constexpr uint8_t kSerialVoiceWakeBegin = 4; +inline constexpr uint8_t kSerialVoiceWakeEnd = 5; inline constexpr std::size_t kSerialVoicePcmBytes = 16000U * 20U / 1000U * sizeof(int16_t); struct SerialVoiceFrameHeader { @@ -49,7 +53,9 @@ class SerialVoiceMagicMatcher final { if (header.kind == kSerialVoicePcm) { return header.payload_bytes == kSerialVoicePcmBytes; } - return (header.kind == kSerialVoiceBegin || header.kind == kSerialVoiceEnd) && header.payload_bytes == 0; + return (header.kind == kSerialVoiceBegin || header.kind == kSerialVoiceEnd || + header.kind == kSerialVoiceWakeBegin || header.kind == kSerialVoiceWakeEnd) && + header.payload_bytes == 0; } } // namespace voicelife::runtime::detail diff --git a/components/voicelife_runtime/src/serial_voice_test.cc b/components/voicelife_runtime/src/serial_voice_test.cc index bd14b63b..515e4c26 100644 --- a/components/voicelife_runtime/src/serial_voice_test.cc +++ b/components/voicelife_runtime/src/serial_voice_test.cc @@ -21,6 +21,12 @@ namespace voicelife::runtime { namespace { constexpr char kTag[] = "SerialVoiceTest"; +// Serial PCM arrives from a host-side real-time fixture. Unlike the I2S +// capture task, this dedicated test task may briefly wait for the bounded +// transport backlog to drain, preserving the exact source utterance instead +// of manufacturing a lost-frame failure during a TLS stall. +constexpr uint32_t kPayloadAcquireTimeoutMs = 2000; +constexpr uint32_t kPayloadAcquirePollMs = 5; Status Unavailable(const char* message) { return Status::Error(ErrorCode::kUnavailable, message); } } // namespace @@ -34,10 +40,14 @@ class SerialVoiceTest::Impl final { return Unavailable("串口语音测试只能在 ESP-IDF 目标运行"); #else if (task_ != nullptr) return Status::Ok(); - if (!callbacks_.begin_turn || !callbacks_.submit_pcm || !callbacks_.end_turn) { + if (!callbacks_.begin_turn || !callbacks_.submit_pcm || !callbacks_.end_turn || !callbacks_.begin_wake || + !callbacks_.end_wake) { return Status::Error(ErrorCode::kInvalidArgument, "串口语音测试回调不完整"); } - payload_pool_ = voice::AudioPayloadPool::Create(16, detail::kSerialVoicePcmBytes); + // The Linx TX worker can briefly retain more than 16 input frames while + // TLS is flushing. Keep the serial fixture from rejecting valid PCM + // merely because the network is momentarily slower than realtime. + payload_pool_ = voice::AudioPayloadPool::Create(32, detail::kSerialVoicePcmBytes); if (payload_pool_ == nullptr) return Unavailable("创建串口语音 payload pool 失败"); if (!usb_serial_jtag_is_driver_installed()) { usb_serial_jtag_driver_config_t config = { @@ -106,6 +116,17 @@ class SerialVoiceTest::Impl final { } } + voice::AudioPayload AcquirePcmPayload() { + const TickType_t deadline = xTaskGetTickCount() + pdMS_TO_TICKS(kPayloadAcquireTimeoutMs); + while (!stopping_.load()) { + voice::AudioPayload payload = payload_pool_->TryAcquire(); + if (payload.pooled()) return payload; + if (xTaskGetTickCount() >= deadline) break; + vTaskDelay(pdMS_TO_TICKS(kPayloadAcquirePollMs)); + } + return {}; + } + void Run() { ESP_LOGI(kTag, "SERIAL_VOICE_TEST_READY=1 protocol=VLVT-v1 pcm=s16le-16000-mono-20ms payload_bytes=%u", static_cast(detail::kSerialVoicePcmBytes)); @@ -138,6 +159,14 @@ class SerialVoiceTest::Impl final { LogResult("TURN_END", callbacks_.end_turn()); continue; } + if (frame_header.kind == detail::kSerialVoiceWakeBegin) { + LogResult("WAKE_BEGIN", callbacks_.begin_wake()); + continue; + } + if (frame_header.kind == detail::kSerialVoiceWakeEnd) { + LogResult("WAKE_END", callbacks_.end_wake()); + continue; + } if (frame_header.kind != detail::kSerialVoicePcm) { ESP_LOGW(kTag, "SERIAL_VOICE_FRAME_REJECT unknown_kind=%u", static_cast(frame_header.kind)); continue; @@ -150,9 +179,10 @@ class SerialVoiceTest::Impl final { .channels = 1, .bits_per_sample = 16, .frame_duration_ms = 20}; - frame.payload = payload_pool_->TryAcquire(); + frame.payload = AcquirePcmPayload(); if (!frame.payload.pooled()) { - ESP_LOGW(kTag, "SERIAL_VOICE_PCM=reject code=%d", static_cast(ErrorCode::kUnavailable)); + ESP_LOGW(kTag, "SERIAL_VOICE_PCM=reject code=%d reason=payload_backpressure_timeout", + static_cast(ErrorCode::kUnavailable)); continue; } std::memcpy(frame.payload.data(), payload.data(), payload.size()); diff --git a/components/voicelife_runtime/src/serial_voice_test.h b/components/voicelife_runtime/src/serial_voice_test.h index 8800def0..9793165e 100644 --- a/components/voicelife_runtime/src/serial_voice_test.h +++ b/components/voicelife_runtime/src/serial_voice_test.h @@ -13,13 +13,17 @@ struct SerialVoiceTestCallbacks { std::function begin_turn; std::function submit_pcm; std::function end_turn; + std::function begin_wake; + std::function end_wake; }; /** * Test-only USB serial PCM reader. * * Protocol: `VLVT`, version 1, kind, little-endian payload length. Kinds are - * begin=1 (empty), pcm=2 (exactly one 20 ms PCM frame), and end=3 (empty). + * begin=1 (empty), pcm=2 (exactly one 20 ms PCM frame), end=3 (empty), + * wake_begin=4 (empty), and wake_end=5 (empty). Wake frames inject audio + * into the local detector without changing the interaction state. */ class SerialVoiceTest final { public: diff --git a/components/voicelife_voice/include/voicelife/voice/voice_interaction_controller.h b/components/voicelife_voice/include/voicelife/voice/voice_interaction_controller.h index 34cc24c8..5c0f13bb 100644 --- a/components/voicelife_voice/include/voicelife/voice/voice_interaction_controller.h +++ b/components/voicelife_voice/include/voicelife/voice/voice_interaction_controller.h @@ -20,6 +20,8 @@ enum class VoiceInteractionEvent { /** 触摸松开：结束手动聆听。 */ kPressUp, kWakeDetected, + /** 普通本地唤醒 detect 已入队；等待可选服务端问候 TTS 或超时。 */ + kWakeDetectionAccepted, /** 打断确认已成功提交给 Provider：kInterrupting → kListening。 */ kInterruptAcknowledged, kCaptureStarted, diff --git a/components/voicelife_voice/include/voicelife/voice/voice_ports.h b/components/voicelife_voice/include/voicelife/voice/voice_ports.h index 706f8f60..166d4672 100644 --- a/components/voicelife_voice/include/voicelife/voice/voice_ports.h +++ b/components/voicelife_voice/include/voicelife/voice/voice_ports.h @@ -102,6 +102,14 @@ class AudioInputPort { * @return 停止成功返回 Ok。 */ virtual Status StopCapture() = 0; + /** @brief 丢弃采集端已经排队的 PCM，建立新的语音回合边界。 + * + * 默认端口没有独立的有界输入队列时无需操作；硬件端口应在自身 + * 队列锁内清除排队帧及未完成组帧状态。 + * @return 清理成功返回 Ok。 + */ + virtual Status DiscardPendingInput() { return Status::Ok(); } + /** @brief 释放硬件资源并清除回调。 */ virtual void Close() = 0; }; diff --git a/components/voicelife_voice/include/voicelife/voice/voice_session.h b/components/voicelife_voice/include/voicelife/voice/voice_session.h index 9c0345a7..b4bcb398 100644 --- a/components/voicelife_voice/include/voicelife/voice/voice_session.h +++ b/components/voicelife_voice/include/voicelife/voice/voice_session.h @@ -29,6 +29,14 @@ class VoiceSession { Status Start(const VoiceSessionConfig& config); /** @brief 开始采集音频。 @return 开始结果。 */ Status BeginCapture(); + /** + * @brief 仅发送 Provider 侧的 listen.start，不启动物理输入。 + * + * 本地唤醒后的 Linx 顺序要求 detect 后立即 start；SparkBot 无 AEC， + * 因此真实麦克风必须等问候 TTS 结束或超时后再打开。 + * @return Provider 监听启动结果。 + */ + Status BeginProviderCapture(); /** @brief 结束采集音频。 @return 结束结果。 */ Status EndCapture(); /** @@ -112,9 +120,17 @@ class VoiceSession { VoiceAudioFormats audio_formats_; VoiceSessionState state_ = VoiceSessionState::kStopped; bool audio_ready_ = false; + // Provider 已发送 listen.start、但物理输入尚未开启时为 true；用于 + // 将本地唤醒的协议监听阶段与无 AEC 板的物理采集阶段分开。 + bool provider_capture_active_ = false; // 本轮是否已收到有效输入（STT/工具调用），仅在其为 true 时接受服务端 TTS， // 避免空闲态误收上一轮残留回复。 bool response_armed_ = false; + // 普通本地唤醒的 listen.detect 可能触发服务端问候 TTS，即使本地没有 + // text_response。该租约只在 detect 发出后有效，直到 tts.start/stop 或超时 + // 由 Interrupt() 失效，不能把问候误判为残留回复。 + bool pending_local_wake_tts_ = false; + bool local_wake_tts_active_ = false; // 每段远端 TTS 仅上报一次首个成功入播放队列的音频帧，供 Runtime // 记录唤醒确认的端到端时延；不携带 PCM 或文本。 bool first_tts_audio_pending_ = false; diff --git a/components/voicelife_voice/include/voicelife/voice/wake_gate_audio_input.h b/components/voicelife_voice/include/voicelife/voice/wake_gate_audio_input.h index 0dfbb78b..bc93fe29 100644 --- a/components/voicelife_voice/include/voicelife/voice/wake_gate_audio_input.h +++ b/components/voicelife_voice/include/voicelife/voice/wake_gate_audio_input.h @@ -112,6 +112,15 @@ class WakeGateAudioInput final : public AudioInputPort { bool physical_running_ = false; bool detector_running_ = false; bool forwarding_ = false; + // StartCapture() restarts the physical producer to establish a hard + // boundary between detector PCM and cloud PCM. Frames delivered while the + // producer is being stopped/restarted belong to the old boundary and are + // dropped. + bool capture_transitioning_ = false; + // One 20 ms hardware period can already be outside the queue when the + // wake callback is consumed. Drop a bounded tail of such frames after the + // queue reset; later frames are the user's actual follow-up speech. + std::size_t capture_boundary_frames_to_drop_ = 0; std::chrono::steady_clock::time_point wake_suppressed_until_{}; }; diff --git a/components/voicelife_voice/src/voice_interaction_controller.cc b/components/voicelife_voice/src/voice_interaction_controller.cc index 9e2b3e5f..cc7efb05 100644 --- a/components/voicelife_voice/src/voice_interaction_controller.cc +++ b/components/voicelife_voice/src/voice_interaction_controller.cc @@ -79,6 +79,14 @@ Result VoiceInteractionController::Handle(VoiceInter return InvalidTransition(state_, event); } break; + case VoiceInteractionEvent::kWakeDetectionAccepted: + // Linx 可能在 detect 后主动发送一段唤醒问候 TTS。这里只确认 + // detect 已进入发送序列，不提前开麦；tts.stop 或有界超时负责开麦。 + if (state_ != VoiceInteractionState::kAcknowledging && state_ != VoiceInteractionState::kSpeaking && + state_ != VoiceInteractionState::kOpeningCapture && state_ != VoiceInteractionState::kListening) { + return InvalidTransition(state_, event); + } + break; case VoiceInteractionEvent::kCaptureStarted: // 事务式启动确认：kOpeningCapture → kListening（采集真正开始）； // 从打断重启采集也必须等待同一确认；kListening 幂等。 diff --git a/components/voicelife_voice/src/voice_session.cc b/components/voicelife_voice/src/voice_session.cc index 8e6db63e..a8f2bd10 100644 --- a/components/voicelife_voice/src/voice_session.cc +++ b/components/voicelife_voice/src/voice_session.cc @@ -75,8 +75,11 @@ Status VoiceSession::Start(const VoiceSessionConfig& config) { config_ = config; audio_formats_ = {.capture = config.audio, .playback = config.audio}; audio_ready_ = false; + provider_capture_active_ = false; state_ = VoiceSessionState::kStarting; response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; first_tts_audio_pending_ = false; awaiting_final_asr_ = false; pending_local_wake_echo_.clear(); @@ -163,6 +166,9 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { bool playback_aborted = false; bool interrupt_fence_reached = false; bool stop_input_for_tts = false; + bool local_wake_tts_started = false; + bool local_wake_tts_stopped = false; + std::string stale_reason; Status flush_status = Status::Ok(); uint64_t generation = 0; std::string pending_wake_word; @@ -173,6 +179,7 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { // session after interrupt or stop invalidated the old epoch. if (event.generation != generation_) { stale = true; + stale_reason = "generation_mismatch"; } else if (event.kind == VoiceEventKind::kDisconnected && audio_ready_ && state_ != VoiceSessionState::kStopped && state_ != VoiceSessionState::kFailed) { ++generation_; @@ -182,7 +189,10 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { generation = generation_; generation_changed = true; disconnected = true; + provider_capture_active_ = false; response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; awaiting_final_asr_ = false; pending_local_wake_echo_.clear(); interrupt_fence_pending_ = false; @@ -193,15 +203,16 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { // 送达上一段识别结果；该事件不得穿透到交互状态机重启“处理”。 // kToolCall 不武装：启动/重连时的 MCP 发现消息（tools/list）并非 // 用户本轮输入，不能提前放行服务端 TTS。 - if (state_ != VoiceSessionState::kCapturing && - !(state_ == VoiceSessionState::kReady && awaiting_final_asr_)) { - stale = true; - } else if (!pending_local_wake_echo_.empty() && event.text == pending_local_wake_echo_) { + if (!pending_local_wake_echo_.empty() && event.text == pending_local_wake_echo_) { // 本地唤醒已被设备消费，服务端回传的同一短语不是用户意图。 - // 这里必须保持采集：Abort 会把紧随唤醒词的真实指令一并丢弃， - // 并可能让 Runtime 的交互状态停留在 Listening。 + // 该回传可能早于或晚于问候 TTS 到达；两种顺序都只抑制这一条， + // 不得 Abort 或武装回复。 pending_local_wake_echo_.clear(); wake_echo_suppressed = true; + } else if (state_ != VoiceSessionState::kCapturing && + !(state_ == VoiceSessionState::kReady && awaiting_final_asr_)) { + stale = true; + stale_reason = "asr_outside_capture"; } else { pending_local_wake_echo_.clear(); response_armed_ = true; @@ -210,14 +221,17 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { } else if (event.kind == VoiceEventKind::kConnected && audio_ready_ && state_ == VoiceSessionState::kStarting) { state_ = VoiceSessionState::kReady; } else if (event.kind == VoiceEventKind::kTtsStarted) { - // 仅接受本轮请求产生的 TTS：必须先收到有效 STT/工具调用（response_armed_）。 + // 仅接受本轮请求产生的 TTS：必须先收到有效 STT/工具调用，或处于 + // 本地唤醒 detect 后等待服务端问候的租约（response_armed_）。 // 允许 kReady（listen.stop 后最终 STT 到达、Session 已回 kReady 的回应路径）、 // kCapturing、kThinking、kSpeaking。空闲且无本轮输入（未 armed）的残留 TTS // 一律忽略，避免设备在没有用户输入时擅自播报。 + const bool local_wake_tts = pending_local_wake_tts_; const bool armed = response_armed_ || state_ == VoiceSessionState::kSpeaking; if (interrupt_fence_pending_ || !armed || state_ == VoiceSessionState::kStopped || state_ == VoiceSessionState::kStarting || state_ == VoiceSessionState::kFailed) { stale = true; + stale_reason = interrupt_fence_pending_ ? "tts_interrupt_fence" : "tts_not_armed"; } else { // This board has no AEC path. Stop capture before accepting the // server's TTS binary frames, rather than running I2S RX/TX as a @@ -225,13 +239,21 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { stop_input_for_tts = state_ == VoiceSessionState::kCapturing; state_ = VoiceSessionState::kSpeaking; first_tts_audio_pending_ = true; + if (local_wake_tts) { + pending_local_wake_tts_ = false; + local_wake_tts_active_ = true; + local_wake_tts_started = true; + } } } else if (event.kind == VoiceEventKind::kTtsStopped) { if (interrupt_fence_pending_) { // 已先 Flush 且 response_armed_=false；这条终止标记是旧流所有 // 在途音频均已越过 WebSocket 顺序边界的唯一依据。 interrupt_fence_pending_ = false; + local_wake_tts_stopped = local_wake_tts_active_; response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; awaiting_final_asr_ = false; playback_aborted = true; interrupt_fence_reached = true; @@ -245,23 +267,30 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { state_ = VoiceSessionState::kReady; generation = generation_; playback_aborted = true; + local_wake_tts_stopped = local_wake_tts_active_; response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; } else if (state_ == VoiceSessionState::kSpeaking) { state_ = VoiceSessionState::kReady; + local_wake_tts_stopped = local_wake_tts_active_; // The response that armed this turn has completed. Retaining // it would allow a delayed, unrelated tts.start to restart // playback while the session is otherwise idle. response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; } else { // Linx 没有为每个 TTS 流携带独立 generation。一次本地唤醒后， // 服务端可能送达旧流的 tts.stop/abort；若当前正在采集或等最终 // STT，接受该事件会错误终止本轮并让 Runtime 永久停在“聆听中”。 stale = true; + stale_reason = "tts_stop_without_speaking"; } } } if (stale) { - Emit("stale_event_dropped", "provider event generation mismatch"); + Emit("stale_event_dropped", stale_reason.empty() ? "stale_state" : stale_reason); } else if (generation_changed) { provider_.SetGeneration(generation); Emit("transport_disconnected", "audio sending disabled until a new hello completes"); @@ -298,7 +327,7 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { return; } } - Emit("tts_started", ""); + Emit("tts_started", local_wake_tts_started ? "local_wake_greeting" : ""); } else if (event.kind == VoiceEventKind::kTtsSentenceStarted) { // 仅当处于播报状态（本轮 TTS 已被 kTtsStarted 接受）时才回显句子； // 空闲态残留 TTS（如服务端闲聊）不显示文本。 @@ -306,7 +335,7 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { Emit("tts_sentence_started", event.text); } } else if (event.kind == VoiceEventKind::kTtsStopped) { - Emit("tts_stopped", ""); + Emit("tts_stopped", local_wake_tts_stopped ? "local_wake_greeting" : ""); } else if (event.kind == VoiceEventKind::kError) { Emit("provider_error", event.text); } @@ -327,6 +356,8 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { { std::lock_guard lock(mutex_); response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; } Emit("interrupt_ack_failed", status.message); } else { @@ -339,19 +370,64 @@ void VoiceSession::HandleEvent(const VoiceEvent& event) { } } +Status VoiceSession::BeginProviderCapture() { + std::lock_guard lifecycle_lock(lifecycle_mutex_); + VoiceMode mode; + { + std::lock_guard lock(mutex_); + if (state_ != VoiceSessionState::kReady) { + return Status::Error(ErrorCode::kUnavailable, "语音会话当前不能开始 Provider 监听"); + } + if (provider_capture_active_) return Status::Ok(); + mode = config_.mode; + // Reserve the protocol phase before the provider call: a synchronous + // callback must not observe a second listen.start opportunity. + provider_capture_active_ = true; + } + Status status = provider_.StartCapture(mode); + if (!status.ok()) { + std::lock_guard lock(mutex_); + provider_capture_active_ = false; + response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; + pending_local_wake_echo_.clear(); + return status; + } + const char* mode_name = "manual"; + if (mode == VoiceMode::kAuto) { + mode_name = "auto"; + } else if (mode == VoiceMode::kRealtime) { + mode_name = "realtime"; + } + Emit("provider_capture_started", mode_name); + return Status::Ok(); +} + Status VoiceSession::BeginCapture() { std::lock_guard lifecycle_lock(lifecycle_mutex_); VoiceMode mode; + bool provider_already_active = false; { std::lock_guard lock(mutex_); if (state_ != VoiceSessionState::kReady) { return Status::Error(ErrorCode::kUnavailable, "语音会话当前不能开始采集"); } mode = config_.mode; + provider_already_active = provider_capture_active_; + if (!provider_already_active) { + // Reserve the protocol phase before the provider call for the same + // synchronous-callback safety as BeginProviderCapture(). + provider_capture_active_ = true; + } } - Status provider_status = provider_.StartCapture(mode); - if (!provider_status.ok()) { - return provider_status; + if (!provider_already_active) { + Status provider_status = provider_.StartCapture(mode); + if (!provider_status.ok()) { + std::lock_guard lock(mutex_); + provider_capture_active_ = false; + return provider_status; + } } Status input_status = input_.StartCapture(mode); if (input_status.ok()) { @@ -361,6 +437,9 @@ Status VoiceSession::BeginCapture() { next_sequence_ = 0; // 新回合开始：清零上一轮武装标记与 VAD 状态，只允许本轮有效输入武装回复。 response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; + pending_local_wake_echo_.clear(); vad_speech_seen_ = false; vad_silence_emitted_ = false; last_speech_at_ = {}; @@ -372,6 +451,14 @@ Status VoiceSession::BeginCapture() { // Input failed after the provider already started listening. The provider // must be stopped so the server does not stay in a half-open capture state. Status rollback = provider_.StopCapture(); + { + std::lock_guard lock(mutex_); + provider_capture_active_ = false; + response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; + pending_local_wake_echo_.clear(); + } if (!rollback.ok()) { { std::lock_guard lock(mutex_); @@ -388,6 +475,12 @@ Status VoiceSession::EndCapture() { { std::lock_guard lock(mutex_); if (state_ != VoiceSessionState::kCapturing) { + // VAD、物理松键和串口夹具可能同时提交结束请求。一次 stop + // 已经把会话收口到 ready 后，重复请求必须是幂等成功，不能 + // 把正常竞态升级成 Runtime failure。 + if (state_ == VoiceSessionState::kReady) { + return Status::Ok(); + } return Status::Error(ErrorCode::kUnavailable, "语音会话当前没有采集"); } } @@ -397,6 +490,7 @@ Status VoiceSession::EndCapture() { { std::lock_guard lock(mutex_); state_ = VoiceSessionState::kReady; + provider_capture_active_ = false; awaiting_final_asr_ = true; } Emit("capture_stopped", ""); @@ -419,6 +513,7 @@ Status VoiceSession::EndCapture() { next_sequence_ = 0; next_generation = generation_; state_ = VoiceSessionState::kReady; + provider_capture_active_ = false; awaiting_final_asr_ = false; } provider_.SetGeneration(next_generation); @@ -577,18 +672,26 @@ Status VoiceSession::NotifyLocalWakeWord(std::string_view wake_word, std::string if (state_ != VoiceSessionState::kReady || wake_word.empty()) { return Status::Error(ErrorCode::kUnavailable, "语音会话当前不能通知本地唤醒"); } - // A text_response is a provider-requested system utterance. Arm only - // that response; unrelated idle TTS remains rejected by HandleEvent. - response_armed_ = !text_response.empty(); + // A normal detect can produce a server-side greeting even without a + // text_response. Arm that one expected TTS stream; unrelated idle TTS + // remains rejected when no local wake lease exists. + response_armed_ = true; + pending_local_wake_tts_ = text_response.empty(); + local_wake_tts_active_ = false; pending_local_wake_echo_ = text_response.empty() ? std::string(wake_word) : std::string{}; } Status status = provider_.NotifyLocalWakeWord(wake_word, text_response); if (!status.ok()) { std::lock_guard lock(mutex_); response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; pending_local_wake_echo_.clear(); } else if (status.ok()) { - Emit("local_wake_ack_requested", ""); + // A normal local wake only needs the protocol detect notification, but + // Linx may still return its own greeting TTS. Keep a bounded local-wake + // lease so that greeting is accepted and capture waits for tts.stop. + Emit(text_response.empty() ? "local_wake_detect_requested" : "local_wake_ack_requested", ""); } return status; } @@ -612,7 +715,10 @@ Status VoiceSession::InterruptAndNotifyLocalWakeWord(std::string_view wake_word, config_.generation = generation_; next_sequence_ = 0; state_ = VoiceSessionState::kReady; + provider_capture_active_ = false; response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; awaiting_final_asr_ = false; pending_local_wake_echo_.clear(); interrupt_fence_pending_ = wait_for_old_tts_stop; @@ -630,6 +736,8 @@ Status VoiceSession::InterruptAndNotifyLocalWakeWord(std::string_view wake_word, pending_interrupt_wake_word_.clear(); pending_interrupt_text_response_.clear(); response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; return !input_status.ok() ? input_status : (!abort_status.ok() ? abort_status : flush_status); } Emit("interrupted", "old audio generation invalidated"); @@ -647,6 +755,8 @@ Status VoiceSession::InterruptAndNotifyLocalWakeWord(std::string_view wake_word, if (!status.ok()) { std::lock_guard lock(mutex_); response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; return status; } Emit("interrupt_ack_requested", ""); @@ -659,11 +769,14 @@ Status VoiceSession::Interrupt() { bool capturing = false; bool needs_interrupt_fence = false; bool finalizing = false; + bool provider_listening = false; uint64_t generation = 0; { std::lock_guard lock(mutex_); finalizing = state_ == VoiceSessionState::kReady && awaiting_final_asr_; - if (state_ != VoiceSessionState::kCapturing && state_ != VoiceSessionState::kSpeaking && !finalizing) { + provider_listening = provider_capture_active_; + if (state_ != VoiceSessionState::kCapturing && state_ != VoiceSessionState::kSpeaking && !finalizing && + !provider_listening) { return Status::Ok(); } capturing = state_ == VoiceSessionState::kCapturing; @@ -675,10 +788,13 @@ Status VoiceSession::Interrupt() { config_.generation = generation_; next_sequence_ = 0; state_ = VoiceSessionState::kReady; + provider_capture_active_ = false; // Abort invalidates both the transport generation and any TTS request // that authorized it. A delayed tts.start must not resurrect the // cancelled turn under the new generation. response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; awaiting_final_asr_ = false; pending_local_wake_echo_.clear(); interrupt_fence_pending_ = needs_interrupt_fence; @@ -716,8 +832,11 @@ Status VoiceSession::Stop() { config_.generation = generation_; next_sequence_ = 0; audio_ready_ = false; + provider_capture_active_ = false; state_ = VoiceSessionState::kStopped; response_armed_ = false; + pending_local_wake_tts_ = false; + local_wake_tts_active_ = false; awaiting_final_asr_ = false; pending_local_wake_echo_.clear(); interrupt_fence_pending_ = false; diff --git a/components/voicelife_voice/src/wake_gate_audio_input.cc b/components/voicelife_voice/src/wake_gate_audio_input.cc index 4da5b432..1f8cbff0 100644 --- a/components/voicelife_voice/src/wake_gate_audio_input.cc +++ b/components/voicelife_voice/src/wake_gate_audio_input.cc @@ -5,6 +5,8 @@ namespace voicelife::voice { namespace { +constexpr std::size_t kCaptureBoundaryFrames = 4; + Status Unavailable(std::string message) { return Status::Error(ErrorCode::kUnavailable, std::move(message)); } } // namespace @@ -82,20 +84,46 @@ void WakeGateAudioInput::SuppressLocalWakeFor(uint32_t duration_ms) { } Status WakeGateAudioInput::StartCapture(VoiceMode mode) { - std::lock_guard lock(mutex_); - if (!opened_) return Unavailable("云端采集前必须先打开输入端口"); - if (forwarding_) return Status::Ok(); - const Status stop_status = StopDetectorLocked(); - if (!stop_status.ok()) return stop_status; - if (!physical_running_) { - const Status status = physical_input_.StartCapture(mode); - if (!status.ok()) { + bool clear_physical_queue = false; + { + std::lock_guard lock(mutex_); + if (!opened_) return Unavailable("云端采集前必须先打开输入端口"); + if (forwarding_) return Status::Ok(); + const Status stop_status = StopDetectorLocked(); + if (!stop_status.ok()) return stop_status; + clear_physical_queue = physical_running_; + // A physical delivery callback may already be in flight. Keep this + // guard set until the queue and assembler have been cleared. + capture_transitioning_ = clear_physical_queue; + capture_boundary_frames_to_drop_ = clear_physical_queue ? kCaptureBoundaryFrames : 0; + } + + if (clear_physical_queue) { + const Status clear_status = physical_input_.DiscardPendingInput(); + if (!clear_status.ok()) { + std::lock_guard lock(mutex_); + capture_transitioning_ = false; (void)StartDetectorLocked(); - return status; + return clear_status; } + } + + if (!clear_physical_queue) { + const Status start_status = physical_input_.StartCapture(mode); + if (!start_status.ok()) { + std::lock_guard lock(mutex_); + capture_transitioning_ = false; + (void)StartDetectorLocked(); + return start_status; + } + } + + { + std::lock_guard lock(mutex_); + capture_transitioning_ = false; physical_running_ = true; + forwarding_ = true; } - forwarding_ = true; return Status::Ok(); } @@ -143,7 +171,14 @@ void WakeGateAudioInput::HandlePhysicalFrame(AudioFrame frame) { LocalWakeDetectorPort* detector = nullptr; { std::lock_guard lock(mutex_); + if (capture_transitioning_) { + return; + } if (forwarding_) { + if (capture_boundary_frames_to_drop_ != 0) { + --capture_boundary_frames_to_drop_; + return; + } audio_sink = audio_sink_; } else if (detector_running_ && std::chrono::steady_clock::now() >= wake_suppressed_until_) { detector = &detector_; diff --git a/config/profiles/esp32s3-voicelife-pcb-serial-voice.json b/config/profiles/esp32s3-voicelife-pcb-serial-voice.json new file mode 100644 index 00000000..6fdb00b5 --- /dev/null +++ b/config/profiles/esp32s3-voicelife-pcb-serial-voice.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 1, + "id": "esp32s3-voicelife-pcb-serial-voice", + "target": "esp32s3", + "adapters": { + "audio": { + "driver": "esp32s3-pcm-port", + "capabilities": [ + "pcm-port", + "bounded-input", + "bounded-output", + "direct-i2s-simplex", + "transport-frame-assembly" + ], + "configRef": "env://VOICELIFE_AUDIO_PROFILE" + }, + "speech": { + "driver": "xrobot-websocket", + "capabilities": ["streaming-asr", "tts", "cancel-generation", "pcm"], + "configRef": "nvs://linx/websocket_url" + }, + "storage": { + "driver": "fatfs-sqlite", + "capabilities": ["persistent-sqlite"] + }, + "im": { + "driver": "voicelife-gateway", + "capabilities": ["https", "secure-credentials"], + "configRef": "nvs://im" + } + }, + "sdkconfig": [ + "CONFIG_LOG_DEFAULT_LEVEL_INFO=y", + "CONFIG_NVS_ENCRYPTION=y", + "CONFIG_NVS_SEC_KEY_PROTECT_USING_HMAC=y", + "CONFIG_NVS_SEC_HMAC_EFUSE_KEY_ID=0", + "CONFIG_VOICELIFE_IM_GATEWAY=y", + "CONFIG_LWIP_DHCP_GET_NTP_SRV=y", + "CONFIG_LWIP_SNTP_MAX_SERVERS=2", + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_CROSS_SIGNED_VERIFY=y", + "CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y", + "CONFIG_ESP_CONSOLE_SECONDARY_NONE=y", + "CONFIG_ESP_WIFI_NVS_ENABLED=n", + "CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384", + "CONFIG_FREERTOS_CHECK_STACKOVERFLOW=2", + "CONFIG_PARTITION_TABLE_CUSTOM_FILENAME=\"config/partitions/voicelife-pcb.csv\"", + "CONFIG_SPIRAM=y", + "CONFIG_SPIRAM_MODE_OCT=y", + "CONFIG_SPIRAM_SPEED_80M=y", + "CONFIG_SPIRAM_USE_MALLOC=y", + "CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM=y", + "CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=2048", + "CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=98304", + "CONFIG_SPIRAM_MEMTEST=n", + "CONFIG_SR_MN_CN_MULTINET7_QUANT=y", + "CONFIG_FATFS_SECTOR_4096=y", + "CONFIG_WL_SECTOR_SIZE_4096=y", + "CONFIG_VOICELIFE_STORAGE_FATFS=y", + "CONFIG_VOICELIFE_STORAGE_SQLITE=y", + "CONFIG_VOICELIFE_SERIAL_VOICE_TEST=y" + ] +} diff --git a/docs/README.md b/docs/README.md index 9fd3def8..6fcce60c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ | 语音日程联调 | [语音日程联调测试矩阵](engineering/schedule-voice-integration-test-matrix.md)、[语音日程边界调研与本期取舍](engineering/schedule-voice-industry-boundaries.md) | | 架构维护者 | [架构与适配器设计](architecture/design-guidelines.md)、[语音子架构](architecture/voice-subarchitecture.md)、[SQLite 存储子架构](architecture/storage-subarchitecture.md)、[ADR](adr/) | | 组件与服务维护者 | [SparkBot 显示组件](components/sparkbot-display.md)、[IM Gateway](services/im-gateway.md) | -| 协作与交付 | [协同开发](engineering/collaboration.md)、[质量门禁](engineering/ci-quality-gates.md)、[契约版本](engineering/contract-versioning.md)、[提交规范](engineering/commit-convention.md) | +| 协作与交付 | [协同开发](engineering/collaboration.md)、[质量门禁](engineering/ci-quality-gates.md)、[E2E 分层门禁与真机 HIL](engineering/e2e-layered-gates.md)、[发布清单](engineering/release-checklist.md)、[契约版本](engineering/contract-versioning.md)、[提交规范](engineering/commit-convention.md) | | 文档归档判断 | [文档放置规则](engineering/document-placement.md)、[历史归档](archive/README.md) | Issue [#264](https://github.com/1024XEngineer/VoiceLife/issues/264) 将旧小智能力迁移计划和旧 ESP32-S3 验证记录移入[历史归档](archive/README.md)，并删除没有独立长期价值的 SparkBot 阶段执行草稿。语音原始研究材料继续归档在 Issue [#150](https://github.com/1024XEngineer/VoiceLife/issues/150)。 diff --git a/docs/engineering/bailian-sparkbot-test-template.md b/docs/engineering/bailian-sparkbot-test-template.md new file mode 100644 index 00000000..9255ed4a --- /dev/null +++ b/docs/engineering/bailian-sparkbot-test-template.md @@ -0,0 +1,147 @@ +# 百炼 SparkBot 固定测试模板 + +结论：以后所有百炼到 SparkBot 的验证都从 `scripts/run_bailian_sparkbot_test.sh` 进入；它会安全读取 Key、先做真实 TTS 预检，再执行唤醒或多轮串口测试。这样不会再把 `key.txt` 中的 API 地址误当成 API Key，也不会把“脚本启动”当成链路通过。 + +下一步：插好 SparkBot 后先运行 `preflight`，预检通过再运行 `wake`；需要上下文时再运行 `multiturn`。每次把脚本生成的 JSON、`.meta.txt` 和串口 `.log` 一起留在本机证据目录。 + +## 1. 固定入口 + +```bash +cd /Users/mac/Desktop/project/VoiceLife + +# 只验证 Key、SDK 和百炼 TTS 的真实连通性，不打开串口 +scripts/run_bailian_sparkbot_test.sh preflight + +# 用百炼 TTS 合成“你好牛牛”，通过 USB 串口注入 SparkBot +scripts/run_bailian_sparkbot_test.sh wake + +# 在同一 Profile 下执行多轮上下文；可重复传入 --text 覆盖默认句集 +scripts/run_bailian_sparkbot_test.sh multiturn \ + --text '请记住今天的主题是日程管理。' \ + --text '把刚才的主题复述一遍。' \ + --text '请用一句话总结我们刚才谈了什么。' +``` + +`preflight` 会真实调用一次 TTS，并在日志目录写入 `preflight-*.json`；只有 JSON 中 `failed` 为 `0` 且 `audio_bytes.total` 大于 `0`，才算百炼可用。`wake` 和 `multiturn` 还会检查串口、`pyserial` 和 `ffmpeg`。 + +## 2. 配置读取 + +本项目本地配置文件可以同时包含兼容 API 地址和密钥，例如： + +```text +https://.../compatible-mode/v1 +https://.../apps/anthropic +sk-... +``` + +它不是 dotenv 文件。只读取第一条完整的 `sk-` 行： + +```bash +KEY_FILE="/Users/mac/Desktop/project/语音模型调用/key.txt" +export BAILIAN_KEY_FILE="$KEY_FILE" +scripts/run_bailian_sparkbot_test.sh preflight +``` + +脚本内部会去掉行首尾空白和 CR，只取第一条完整的 `sk-...` 行。等价的读取逻辑是： + +```bash +awk '{ line=$0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line ~ /^sk-[[:alnum:]_-]+$/) { print line; exit } }' "$KEY_FILE" +``` + +禁止使用 `DASHSCOPE_API_KEY="$(< key.txt)"`，也禁止把 Key 写入日志、命令输出或 Git。仓库脚本 `scripts/run_bailian_sparkbot_test.sh` 已内置同样的解析和检查逻辑。 + +## 3. 固定 Profile + +| 项目 | 默认值 | +| --- | --- | +| TTS | `qwen-audio-3.0-tts-flash` | +| 音色 | `longanlingxi` | +| 设备 | SparkBot 实板 | +| 串口 | `/dev/cu.usbmodem14401` | +| Linx 音频 | PCM、16 kHz、单声道、16 bit、20 ms | +| 日志目录 | `/tmp/voicelife-bailian-tests` | + +如果切换模型、音色、串口或地域，必须在测试记录中显式写出新值；不要只改命令而不改证据。 + +## 4. 预检与唤醒 + +不要再直接拼接 `DASHSCOPE_API_KEY` 或直接调用带有另一组默认模型的底层脚本。固定预检命令是： + +```bash +cd /Users/mac/Desktop/project/VoiceLife +scripts/run_bailian_sparkbot_test.sh preflight +``` + +预检通过后再执行实板唤醒： + +```bash +cd /Users/mac/Desktop/project/VoiceLife +scripts/run_bailian_sparkbot_test.sh wake +``` + +唤醒通过必须同时出现： + +```text +SERIAL_VOICE_EVIDENCE event=standby_ready +WAKE_DETECTED word=你好牛牛 +SERIAL_VOICE_EVIDENCE event=local_wake_ack_requested +SERIAL_VOICE_EVIDENCE event=tts_started +SERIAL_VOICE_EVIDENCE event=tts_stopped +LINX_SEND listen state=start mode=auto +SERIAL_VOICE_EVIDENCE event=capture_started +``` + +小智 SparkBot 的参考实现关闭 AEC 时使用 `auto`，并且在 +`CONFIG_SEND_WAKE_WORD_DATA=y` 时先发送 `listen.detect`；它只有开启设备或 +服务端 AEC 才切换 `realtime`。Linx 文档虽然推荐 `realtime`，但 VoiceLife +当前使用 `auto`，因为 SparkBot 没有 AEC、VoiceLife 也没有经过验证的本地 +回采打断能力。SparkBot 的本地 MultiNet 没有唤醒词 Opus 缓存， +但 Linx 仍要求先发送 `listen.detect` 建立会话；本链路只携带唤醒词，并请求 +一次短确认音。确认音结束后才发送 `listen.start(auto)`，屏幕再进入“聆听中”。 +这样避免服务端欢迎音频与首轮 PCM 交错，降低 Linx 会话边界断连。 +“别说了”打断和定时提醒仍可以单独使用正式的远端 TTS。 + +唤醒脚本用 `WAKE_BEGIN` 的请求/响应确认测试任务和待机状态，不把只在固件启动时打印一次的 +`SERIAL_VOICE_TEST_READY=1` 当作每次串口连接的就绪信号。因此设备已经运行、重新打开串口时也可以重复执行； +显式传入 `--reset-before-run` 时则会先通过 USB-Serial/JTAG 复位，再等待同一个握手。 + +若出现 `STARTUP_ERROR stage=session_start`、`provider_connect_failed` 或 `Connection reset by peer`，先标记为 Linx 连接失败并重试启动，不得把它写成“唤醒词识别失败”。 + +## 5. 普通对话和多轮 + +唤醒通过后，用同一套模型运行多轮上下文和日程语音测试。最小多轮模板： + +```bash +cd /Users/mac/Desktop/project/VoiceLife +scripts/run_bailian_sparkbot_test.sh multiturn \ + --text '请简单介绍一下你自己。' \ + --text '把刚才的回答缩短成一句话。' \ + --text '请说明我们刚才谈了什么。' +``` + +每轮必须记录并检查 `capture_stopped`、`stt_text_received`、`tts_started`、`tts_first_audio`、`tts_stopped`，以及屏幕的 `聆听中/处理中/说话中` 状态。日程验收在同一 Profile 下追加：创建、查询、修改、查询修改结果、删除、删除后查询；标题使用唯一值，避免历史数据干扰。 + +## 6. 重启与重连 + +1. 保存脚本产生的串口 `.log` 和 `.json`，记录固件 commit、Wi-Fi、Linx hello 和 `AUDIO_STATS`。 +2. 硬重启后等待 `standby_ready`，再次执行 `wake`，确认唤醒和普通对话仍成功。 +3. 观察一次 WebSocket 断线后的 `transport_disconnected`、重连和新的 `transport_connected`，再执行唤醒。 +4. 任何 `in_drop`、`out_reject`、`short_write`、I2S 错误、`INTERACTION_REJECTED` 或 MCP 响应超时都要单列，不能用“最终播报了”掩盖。 + +## 7. 证据记录 + +脚本默认将原始串口日志、JSON 汇总和不含密钥的 `.meta.txt` 写入 `/tmp/voicelife-bailian-tests`。PR/Issue 只粘贴时间、模型、固件、命令、成功/失败 marker 和汇总计数；密钥、Authorization、原始私密音频不进入仓库。每次测试都复制以下小表并填写： + +```text +日期/固件: +板卡/串口: +TTS 模型/音色: +唤醒: PASS/FAIL（日志） +普通对话: PASS/FAIL（轮数） +多轮上下文: PASS/FAIL（轮数） +日程 CRUD: PASS/FAIL（创建/查/改/删） +重启后唤醒: PASS/FAIL +重连后唤醒: PASS/FAIL +异常计数: in_drop= out_reject= short_write= i2s_err= queue_drop= +未决问题: +``` diff --git a/docs/engineering/e2e-journey-template.md b/docs/engineering/e2e-journey-template.md new file mode 100644 index 00000000..3bb2a526 --- /dev/null +++ b/docs/engineering/e2e-journey-template.md @@ -0,0 +1,42 @@ +# E2E Journey 模板 + +新增 journey 前复制本页到对应 Issue/PR，并在代码中复用 `scripts/run_e2e.py` 的统一生命周期。 + +## 标识 + +- Journey：`` +- 层级：`host` / `hil` +- Profile：`host` / `sparkbot` / `pcb` +- 负责人和依赖：``, `` + +## 生命周期 + +| 阶段 | 资源与动作 | 超时 | 清理 | 失败分类 | +| --- | --- | --- | --- | --- | +| prepare | 租约、临时目录、进程或设备检查 | `` | `` | configuration/lease/device/infrastructure | +| run | 真实请求或串口旅程 | `` | `` | external/product/device | +| assert | 有序状态和终态断言 | `` | `` | product | +| collect | 只采集 allowlist 字段 | `` | `` | infrastructure | +| cleanup | 撤销凭据、复位设备、释放租约 | `` | `` | cleanup | + +## Evidence + +- 公开字段：`` +- 明确禁止：原始串口、token、密码、SSID、个人数据、完整 URL 或命令输出 +- 故意失败测试：`` +- 脱敏校验：`python3 scripts/check_e2e_artifacts.py ` + +## 门禁入口 + +- 本地：`` +- PR / nightly / manual：`` +- artifact retention：`` +- required check 计划：先观察，达到稳定门槛后再提议 + +## 验收 + +- [ ] Host 和 HIL（如适用）使用同一 runner/evidence schema +- [ ] 硬超时、默认 retries=0、并发隔离已测试 +- [ ] 无设备、外部服务和产品断言能分别分类 +- [ ] 失败与清理 evidence 可上传且无敏感字段 +- [ ] 本地命令、workflow 命令和文档一致 diff --git a/docs/engineering/e2e-layered-gates.md b/docs/engineering/e2e-layered-gates.md new file mode 100644 index 00000000..15b06bf4 --- /dev/null +++ b/docs/engineering/e2e-layered-gates.md @@ -0,0 +1,116 @@ +# E2E 分层门禁与真机 HIL + +本文是 Host E2E、ESP32-S3 HIL 和发布验收的共同入口。工作流只负责编排；旅程语义、断言和 evidence schema 由 `scripts/run_e2e.py` 与 adapter 维护。 + +## 测试层级 + +| 层级 | 运行位置 | 入口 | 作用 | 门禁策略 | +| --- | --- | --- | --- | --- | +| unit | 主机 | `./scripts/run_checks.sh` | 领域、契约和适配器边界 | PR required | +| integration | 主机 + Postgres | Gateway `pnpm run ci` | 服务边界、持久化和 SSE | PR required | +| Host E2E | GitHub-hosted runner | `python3 scripts/run_e2e.py --layer host ...` | 真实 Gateway 进程和最小强提醒旅程 | PR required，硬超时，retries=0 | +| HIL smoke | 受控 self-hosted + 一台设备 | `--layer hil --journey im-pairing` | 烧录、配网、就绪和配对主链路 | manual，非 required | +| HIL E2E | 受控 self-hosted + 设备池 | 同一 runner CLI，按 profile 矩阵 | SparkBot/PCB 真实串口旅程 | 稳定门槛达成后再升级 | +| 手工验收 | 真实平台和物理场景 | [release checklist](release-checklist.md) | 微信、声学、显示、掉电和真实 Linx/ASR/TTS | 发布阻断，不由 Host/HIL 替代 | + +## 本地命令 + +Host E2E 需要 Node 24、pnpm lockfile 依赖和本地 Postgres： + +```bash +pnpm --dir services/im-gateway install --frozen-lockfile +DATABASE_URL=postgres://voicelife:voicelife@127.0.0.1:5432/voicelife \ + python3 scripts/run_e2e.py --layer host \ + --journey im-gateway-strong-reminder --profile host \ + --artifact-dir artifacts/host-e2e --timeout 60 --retries 0 +python3 scripts/check_e2e_artifacts.py artifacts/host-e2e +python3 scripts/render_e2e_summary.py artifacts/host-e2e +``` + +HIL 只能在已批准的设备和私有 Gateway 上运行。设备描述文件只含 `name`、`port`、`profile`，放在设备池的受控目录，不提交仓库： + +```bash +python3 -m pip install pyserial esptool +python3 scripts/run_e2e.py --layer hil --journey im-pairing \ + --profile sparkbot --artifact-dir artifacts/hil-sparkbot \ + --timeout 900 --retries 0 \ + --device "$VOICELIFE_HIL_DEVICE_ROOT/sparkbot.json" \ + --lease-dir "$VOICELIFE_HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" +python3 scripts/check_e2e_artifacts.py artifacts/hil-sparkbot +``` + +SparkBot 与 PCB 真实语音 HIL 分别使用 `esp32s3-esp-sparkbot-serial-voice` 和 +`esp32s3-voicelife-pcb-serial-voice` 测试 Profile。它需要受控环境中的 +`DASHSCOPE_API_KEY`、`dashscope` Python 包和 `ffmpeg`；API Key 不进入命令行参数或 evidence： + +```bash +DASHSCOPE_API_KEY="$DASHSCOPE_API_KEY" \ +python3 scripts/run_e2e.py --layer hil --journey voice \ + --profile sparkbot --artifact-dir artifacts/hil-voice \ + --timeout 900 --retries 0 \ + --device "$VOICELIFE_HIL_DEVICE_ROOT/sparkbot.json" \ + --lease-dir "$VOICELIFE_HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" \ + --input-tts dashscope \ + --tts-model qwen-audio-3.0-tts-flash \ + --voice longanlingxi \ + --text '你好牛牛，请介绍一下你自己。' \ + --text '把刚才的回答再简短一点。' +python3 scripts/check_e2e_artifacts.py artifacts/hil-voice +``` + +`voice` journey 会先独立执行输入 TTS preflight，再完成 application-only flash、临时设备注册、USB 配网和 +readiness。SparkBot 随后先通过真实唤醒词 PCM 验证 `wake_detected -> listen.detect -> 确认 TTS -> +capture_started`，复位回 readiness 后才运行多轮旅程；PCB 没有本地唤醒阶段，evidence 明确记录为 +`not_applicable`。多轮脚本将输入 TTS 结果转为 16 kHz mono S16LE PCM 注入真实设备。 +SparkBot 验证 LVGL 文本轨迹与滚动；PCB 验证 SSD1306 在每轮中实际绘制。结果证明真实云端语音和 +设备数字播放链路，不替代麦克风声学、AEC 或全双工验收。 + +输入夹具支持 `dashscope` 和 `aliyun-nls`。DashScope 使用 `DASHSCOPE_API_KEY`；NLS 使用 +`ALIYUN_NLS_APPKEY`、短期 `ALIYUN_NLS_TOKEN` 和可选 `ALIYUN_NLS_URL`。两者只负责生成用户输入， +Linx 下行 TTS 仍通过 `tts_started/tts_first_audio/tts_stopped`、I2S 帧和显示证据验证，禁止把二者混称为 +同一个 TTS Provider。完整接线和次日跑板步骤见 [语音 HIL 实板操作手册](hil-voice-device-runbook.md)。 + +Gateway host、目录、origin 和 user id 通过环境变量或受控 secret 注入；token 不进入 shell 参数、日志或 evidence。HIL 默认 `retries=0`，只允许基础设施层在 workflow 外显式重试，并且必须在 summary 中可见。 + +## workflow 与设备矩阵 + +- `.github/workflows/ci.yml` 仅运行稳定 Host E2E，并上传 14 天的 JSON evidence。 +- PR Host job 另运行一次受控的 `lifecycle-example` 故意失败，验证失败 evidence 和 `product` 分类仍能通过脱敏校验。 +- `.github/workflows/hil-nightly.yml` 当前仅开放 `workflow_dispatch`，只接受 `self-hosted, voicelife-hil` 及 profile 标签；`fail-fast=false`，SparkBot 和 PCB 独立汇总、独立上传 artifact。`voice` 需要受控 `DASHSCOPE_API_KEY`。 +- 手工触发可选 `sparkbot`、`pcb` 或 `all`，并可在选择单一 Profile 后指定 device descriptor 名称，用于隔离故障设备；配置受控 Runner 后再执行。 +- public GitHub-hosted Runner 不接触硬件或长期平台凭据。受控 Runner 只保存原始串口日志；公开 artifact 只包含通过 `scripts/check_e2e_artifacts.py` 校验的脱敏 JSON。 +- artifact 保留 14 天，访问权限跟随仓库 Actions 权限；发现凭据或个人数据时立即删除公开 artifact，保留受控私有原始日志并按 [SECURITY.md](../../SECURITY.md) 上报。 + +## 失败分类与重试 + +每个 evidence 都带 `failure_category`、`failed_phase` 和稳定 `message_code`，job summary 按 profile 展示： + +| 分类 | 例子 | 处理 | +| --- | --- | --- | +| `configuration` | 缺 descriptor、缺 pyserial、Profile 参数错误 | 修 Runner 配置，不重试 | +| `lease` | 设备或串口租约冲突 | 等待或释放租约后重跑，不归因于产品 | +| `device` | 串口打开失败、分区/Profile 不匹配 | 隔离该设备，修复或替换后手动重跑 | +| `infrastructure` | 构建工具、SSH/进程或 artifact 写入失败 | 修 Runner 基础设施；不把产品判定为失败 | +| `external` | Gateway/微信/ASR/TTS 服务不可用 | 记录外部依赖和时间窗口，允许一次可见的基础设施重试 | +| `product` | readiness 或 pairing 断言失败 | 保留 evidence，按产品缺陷处理，不自动重跑掩盖 | +| `timeout` / `cleanup` | 硬超时或资源回收失败 | 标记为失败并阻止设备继续进入池中 | + +没有设备租约、串口异常、外部服务故障和产品断言必须在 summary 中保持不同类别。设备 workflow 默认 retries=0；只有 `infrastructure`/`external` 经批准后才能有限重试。 + +## Evidence 与新增 journey + +公共 evidence 只允许 run/correlation id、时间、Profile、阶段状态、断言、数值指标和 HIL 的固件/commit/fingerprint。禁止原始串口、token、密码、SSID、用户/设备 ID、URL 凭据和任意命令输出。新增旅程按 [journey template](e2e-journey-template.md) 补齐：准备/运行/断言/采集/清理、失败类别、清理动作、超时预算、脱敏字段、Host/HIL 入口和一条故意失败测试。 + +## 稳定门槛 + +HIL 连续 14 次手工执行、每个 Profile 至少 10 次通过，且没有未分类失败、泄露扫描失败或设备租约泄露，才能提议升级为 required check。任意设备连续两次 `device`/`infrastructure` 失败即隔离并暂停该 Profile；不得通过增加 retries 把红灯隐藏。 + +真实微信公众号、H5 推迟、SSE 重连、声学、显示、物理输入和掉电验收仍见 [release checklist](release-checklist.md) 与 [Issue #132](https://github.com/1024XEngineer/VoiceLife/issues/132)。DashScope PCM 注入只能作为真实语音数字链路证据，不能替代这些声学和体验验收。 diff --git a/docs/engineering/hil-voice-device-runbook.md b/docs/engineering/hil-voice-device-runbook.md new file mode 100644 index 00000000..bb8e1bba --- /dev/null +++ b/docs/engineering/hil-voice-device-runbook.md @@ -0,0 +1,69 @@ +# 语音 HIL 实板操作手册 + +本文用于 SparkBot/PCB 接入受控 HIL Runner 后的首次验证。没有真实串口、设备和 Linx 凭据时，只能完成 +Host/contract 验证，不得宣称 HIL 通过。 + +## 1. Runner 准备 + +- 安装 `pyserial`、`esptool`、`ffmpeg` 和 ESP-IDF 构建依赖。 +- DashScope 路线安装 `dashscope` 并设置 `DASHSCOPE_API_KEY`。 +- 阿里云 NLS 路线安装官方 SDK（`pip install alibabacloud-nls-python-sdk`），设置 + `ALIYUN_NLS_APPKEY`、短期 + `ALIYUN_NLS_TOKEN`，按地域需要设置 `ALIYUN_NLS_URL`。 +- 准备项目外 descriptor：`{"schema_version":1,"name":"bench-a","port":"/dev/cu...","profile":"sparkbot"}`。 +- 设置 Gateway、设备池和租约目录相关的 `VOICELIFE_HIL_*` 环境变量；任何密钥、token、SSID 和原始串口 + 日志不得进入 Git 或公开 artifact。 + +## 2. 首次执行 + +先确认串口名称且没有其它串口监视器占用，然后执行： + +```bash +python3 scripts/run_e2e.py --layer hil --journey voice \ + --profile sparkbot \ + --artifact-dir artifacts/hil-voice-sparkbot \ + --timeout 900 --retries 0 \ + --device "$VOICELIFE_HIL_DEVICE_ROOT/sparkbot.json" \ + --lease-dir "$VOICELIFE_HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" \ + --input-tts dashscope \ + --tts-model qwen-audio-3.0-tts-flash \ + --voice longanlingxi \ + --text '你好牛牛，请介绍一下你自己。' \ + --text '把刚才的回答再简短一点。' \ + --text '请用一句话总结我们刚才的对话。' \ + --expect-terminal +``` + +改用 NLS 时只替换 provider、模型标签和 NLS 音色；模型与音色必须是在该 NLS 项目中真实可用的配置： + +```bash +--input-tts aliyun-nls --tts-model aliyun-nls-v1 --voice xiaoyun +``` + +## 3. 必须通过的证据 + +- TTS preflight：有音频字节、有 16 kHz PCM 帧，并记录请求数、首包和总时延。 +- SparkBot 唤醒：严格出现 `wake_detected`、`local_wake_ack_requested`、`tts_started`、 + `tts_first_audio`、`tts_stopped`、`capture_started`；确认音播放完之前不得开物理采集。 +- 每轮对话：ASR 严格匹配，出现下行 TTS start/first-audio/stop，显示状态完整。 +- 音频与队列：`in_drop=0`、`out_reject=0`、`short_write=0`、I2S error 为零、交互队列无 drop。 +- 终结回合：8 秒 wake guard 内无重新唤醒。 +- evidence：`scripts/check_e2e_artifacts.py` 校验通过；公开 JSON 不含原始话术、设备 ID 或凭据。 + +## 4. 失败处理 + +- `configuration`：补依赖、凭据或 descriptor，不重试掩盖。 +- `external`：记录 TTS/Linx 服务时间窗口；只允许一次显式基础设施重试。 +- `device`：关闭串口占用，核对板型和分区；连续两次失败先隔离设备。 +- `product`：保留私有串口日志和脱敏 evidence，不改成通过。 +- `Connection reset by peer`：记录发生在唤醒确认、长播报或普通回合哪个阶段；重连后再跑一轮，当前不得宣称 + 长播报重连闭环通过。 + +## 5. 跑板后回填 + +记录固件 commit、Gateway commit、板卡、串口、输入 TTS provider/model/voice、三轮完成数、ASR 精确匹配数、 +音频错误计数、唤醒结果、终结 guard、Linx RST/重连情况和脱敏 artifact 链接。SparkBot 与 PCB 分别保存结果。 diff --git a/docs/engineering/release-checklist.md b/docs/engineering/release-checklist.md new file mode 100644 index 00000000..0d673ea5 --- /dev/null +++ b/docs/engineering/release-checklist.md @@ -0,0 +1,30 @@ +# Release Checklist + +Host E2E 和 HIL 结果只能证明软件旅程的有限边界，不能替代真实平台和体验验收。发布 PR 必须记录 firmware commit、Gateway commit、Profile、测试账号类型和脱敏 evidence 位置。 + +## 自动门禁 + +- [ ] PR 的 Host unit/integration/Host E2E 通过，失败 evidence 已上传并完成敏感字段扫描。 +- [ ] 最近一次手工 HIL 对 SparkBot、PCB 分别有结果；HIL 仍为非 required 时记录原因。 +- [ ] workflow、设备标签、artifact retention 和 `retries=0` 配置未漂移。 +- [ ] 失败按 product / infrastructure / device / external / configuration 分类，没有用重试掩盖。 + +## 真实平台验收（#132） + +- [ ] 真实微信公众号绑定、强提醒通知和动作回执通过。 +- [ ] H5 推迟/确认链接在真实 HTTPS origin 下通过，过期、重复点击和错误 scope 有记录。 +- [ ] SSE 断线重连、Gateway 重启和 PostgreSQL 恢复通过。 +- [ ] 真实 Linx/ASR/TTS 凭据在受控环境使用；凭据未进入 PR、artifact 或设备镜像。 +- [ ] SparkBot、PCB voice HIL 通过：DashScope TTS -> PCM 注入 -> ASR -> Linx/WSS -> 下行 TTS/I2S/显示；记录 model、voice、Profile、固件 commit 和脱敏 evidence。 +- [ ] 声学：唤醒、AEC、全双工抢话和误唤醒在真实扬声器/麦克风下观察并记录。 +- [ ] 显示：SparkBot 状态、表情、字幕和物理输入在实板观察通过。 +- [ ] 掉电、重启、配网和旧凭据撤销/恢复通过。 + +## 证据与回退 + +- [ ] PR 只附最小连续非敏感摘录；原始串口和音频保留在受控私有目录。 +- [ ] 记录测试时间窗口、板型、固件/Gateway commit、Profile、账号类型和 evidence URL。 +- [ ] 已知 flaky 或外部故障有分类和后续负责人，不标记为产品通过。 +- [ ] 发布后发现回归时可暂停 HIL 手工 workflow 或将 Host E2E 降为非 required，但不得删除 journey、失败 evidence 或本清单。 + +关联：[#132](https://github.com/1024XEngineer/VoiceLife/issues/132)、[#288](https://github.com/1024XEngineer/VoiceLife/issues/288)。 diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 4ee42003..713ebee4 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -50,15 +50,16 @@ config VOICELIFE_STATE_FLOW_TEST is a test-only build option and must remain disabled in production. config VOICELIFE_SERIAL_VOICE_TEST - bool "Enable SparkBot serial PCM voice regression harness" + bool "Enable serial PCM voice regression harness" default n - depends on VOICELIFE_BOARD_ESP_SPARKBOT && ESP_CONSOLE_USB_SERIAL_JTAG && !VOICELIFE_STATE_FLOW_TEST + depends on ESP_CONSOLE_USB_SERIAL_JTAG && !VOICELIFE_STATE_FLOW_TEST help Start a test-only USB Serial/JTAG PCM input reader. It accepts a fixed, bounded 16 kHz S16LE mono protocol and routes frames through the normal - audio queue, VoiceSession, Linx transport and ES8311 output. The test - build emits unredacted voice text and detailed hardware telemetry for - board diagnosis. It must remain disabled in production profiles. + audio queue, VoiceSession, Linx transport and board-specific output. + The test build emits unredacted voice text and detailed hardware + telemetry for board diagnosis. It must remain disabled in production + profiles. if VOICELIFE_AUDIO_PROBE diff --git a/main/platform_assemblies.cc b/main/platform_assemblies.cc index 44ec9ebb..978bbbb8 100644 --- a/main/platform_assemblies.cc +++ b/main/platform_assemblies.cc @@ -22,12 +22,11 @@ namespace voicelife::runtime { namespace { // Linx can deliver several 20 ms PCM frames in one TLS/WebSocket burst. The -// 12-frame / 240 ms bound rejected audio during the real eight-turn context -// stress run, while 16 / 320 ms completed it losslessly with more than 4 MiB -// of free heap remaining. Keep this finite: it absorbs short transport bursts -// without turning downlink playback into an unbounded backlog. -constexpr std::size_t kSparkBotPlaybackQueueDepth = 16; -constexpr uint32_t kSparkBotPlaybackLatencyBudgetMs = 320; +// The V3 CRUD probe exposed a longer Linx TTS burst than the previous 960 ms +// window. Keep a finite 1.92 s jitter buffer so a burst is played in order +// without dropping PCM or turning playback into an unbounded cache. +constexpr std::size_t kSparkBotPlaybackQueueDepth = 96; +constexpr uint32_t kSparkBotPlaybackLatencyBudgetMs = 1920; /** @brief 从官方 SparkBot 板级 Profile 填充 LVGL 显示配置。 */ voicelife::display_sparkbot::SparkBotLcdConfig MakeSparkBotLcdConfig() { diff --git a/scripts/check_architecture.cmake b/scripts/check_architecture.cmake index c3a03ea2..344a1463 100644 --- a/scripts/check_architecture.cmake +++ b/scripts/check_architecture.cmake @@ -115,7 +115,7 @@ assert_dependencies(voicelife_mcp PRIVATE voicelife_schedule yyjson) assert_dependencies(voicelife_voice PUBLIC voicelife_contracts) assert_dependencies(voicelife_voice PRIVATE) assert_dependencies(voicelife_linx PUBLIC voicelife_contracts voicelife_voice) -assert_dependencies(voicelife_linx PRIVATE) +assert_dependencies(voicelife_linx PRIVATE pthread) assert_dependencies(voicelife_linx_esp PUBLIC voicelife_contracts voicelife_linx) assert_dependencies(voicelife_linx_esp PRIVATE esp_websocket_client esp-tls esp_event esp_timer freertos heap) assert_dependencies(voicelife_audio_esp PUBLIC voicelife_contracts voicelife_voice) diff --git a/scripts/check_e2e_artifacts.py b/scripts/check_e2e_artifacts.py new file mode 100644 index 00000000..ce76f66e --- /dev/null +++ b/scripts/check_e2e_artifacts.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Validate public E2E artifacts without printing their contents.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from e2e_evidence import EvidenceValidationError, scan_sensitive, validate_evidence + + +def validate_directory(root: Path) -> tuple[int, int]: + """Return (file_count, evidence_count) after validating an artifact directory.""" + if not root.is_dir(): + raise ValueError("artifact directory is missing") + files = sorted(path for path in root.rglob("*") if path.is_file()) + if not files: + raise ValueError("artifact directory is empty") + evidence_count = 0 + for path in files: + if path.suffix != ".json": + raise ValueError("public artifacts may only contain JSON") + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError("artifact JSON is unreadable") from error + if scan_sensitive(document): + raise ValueError("artifact contains sensitive data") + if path.name.startswith("evidence-"): + if not isinstance(document, dict): + raise ValueError("E2E evidence must be a JSON object") + try: + validate_evidence(document) + except EvidenceValidationError as error: + raise ValueError("E2E evidence does not match the public schema") from error + evidence_count += 1 + if evidence_count == 0: + raise ValueError("artifact directory has no E2E evidence") + return len(files), evidence_count + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact_dir", type=Path) + args = parser.parse_args(argv) + try: + file_count, evidence_count = validate_directory(args.artifact_dir) + except ValueError as error: + print(str(error), file=sys.stderr) + return 1 + print(f"validated {evidence_count} E2E evidence file(s) in {file_count} public artifact file(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/e2e_evidence.py b/scripts/e2e_evidence.py index 1a17f3e0..bf8ff126 100644 --- a/scripts/e2e_evidence.py +++ b/scripts/e2e_evidence.py @@ -16,6 +16,7 @@ HEX_COMMIT = re.compile(r"^[0-9a-f]{40}$") HEX_FINGERPRINT = re.compile(r"^[0-9a-f]{16}$") SAFE_NAME = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") +SAFE_MODEL = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") UTC_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3,6})?Z$") JWT_PATTERN = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}\b") SENSITIVE_KEY_PATTERN = re.compile( @@ -58,9 +59,54 @@ HIL_KEYS = frozenset( {"firmware_sha256", "gateway_commit", "device_fingerprint", "readiness_markers", "pairing_markers"} ) -METRIC_KEYS = frozenset({"resource_count", "bound_port_count", "namespace_count"}) +VOICE_HIL_KEYS = frozenset( + { + "firmware_sha256", + "gateway_commit", + "device_fingerprint", + "readiness_markers", + "input_tts_provider", + "input_tts_model", + "input_tts_voice", + "wake_stage", + } +) +METRIC_KEYS = frozenset( + { + "resource_count", + "bound_port_count", + "namespace_count", + "requested_turns", + "completed_turns", + "asr_exact_matches", + "input_tts_requests", + "input_tts_audio_bytes", + "input_tts_first_packet_ms_max", + "input_tts_total_ms_max", + "wake_completed", + "test_in_frames", + "out_frames", + "in_drop", + "out_reject", + "short_write", + "in_i2s_err", + "out_i2s_err", + "serial_pcm_rejections", + "display_content_snapshots", + } +) FAILURE_CATEGORIES = frozenset( - {"configuration", "infrastructure", "product", "device", "external", "timeout", "interrupted", "cleanup"} + { + "configuration", + "infrastructure", + "product", + "device", + "lease", + "external", + "timeout", + "interrupted", + "cleanup", + } ) PHASE_ORDER = ("prepare", "run", "assert", "collect", "cleanup") PHASES = frozenset(PHASE_ORDER) @@ -133,7 +179,7 @@ def _validate_assertion(assertion: Any) -> None: def _validate_metrics(metrics: Any) -> None: _reject(not isinstance(metrics, dict) or not set(metrics).issubset(METRIC_KEYS)) for value in metrics.values(): - _reject(type(value) is not int or not 0 <= value <= 1_000_000) + _reject(type(value) is not int or not 0 <= value <= 100_000_000) def _validate_hil(hil: Any) -> None: @@ -148,13 +194,27 @@ def _validate_hil(hil: Any) -> None: _reject(value["pairing_markers"] != ["scope_matched", "code_valid", "pending", "expired"]) +def _validate_voice_hil(hil: Any) -> None: + value = _exact_keys(hil, VOICE_HIL_KEYS) + _reject(not isinstance(value["firmware_sha256"], str) or HEX_SHA256.fullmatch(value["firmware_sha256"]) is None) + _reject(not isinstance(value["gateway_commit"], str) or HEX_COMMIT.fullmatch(value["gateway_commit"]) is None) + _reject( + not isinstance(value["device_fingerprint"], str) + or HEX_FINGERPRINT.fullmatch(value["device_fingerprint"]) is None + ) + _reject(value["readiness_markers"] != ["provisioned", "wifi_ready", "sntp_synced", "ready"]) + _reject(value["input_tts_provider"] not in {"dashscope", "aliyun-nls"}) + _reject(SAFE_MODEL.fullmatch(value["input_tts_model"]) is None or not _safe_name(value["input_tts_voice"])) + _reject(value["wake_stage"] not in {"passed", "not_applicable"}) + + def validate_evidence(document: dict[str, object]) -> None: """Validate the complete evidence allowlist and cross-field relations.""" value = _exact_keys(document, TOP_LEVEL_KEYS) _reject(value["schema_version"] != 1) _reject(not isinstance(value["run_id"], str) or HEX_ID.fullmatch(value["run_id"]) is None) _reject(not isinstance(value["correlation_id"], str) or HEX_ID.fullmatch(value["correlation_id"]) is None) - _reject(value["scope"] not in {"runner_contract_only", "hil_im_pairing"}) + _reject(value["scope"] not in {"runner_contract_only", "hil_im_pairing", "hil_voice"}) _reject(value["layer"] not in {"host", "hil"}) _reject(not _safe_name(value["journey"]) or not _safe_name(value["profile"])) _reject(not isinstance(value["started_at"], str) or UTC_TIMESTAMP.fullmatch(value["started_at"]) is None) @@ -165,7 +225,7 @@ def validate_evidence(document: dict[str, object]) -> None: _reject(type(value["hardware_verified"]) is not bool) if value["scope"] == "runner_contract_only": _reject(value["hardware_verified"] or value["hil"] is not None) - else: + elif value["scope"] == "hil_im_pairing": _reject( value["layer"] != "hil" or value["journey"] != "im-pairing" @@ -173,6 +233,15 @@ def validate_evidence(document: dict[str, object]) -> None: or value["hardware_verified"] is not True ) _validate_hil(value["hil"]) + else: + _reject( + value["layer"] != "hil" + or value["journey"] != "voice" + or value["profile"] not in {"sparkbot", "pcb"} + or value["status"] != "passed" + or value["hardware_verified"] is not True + ) + _validate_voice_hil(value["hil"]) failure_category = value["failure_category"] failed_phase = value["failed_phase"] @@ -216,6 +285,13 @@ def validate_evidence(document: dict[str, object]) -> None: _validate_assertion(assertion) _validate_metrics(value["metrics"]) + if value["scope"] == "hil_voice": + voice_hil = value["hil"] + voice_metrics = value["metrics"] + if value["profile"] == "sparkbot": + _reject(voice_hil["wake_stage"] != "passed" or voice_metrics.get("wake_completed") != 1) + else: + _reject(voice_hil["wake_stage"] != "not_applicable" or voice_metrics.get("wake_completed") != 0) cleanup = _exact_keys(value["cleanup"], CLEANUP_KEYS) _reject(cleanup["status"] not in {"passed", "failed"}) error_codes = cleanup["error_codes"] diff --git a/scripts/e2e_example_adapters.py b/scripts/e2e_example_adapters.py index 68947560..4cc07af6 100644 --- a/scripts/e2e_example_adapters.py +++ b/scripts/e2e_example_adapters.py @@ -43,6 +43,8 @@ def run(self, context: RunContext) -> dict[str, bool]: def assert_result(self, context: RunContext, result: object) -> list[AssertionResult]: values = result if isinstance(result, dict) else {} + if os.environ.get("VOICELIFE_E2E_CONTRACT_FAILURE") == "1": + return [AssertionResult(name="lifecycle_complete", passed=False, code="contract_failure")] passed = all(values.get(name) is True for name in values) return [AssertionResult(name="lifecycle_complete", passed=passed, code="ok" if passed else "incomplete")] @@ -159,8 +161,9 @@ def __init__(self, artifact_directory: Path) -> None: self._process: subprocess.Popen[str] | None = None def prepare(self, context: RunContext) -> None: - self.artifact_directory.mkdir(parents=True, exist_ok=True) - detail_path = self.artifact_directory / f"recovery-{context.run_id}.json" + # Detailed recovery snapshots may contain internal database fields; keep them + # in the runner-owned temporary directory instead of the public artifact tree. + detail_path = context.temporary_directory / "recovery-details" / f"recovery-{context.run_id}.json" environment = { **os.environ, "E2E_RUN_ID": context.run_id, diff --git a/scripts/e2e_hil_adapters.py b/scripts/e2e_hil_adapters.py index 535fe9b1..cb30ee7e 100644 --- a/scripts/e2e_hil_adapters.py +++ b/scripts/e2e_hil_adapters.py @@ -5,9 +5,12 @@ import hashlib import json +import os +import shutil import subprocess +import sys import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Protocol @@ -26,10 +29,22 @@ load_device_descriptor, validate_device_layout, ) -from e2e_runner import AssertionResult, FailureCategory, RunContext, RunnerDeadlineExceeded, RunnerFailure +from e2e_runner import ( + AssertionResult, + FailureCategory, + RunContext, + RunnerDeadlineExceeded, + RunnerFailure, +) from start_im_pairing import PairingLifecycle, PairingLifecycleError +from voice_tts_fixture import TtsFixtureError, validate_provider_environment ROOT = Path(__file__).resolve().parents[1] +SQLITE_COMPONENT_FILES = ( + ROOT / "third_party" / "sqlite3" / "sqlite3.c", + ROOT / "third_party" / "sqlite3" / "sqlite3.h", + ROOT / "third_party" / "sqlite3" / "CMakeLists.txt", +) @dataclass @@ -76,12 +91,31 @@ def _runner_failure(error: Exception) -> RunnerFailure: if isinstance(error, HilConfigurationError): return RunnerFailure(FailureCategory.CONFIGURATION, "hil_configuration_invalid") if isinstance(error, HilLeaseUnavailable): - return RunnerFailure(FailureCategory.DEVICE, "device_lease_unavailable") + return RunnerFailure(FailureCategory.LEASE, "device_lease_unavailable") if isinstance(error, HilProfileMismatch): return RunnerFailure(FailureCategory.DEVICE, "device_profile_mismatch") raise error +def ensure_sqlite_component() -> None: + """Prepare the checked-in SQLite component before an ESP-IDF build.""" + if all(path.is_file() for path in SQLITE_COMPONENT_FILES): + return + try: + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "prepare_sqlite.py")], + cwd=ROOT, + capture_output=True, + text=True, + timeout=180, + check=True, + ) + except subprocess.TimeoutExpired as error: + raise RunnerDeadlineExceeded from error + except (OSError, subprocess.CalledProcessError) as error: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "sqlite_component_prepare_failed") from error + + class HilPairingAdapter: """Execute the real HIL flow through injected hardware boundary adapters.""" @@ -190,6 +224,382 @@ def collect(self, context: RunContext, result: object, assertions: list[Assertio } +class HilVoiceAdapter: + """Run provider-neutral input TTS, wake, and multi-turn device voice HIL.""" + + VOICE_FIRMWARE_PROFILES = { + "sparkbot": "esp32s3-esp-sparkbot-serial-voice", + "pcb": "esp32s3-voicelife-pcb-serial-voice", + } + CONVERSATION_ACCEPTANCE_KEYS = frozenset( + { + "input_tts_preflight", + "audio_stats_present", + "audio_flow_observed", + "zero_loss", + "zero_serial_pcm_rejections", + "asr_text_matches_input", + "input_not_endpoint_truncated", + "interaction_queue_clean", + "no_interaction_rejection", + "no_provider_error", + "state_flow_complete", + "display_flow_complete", + "display_text_trace_complete", + "display_scroll_observed", + "terminal_guard_clean", + } + ) + WAKE_ACCEPTANCE_KEYS = frozenset( + { + "input_tts_preflight", + "wake_detected", + "wake_ack_requested", + "wake_ack_tts_started", + "wake_ack_tts_first_audio", + "wake_ack_tts_stopped", + "capture_started_after_ack", + } + ) + PREFLIGHT_ACCEPTANCE_KEYS = frozenset({"audio_present", "pcm_frames_present"}) + + def __init__( + self, + descriptor_path: Path, + lease_root: Path, + *, + hardware: HilHardware, + input_tts: str, + tts_model: str, + voice: str, + texts: list[str] | None, + expect_terminal: bool, + response_timeout: float, + ) -> None: + self._descriptor_path = descriptor_path + self._lease_root = lease_root + self._hardware = hardware + self._input_tts = input_tts + self._tts_model = tts_model + self._voice = voice + self._texts = texts or [] + self._expect_terminal = expect_terminal + self._response_timeout = response_timeout + self._descriptor: DeviceDescriptor | None = None + self._lease: DeviceLease | None = None + self._partitions: list[Partition] = [] + self._image: ApplicationImage | None = None + self._identity: TemporaryIdentity | None = None + self._pending_device_id = "" + self._readiness: list[dict[str, object]] = [] + self._device_fingerprint = "" + self._result: dict[str, object] = {} + self.lease_held = False + + def prepare(self, context: RunContext) -> None: + if shutil.which("ffmpeg") is None: + raise RunnerFailure(FailureCategory.CONFIGURATION, "ffmpeg_unavailable") + try: + validate_provider_environment(self._input_tts) + except TtsFixtureError as error: + raise RunnerFailure(FailureCategory.CONFIGURATION, error.code) from error + try: + descriptor = load_device_descriptor(self._descriptor_path, context.config.profile) + if descriptor.profile not in self.VOICE_FIRMWARE_PROFILES: + raise HilProfileMismatch("voice journey requires a supported profile") + lease = DeviceLease(descriptor, self._lease_root) + lease.acquire() + self.lease_held = True + self._descriptor = descriptor + self._lease = lease + context.cleanup.push("hil-voice-device-lease", self._release_lease, timeout_required=False) + self._partitions = self._hardware.inspect(self._descriptor, context.temporary_directory) + validate_device_layout(self._descriptor, self._partitions) + except (HilConfigurationError, HilLeaseUnavailable, HilProfileMismatch) as error: + raise _runner_failure(error) from error + + def _release_lease(self) -> None: + if self._lease is not None: + self._lease.release() + self.lease_held = False + + def _recover(self) -> None: + if self._descriptor is not None: + self._hardware.recover(self._descriptor) + + def _revoke(self) -> None: + if self._identity is not None: + self._hardware.revoke(self._identity) + elif self._pending_device_id: + self._hardware.revoke_device_id(self._pending_device_id) + + @staticmethod + def _classify_voice_exit(returncode: int, stderr: str) -> RunnerFailure: + if returncode == 1: + return RunnerFailure(FailureCategory.PRODUCT, "voice_acceptance_failed") + if "cannot open serial port" in stderr: + return RunnerFailure(FailureCategory.DEVICE, "voice_serial_unavailable") + if any(marker in stderr for marker in ("_missing", "_unavailable", "pyserial is required")): + return RunnerFailure(FailureCategory.CONFIGURATION, "voice_dependency_missing") + if "input_preparation_failed" in stderr: + return RunnerFailure(FailureCategory.EXTERNAL, "voice_tts_failed") + return RunnerFailure(FailureCategory.EXTERNAL, "voice_harness_failed") + + @staticmethod + def _fixture_valid(value: object) -> bool: + if not isinstance(value, dict) or set(value) != { + "provider", + "model", + "voice", + "requests", + "audio_bytes", + "first_packet_ms_max", + "total_ms_max", + }: + return False + return ( + isinstance(value.get("provider"), str) + and isinstance(value.get("model"), str) + and isinstance(value.get("voice"), str) + and all( + type(value.get(key)) is int and value.get(key, -1) >= 0 + for key in ( + "requests", + "audio_bytes", + "first_packet_ms_max", + "total_ms_max", + ) + ) + and value.get("requests", 0) > 0 + and value.get("audio_bytes", 0) > 0 + ) + + @classmethod + def _acceptance_valid(cls, value: object, expected: frozenset[str]) -> bool: + return isinstance(value, dict) and set(value) == expected and all(item is True for item in value.values()) + + def _run_harness( + self, context: RunContext, script_name: str, command: list[str], result_name: str + ) -> dict[str, object]: + result_path = context.temporary_directory / result_name + command.extend(("--result-json", str(result_path))) + try: + completed = subprocess.run( + command, + cwd=ROOT, + env=os.environ.copy(), + capture_output=True, + text=True, + timeout=max(0.1, context.remaining()), + check=False, + ) + except subprocess.TimeoutExpired as error: + raise RunnerDeadlineExceeded from error + except OSError as error: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, f"{script_name}_harness_unavailable") from error + if completed.returncode != 0: + raise self._classify_voice_exit(completed.returncode, completed.stderr) + try: + value = json.loads(result_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, TypeError) as error: + raise RunnerFailure(FailureCategory.PRODUCT, f"{script_name}_result_invalid") from error + if not isinstance(value, dict): + raise RunnerFailure(FailureCategory.PRODUCT, f"{script_name}_result_invalid") + return value + + def _run_wake_script(self, context: RunContext) -> dict[str, object]: + if self._descriptor is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "voice_device_not_prepared") + command = [ + sys.executable, + str(ROOT / "scripts" / "voice_linx_wake_injection_test.py"), + "--port", + str(self._descriptor.port), + "--input-tts", + self._input_tts, + "--tts-model", + self._tts_model, + "--voice", + self._voice, + "--timeout", + str(self._response_timeout), + "--serial-log", + str(context.temporary_directory / "voice-wake-serial.log"), + ] + return self._run_harness(context, "voice_wake", command, "voice-wake-result.json") + + def _run_tts_preflight(self, context: RunContext) -> dict[str, object]: + command = [ + sys.executable, + str(ROOT / "scripts" / "voice_tts_preflight.py"), + "--input-tts", + self._input_tts, + "--tts-model", + self._tts_model, + "--voice", + self._voice, + ] + return self._run_harness(context, "voice_tts_preflight", command, "voice-tts-preflight-result.json") + + def _run_voice_script(self, context: RunContext) -> dict[str, object]: + if self._descriptor is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "voice_device_not_prepared") + script = ROOT / "scripts" / "voice_linx_serial_multiturn_test.py" + serial_log = context.temporary_directory / "voice-serial.log" + command = [ + sys.executable, + str(script), + "--port", + str(self._descriptor.port), + "--input-tts", + self._input_tts, + "--tts-model", + self._tts_model, + "--voice", + self._voice, + "--display-profile", + self._descriptor.profile, + "--response-timeout", + str(self._response_timeout), + "--serial-log", + str(serial_log), + ] + if self._expect_terminal: + command.append("--expect-terminal") + for text in self._texts: + command.extend(("--text", text)) + return self._run_harness(context, "voice", command, "voice-result.json") + + def run(self, context: RunContext) -> dict[str, object]: + if self._descriptor is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "hil_device_not_prepared") + try: + preflight_result = self._run_tts_preflight(context) + voice_descriptor = replace( + self._descriptor, + firmware_profile=self.VOICE_FIRMWARE_PROFILES[self._descriptor.profile], + ) + build_directory = self._hardware.build(voice_descriptor) + self._image = self._hardware.image(build_directory, self._descriptor, self._partitions) + self._hardware.flash(voice_descriptor, self._image, context.remaining()) + self._pending_device_id = f"e2e-{context.run_id}" + context.cleanup.push("voice-gateway-device-revoke", self._revoke) + self._identity = self._hardware.register(context.run_id) + self._device_fingerprint = device_fingerprint(self._identity.device_id, context.run_id) + context.cleanup.push("voice-hil-device-recovery", self._recover) + self._hardware.provision(self._descriptor, self._identity, context.remaining()) + self._readiness = self._hardware.reboot_and_readiness(self._descriptor, context.remaining()) + ready, _ = hil_readiness_status(self._readiness) + if not ready: + raise RunnerFailure(FailureCategory.PRODUCT, "voice_readiness_incomplete") + wake_result: dict[str, object] | None = None + if self._descriptor.profile == "sparkbot": + wake_result = self._run_wake_script(context) + self._readiness = self._hardware.reboot_and_readiness(self._descriptor, context.remaining()) + ready, _ = hil_readiness_status(self._readiness) + if not ready: + raise RunnerFailure(FailureCategory.PRODUCT, "voice_post_wake_readiness_incomplete") + conversation_result = self._run_voice_script(context) + self._result = { + "schema_version": 1, + "preflight": preflight_result, + "wake": wake_result, + "conversation": conversation_result, + } + return self._result + except (HilConfigurationError, HilLeaseUnavailable, HilProfileMismatch) as error: + raise _runner_failure(error) from error + + def assert_result(self, context: RunContext, result: object) -> list[AssertionResult]: + values = result if isinstance(result, dict) and result.get("schema_version") == 1 else {} + conversation = values.get("conversation") if isinstance(values.get("conversation"), dict) else {} + preflight = values.get("preflight") if isinstance(values.get("preflight"), dict) else {} + acceptance = conversation.get("acceptance") + acceptance_values = acceptance if isinstance(acceptance, dict) else {} + wake = values.get("wake") + wake_clean = self._descriptor is not None and ( + self._descriptor.profile == "pcb" + or isinstance(wake, dict) + and wake.get("schema_version") == 1 + and self._fixture_valid(wake.get("fixture")) + and self._acceptance_valid(wake.get("acceptance"), self.WAKE_ACCEPTANCE_KEYS) + ) + checks = { + "voice_input_tts_preflight_clean": preflight.get("schema_version") == 1 + and self._fixture_valid(preflight.get("fixture")) + and self._acceptance_valid(preflight.get("acceptance"), self.PREFLIGHT_ACCEPTANCE_KEYS), + "voice_input_fixture_clean": self._fixture_valid(conversation.get("fixture")), + "voice_wake_sequence_clean": wake_clean, + "voice_turns_complete": conversation.get("completed_turns") == conversation.get("requested_turns") + and isinstance(conversation.get("requested_turns"), int) + and conversation.get("requested_turns", 0) > 0, + "voice_state_flow_clean": acceptance_values.get("state_flow_complete") is True, + "voice_display_flow_clean": acceptance_values.get("display_flow_complete") is True + and acceptance_values.get("display_text_trace_complete") is True, + "voice_wake_guard_clean": acceptance_values.get("terminal_guard_clean") is True, + "voice_acceptance_clean": self._acceptance_valid(acceptance, self.CONVERSATION_ACCEPTANCE_KEYS), + } + return [ + AssertionResult(name=name, passed=passed, code="ok" if passed else "mismatch") + for name, passed in checks.items() + ] + + def collect(self, context: RunContext, result: object, assertions: list[AssertionResult]) -> dict[str, object]: + if self._descriptor is None or self._image is None or self._identity is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "voice_evidence_incomplete") + values = result if isinstance(result, dict) else {} + conversation = values.get("conversation") if isinstance(values.get("conversation"), dict) else {} + preflight = values.get("preflight") if isinstance(values.get("preflight"), dict) else {} + preflight_fixture = preflight.get("fixture") if isinstance(preflight.get("fixture"), dict) else {} + fixture = conversation.get("fixture") if isinstance(conversation.get("fixture"), dict) else {} + audio = conversation.get("audio_stats") if isinstance(conversation.get("audio_stats"), dict) else {} + display = conversation.get("display") if isinstance(conversation.get("display"), dict) else {} + serial_rejections = conversation.get("serial_pcm_rejections") + wake = values.get("wake") if isinstance(values.get("wake"), dict) else {} + wake_fixture = wake.get("fixture") if isinstance(wake.get("fixture"), dict) else {} + fixture_sources = (preflight_fixture, wake_fixture, fixture) + + def fixture_metric(item: dict[str, object], key: str) -> int: + value = item.get(key, 0) + return value if type(value) is int and value >= 0 else 0 + + return { + "scope": "hil_voice", + "hardware_verified": all(assertion.passed for assertion in assertions), + "firmware_sha256": self._image.sha256, + "gateway_commit": self._identity.gateway_commit, + "device_fingerprint": self._device_fingerprint, + "readiness_markers": hil_readiness_markers(self._readiness), + "input_tts_provider": self._input_tts, + "input_tts_model": self._tts_model, + "input_tts_voice": self._voice, + "wake_stage": "passed" if self._descriptor.profile == "sparkbot" else "not_applicable", + "metrics": { + "input_tts_requests": sum(fixture_metric(item, "requests") for item in fixture_sources), + "input_tts_audio_bytes": sum(fixture_metric(item, "audio_bytes") for item in fixture_sources), + "input_tts_first_packet_ms_max": max( + (fixture_metric(item, "first_packet_ms_max") for item in fixture_sources), default=0 + ), + "input_tts_total_ms_max": max( + (fixture_metric(item, "total_ms_max") for item in fixture_sources), default=0 + ), + "wake_completed": 1 if wake else 0, + "requested_turns": conversation.get("requested_turns", 0), + "completed_turns": conversation.get("completed_turns", 0), + "asr_exact_matches": conversation.get("asr_exact_matches", 0), + "test_in_frames": audio.get("test_in_frames", 0), + "out_frames": audio.get("out_frames", 0), + "in_drop": audio.get("in_drop", 0), + "out_reject": audio.get("out_reject", 0), + "short_write": audio.get("short_write", 0), + "in_i2s_err": audio.get("in_i2s_err", 0), + "out_i2s_err": audio.get("out_i2s_err", 0), + "serial_pcm_rejections": len(serial_rejections) if isinstance(serial_rejections, list) else 0, + "display_content_snapshots": display.get("content_snapshots", 0), + }, + } + + class RealHilHardware: """Production adapters that reuse existing build, flash, provisioning and pairing scripts.""" @@ -247,6 +657,7 @@ def build(self, descriptor: DeviceDescriptor) -> Path: from firmware import build try: + ensure_sqlite_component() return build(descriptor.firmware_profile) except RunnerDeadlineExceeded: raise @@ -267,7 +678,12 @@ def flash(self, descriptor: DeviceDescriptor, image: ApplicationImage, timeout_s self._run(list(operation.argv), timeout_s) def _remote(self, script: str, timeout_s: float = 180.0) -> str: - return self._run(["ssh", "-o", "BatchMode=yes", self._server, "bash", "-s"], timeout_s, input_text=script) + try: + return self._run(["ssh", "-o", "BatchMode=yes", self._server, "bash", "-s"], timeout_s, input_text=script) + except RunnerFailure as error: + if error.message_code == "hil_command_failed": + raise RunnerFailure(FailureCategory.EXTERNAL, "external_service_unavailable") from error + raise def register(self, run_id: str) -> TemporaryIdentity: from provision_device import server_register_script, validate_credential diff --git a/scripts/e2e_hil_device.py b/scripts/e2e_hil_device.py index 3fdbc568..58135db7 100644 --- a/scripts/e2e_hil_device.py +++ b/scripts/e2e_hil_device.py @@ -20,9 +20,9 @@ fcntl = None try: - from sqlite_board_probe_protocol import Partition, partition_by_label + from sqlite_board_probe_protocol import Partition, ProbeError, parse_partition_table, partition_by_label except ModuleNotFoundError: - from scripts.sqlite_board_probe_protocol import Partition, partition_by_label + from scripts.sqlite_board_probe_protocol import Partition, ProbeError, parse_partition_table, partition_by_label SAFE_NAME = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") HEX_SHA256 = re.compile(r"^[0-9a-f]{64}$") @@ -66,6 +66,7 @@ class OfficialProfile: ("linx_secrets", 1, 2, 0x2E0000, 0x10000), ("assets", 1, 0x82, 0x300000, 0x100000), ("model", 1, 0x82, 0x400000, 0x300000), + ("voicelife", 1, 0x81, 0x700000, 0x900000), ), ), "pcb": OfficialProfile( @@ -267,6 +268,16 @@ def load_application_image( build_directory: Path, descriptor: DeviceDescriptor, partitions: list[Partition] ) -> ApplicationImage: application = validate_device_layout(descriptor, partitions) + built_partition_table = build_directory / "partition_table" / "partition-table.bin" + if built_partition_table.is_file(): + try: + built_partitions = parse_partition_table(built_partition_table.read_bytes()) + except (OSError, ProbeError, ValueError) as error: + raise HilConfigurationError("application build partition table is invalid") from error + if tuple(_partition_tuple(partition) for partition in built_partitions) != tuple( + _partition_tuple(partition) for partition in partitions + ): + raise HilProfileMismatch("application build partition layout does not match board") binary = build_directory / "voicelife.bin" flasher = build_directory / "flasher_args.json" try: diff --git a/scripts/e2e_runner.py b/scripts/e2e_runner.py index b6629793..08b9c2db 100644 --- a/scripts/e2e_runner.py +++ b/scripts/e2e_runner.py @@ -29,6 +29,7 @@ class FailureCategory(str, Enum): INFRASTRUCTURE = "infrastructure" PRODUCT = "product" DEVICE = "device" + LEASE = "lease" EXTERNAL = "external" TIMEOUT = "timeout" INTERRUPTED = "interrupted" @@ -43,6 +44,7 @@ class ExitCode(IntEnum): INFRASTRUCTURE = 10 PRODUCT = 20 DEVICE = 30 + LEASE = 31 EXTERNAL = 40 TIMEOUT = 60 INTERRUPTED = 70 @@ -59,6 +61,7 @@ class RunStatus(str, Enum): FailureCategory.INFRASTRUCTURE: ExitCode.INFRASTRUCTURE, FailureCategory.PRODUCT: ExitCode.PRODUCT, FailureCategory.DEVICE: ExitCode.DEVICE, + FailureCategory.LEASE: ExitCode.LEASE, FailureCategory.EXTERNAL: ExitCode.EXTERNAL, FailureCategory.TIMEOUT: ExitCode.TIMEOUT, FailureCategory.INTERRUPTED: ExitCode.INTERRUPTED, diff --git a/scripts/render_e2e_summary.py b/scripts/render_e2e_summary.py new file mode 100644 index 00000000..be83c5fb --- /dev/null +++ b/scripts/render_e2e_summary.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Render a safe GitHub Actions job summary from E2E evidence.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from e2e_evidence import EvidenceValidationError, validate_evidence + + +def render(root: Path) -> str: + rows: list[str] = [] + paths = sorted(root.rglob("evidence-*.json")) + for path in paths: + try: + document = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, dict): + raise ValueError + validate_evidence(document) + except (OSError, UnicodeError, json.JSONDecodeError, ValueError, EvidenceValidationError) as error: + raise ValueError("cannot render invalid E2E evidence") from error + status = str(document["status"]).upper() + category = document["failure_category"] or "none" + phase = document["failed_phase"] or "none" + rows.append( + f"| `{document['profile']}` | `{document['journey']}` | {status} | `{category}` | " + f"`{phase}` | `{document['message_code']}` |" + ) + if not rows: + raise ValueError("no E2E evidence found") + return ( + "## E2E result\n\n" + "| Profile | Journey | Status | Failure category | Failed phase | Message |\n" + "| --- | --- | --- | --- | --- | --- |\n" + "\n".join(rows) + "\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact_dir", type=Path) + parser.add_argument("--output", type=Path, default=None) + args = parser.parse_args(argv) + try: + summary = render(args.artifact_dir) + if args.output is None: + print(summary, end="") + else: + args.output.write_text(summary, encoding="utf-8") + except (OSError, ValueError) as error: + print(str(error), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_bailian_sparkbot_test.sh b/scripts/run_bailian_sparkbot_test.sh new file mode 100755 index 00000000..60cf14b2 --- /dev/null +++ b/scripts/run_bailian_sparkbot_test.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Run the repeatable Bailian -> SparkBot serial tests without exposing credentials. + +set -euo pipefail + +usage() { + cat <<'EOF' +用法: + run_bailian_sparkbot_test.sh preflight [额外参数] + run_bailian_sparkbot_test.sh wake [额外参数] + run_bailian_sparkbot_test.sh multiturn [额外参数] + +环境变量: + BAILIAN_KEY_FILE 含有配置行和 sk-... 的文件，默认读取项目外的本地 key.txt + SPARKBOT_SERIAL SparkBot USB 串口，默认 /dev/cu.usbmodem14401 + BAILIAN_TTS_MODEL 默认 qwen-audio-3.0-tts-flash + BAILIAN_TTS_VOICE 默认 longanlingxi + BAILIAN_TEST_LOG_DIR 默认 /tmp/voicelife-bailian-tests + BAILIAN_TEST_TEXT preflight 使用的固定短句 + +key.txt 可以包含公共 API 地址等其他配置，但脚本只接受第一条完整的 sk- 行。 +EOF +} + +if [[ $# -lt 1 ]]; then + usage >&2 + exit 2 +fi + +MODE="$1" +shift +case "$MODE" in + preflight|wake|multiturn) ;; + *) usage >&2; exit 2 ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KEY_FILE="${BAILIAN_KEY_FILE:-/Users/mac/Desktop/project/语音模型调用/key.txt}" +SERIAL_PORT="${SPARKBOT_SERIAL:-/dev/cu.usbmodem14401}" +TTS_MODEL="${BAILIAN_TTS_MODEL:-qwen-audio-3.0-tts-flash}" +TTS_VOICE="${BAILIAN_TTS_VOICE:-longanlingxi}" +LOG_DIR="${BAILIAN_TEST_LOG_DIR:-/tmp/voicelife-bailian-tests}" + +if [[ ! -r "$KEY_FILE" ]]; then + echo "配置文件不可读: $KEY_FILE" >&2 + exit 2 +fi + +# key.txt is a small local profile, not a dotenv file: the first two lines can +# be API URLs. Strip whitespace/CR first and never use the whole file as +# DASHSCOPE_API_KEY. +API_KEY="$(awk '{ line=$0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line ~ /^sk-[[:alnum:]_-]+$/) { print line; exit } }' "$KEY_FILE")" +if [[ -z "$API_KEY" ]]; then + echo "配置文件中没有找到完整的 sk- API Key: $KEY_FILE" >&2 + exit 2 +fi + +mkdir -p "$LOG_DIR" +STAMP="$(date +%Y%m%d-%H%M%S)" +export DASHSCOPE_API_KEY="$API_KEY" + +METADATA_FILE="$LOG_DIR/$MODE-$STAMP.meta.txt" +{ + printf 'mode=%s\n' "$MODE" + printf 'tts_model=%s\n' "$TTS_MODEL" + printf 'tts_voice=%s\n' "$TTS_VOICE" + printf 'serial=%s\n' "$SERIAL_PORT" + printf 'api_key_source=first_matching_sk_line\n' + printf 'key_file=%s\n' "$KEY_FILE" + if git -C "$REPO_DIR" rev-parse --short HEAD >/dev/null 2>&1; then + printf 'firmware_source_commit=%s\n' "$(git -C "$REPO_DIR" rev-parse --short HEAD)" + fi +} > "$METADATA_FILE" + +if ! python3 -c 'import dashscope' >/dev/null 2>&1; then + echo "缺少 dashscope Python 依赖，请先安装项目测试环境" >&2 + exit 2 +fi + +if [[ "$MODE" == preflight ]]; then + RESULT_FILE="$LOG_DIR/preflight-$STAMP.json" + exec python3 "$REPO_DIR/scripts/voice_bailian_load_test.py" \ + --mode tts \ + --requests 1 \ + --concurrency 1 \ + --turns-per-conversation 1 \ + --tts-model "$TTS_MODEL" \ + --voice "$TTS_VOICE" \ + --text "${BAILIAN_TEST_TEXT:-这是 SparkBot 百炼固定连通性测试。}" \ + --result-json "$RESULT_FILE" \ + "$@" +fi + +if [[ ! -e "$SERIAL_PORT" ]]; then + echo "串口不存在: $SERIAL_PORT（用 SPARKBOT_SERIAL 覆盖）" >&2 + exit 2 +fi +if ! python3 -c 'import serial' >/dev/null 2>&1; then + echo "缺少 pyserial Python 依赖，请先安装项目测试环境" >&2 + exit 2 +fi +if ! command -v ffmpeg >/dev/null 2>&1; then + echo "缺少 ffmpeg，无法把百炼音频转换为 16 kHz PCM" >&2 + exit 2 +fi + +if [[ "$MODE" == wake ]]; then + LOG_FILE="$LOG_DIR/wake-$STAMP.log" + exec python3 "$REPO_DIR/scripts/voice_linx_wake_injection_test.py" \ + --port "$SERIAL_PORT" \ + --tts-model "$TTS_MODEL" \ + --voice "$TTS_VOICE" \ + --serial-log "$LOG_FILE" \ + "$@" +fi + +LOG_FILE="$LOG_DIR/multiturn-$STAMP.log" +RESULT_FILE="$LOG_DIR/multiturn-$STAMP.json" +exec python3 "$REPO_DIR/scripts/voice_linx_serial_multiturn_test.py" \ + --port "$SERIAL_PORT" \ + --tts-model "$TTS_MODEL" \ + --voice "$TTS_VOICE" \ + --serial-log "$LOG_FILE" \ + --result-json "$RESULT_FILE" \ + "$@" diff --git a/scripts/run_e2e.py b/scripts/run_e2e.py index 1846392d..3612f7e0 100644 --- a/scripts/run_e2e.py +++ b/scripts/run_e2e.py @@ -16,8 +16,15 @@ HostImGatewayRecoveryE2EAdapter, HostLifecycleExampleAdapter, ) -from e2e_hil_adapters import HilPairingAdapter, RealHilHardware -from e2e_runner import ExitCode, FailureCategory, RunnerConfig, RunnerResult, exit_code_for, run_e2e +from e2e_hil_adapters import HilPairingAdapter, HilVoiceAdapter, RealHilHardware +from e2e_runner import ( + ExitCode, + FailureCategory, + RunnerConfig, + RunnerResult, + exit_code_for, + run_e2e, +) PROFILES = {"host": frozenset({"host"}), "hil": frozenset({"sparkbot", "pcb"})} @@ -43,6 +50,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--server-dir") parser.add_argument("--gateway-origin") parser.add_argument("--user-id") + parser.add_argument("--input-tts", choices=("dashscope", "aliyun-nls"), default="dashscope") + parser.add_argument("--tts-model", default="qwen-audio-3.0-tts-flash") + parser.add_argument("--voice", default="longanlingxi") + parser.add_argument("--text", action="append") + parser.add_argument("--expect-terminal", action="store_true") + parser.add_argument("--response-timeout", type=float, default=45.0) return parser.parse_args(argv) @@ -63,7 +76,7 @@ def build_adapter(layer: str, journey: str, args: argparse.Namespace | Path | No artifact_directory = args.artifact_dir if isinstance(args, argparse.Namespace) else args hil_options = ("device", "lease_dir", "server", "server_dir", "gateway_origin", "user_id") if ( - journey != "im-pairing" + journey not in {"im-pairing", "voice"} and isinstance(args, argparse.Namespace) and any(getattr(args, name) is not None for name in hil_options) ): @@ -79,6 +92,29 @@ def build_adapter(layer: str, journey: str, args: argparse.Namespace | Path | No lease_directory = args.lease_dir or Path.home() / ".voicelife" / "hil-leases" hardware = RealHilHardware(str(server), str(server_directory), str(gateway_origin), str(user_id)) return HilPairingAdapter(Path(device), lease_directory, hardware=hardware) + if journey == "voice": + if layer != "hil" or not isinstance(args, argparse.Namespace) or args.profile not in {"sparkbot", "pcb"}: + raise ValueError("voice journey requires a supported HIL profile") + device = _required_hil_option(args, "device") + server = _required_hil_option(args, "server") + server_directory = _required_hil_option(args, "server_dir") + gateway_origin = _required_hil_option(args, "gateway_origin") + user_id = _required_hil_option(args, "user_id") + if args.response_timeout <= 0: + raise ValueError("response-timeout must be positive") + lease_directory = args.lease_dir or Path.home() / ".voicelife" / "hil-leases" + hardware = RealHilHardware(str(server), str(server_directory), str(gateway_origin), str(user_id)) + return HilVoiceAdapter( + Path(device), + lease_directory, + hardware=hardware, + input_tts=args.input_tts, + tts_model=args.tts_model, + voice=args.voice, + texts=args.text, + expect_terminal=args.expect_terminal, + response_timeout=args.response_timeout, + ) if journey == "im-gateway-strong-reminder" and layer == "host": return HostImGatewayE2EAdapter() if journey == "im-gateway-recovery" and layer == "host": @@ -135,14 +171,24 @@ def build_evidence(result: RunnerResult, config: RunnerConfig) -> dict[str, obje else False ) hil = None - if scope == "hil_im_pairing" and result.status.value == "passed": + if scope in {"hil_im_pairing", "hil_voice"} and result.status.value == "passed": hil = { "firmware_sha256": collected.get("firmware_sha256"), "gateway_commit": collected.get("gateway_commit"), "device_fingerprint": collected.get("device_fingerprint"), "readiness_markers": collected.get("readiness_markers"), - "pairing_markers": collected.get("pairing_markers"), } + if scope == "hil_im_pairing": + hil["pairing_markers"] = collected.get("pairing_markers") + else: + hil.update( + { + "input_tts_provider": collected.get("input_tts_provider"), + "input_tts_model": collected.get("input_tts_model"), + "input_tts_voice": collected.get("input_tts_voice"), + "wake_stage": collected.get("wake_stage"), + } + ) return { "schema_version": 1, "run_id": result.run_id, diff --git a/scripts/voice_linx_serial_multiturn_test.py b/scripts/voice_linx_serial_multiturn_test.py index de2f89a8..b0d113fb 100644 --- a/scripts/voice_linx_serial_multiturn_test.py +++ b/scripts/voice_linx_serial_multiturn_test.py @@ -5,24 +5,20 @@ import argparse import json -import os import re import subprocess import sys -import tempfile import threading import time from dataclasses import dataclass from pathlib import Path +from voice_tts_fixture import TtsFixtureError, synthesize_fixture, validate_provider_environment + try: - import dashscope import serial - from dashscope.audio.tts_v2 import SpeechSynthesizer except ImportError: - dashscope = None serial = None - SpeechSynthesizer = None MAGIC, VERSION, BEGIN, PCM, END = b"VLVT", 1, 1, 2, 3 @@ -53,6 +49,8 @@ class TurnResult: class PreparedTurn: input_text: str tts_ms: int + first_packet_ms: int + audio_bytes: int frames: list[bytes] @@ -87,6 +85,20 @@ def wait_for(self, marker: str, after: int, timeout: float) -> tuple[int, str]: raise TimeoutError(marker) self._condition.wait(timeout=remaining) + def wait_for_any(self, markers: tuple[str, ...], after: int, timeout: float) -> tuple[int, str]: + """Wait for the first event in a protocol alternative set.""" + deadline = time.monotonic() + timeout + with self._condition: + while True: + for index in range(after, len(self._items)): + line = self._items[index][1] + if any(marker in line for marker in markers): + return index + 1, line + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(" or ".join(markers)) + self._condition.wait(timeout=remaining) + def lines_since(self, after: int) -> list[str]: with self._condition: return [line for _, line in self._items[after:]] @@ -121,68 +133,68 @@ def packet(kind: int, payload: bytes = b"") -> bytes: return MAGIC + bytes((VERSION, kind)) + len(payload).to_bytes(2, "little") + payload -def synthesize(text: str, model: str, voice: str) -> bytes: - synthesizer = SpeechSynthesizer(model=model, voice=voice) - audio = synthesizer.call(text) - if not isinstance(audio, (bytes, bytearray)) or not audio: - raise RuntimeError("empty_tts_audio") - return bytes(audio) - +def open_serial(port: str, baud: int) -> serial.Serial: + """Open USB UART without toggling SparkBot reset lines by default.""" + device = serial.Serial() + device.port = port + device.baudrate = baud + device.timeout = 0.2 + device.write_timeout = 5 + # SparkBot maps RTS to EN. Keep both modem-control lines inactive while + # pyserial opens the USB-JTAG endpoint; enabling flow control here can + # suppress the later reset pulse or reset the board during open(). + device.dsrdtr = False + device.rtscts = False + device.dtr = False + device.rts = False + device.open() + return device -def synthesize_macos_say(text: str, voice: str) -> bytes: - """Generate local AIFF only for board-harness fallback input. - This path is intentionally opt-in. It lets the serial state-machine test - remain reproducible when the external TTS provider is unavailable; it is - not reported as a DashScope model result. - """ - path: Path | None = None - try: - with tempfile.NamedTemporaryFile(prefix="voicelife-serial-input-", suffix=".aiff", delete=False) as output: - path = Path(output.name) - subprocess.run(["say", "-v", voice, "-o", str(path), text], check=True, capture_output=True) - audio = path.read_bytes() - if not audio: - raise RuntimeError("empty_local_tts_audio") - return audio - except FileNotFoundError as error: - raise RuntimeError("macos_say_unavailable") from error - except subprocess.CalledProcessError as error: - raise RuntimeError("macos_say_failed") from error - finally: - if path is not None: - path.unlink(missing_ok=True) - - -def to_pcm_frames(audio: bytes) -> list[bytes]: - result = subprocess.run( - [ - "ffmpeg", - "-hide_banner", - "-loglevel", - "error", - "-i", - "pipe:0", - "-f", - "s16le", - "-acodec", - "pcm_s16le", - "-ac", - "1", - "-ar", - "16000", - "pipe:1", - ], - input=audio, - capture_output=True, - check=False, - ) - if result.returncode != 0 or not result.stdout: - raise RuntimeError("ffmpeg_pcm_decode_failed") +def reset_usb_serial_jtag(device: serial.Serial) -> None: + """Reset SparkBot's application through the USB-Serial/JTAG EN line.""" + # SparkBot exposes EN on RTS and has no boot-button automation. Keep DTR + # deasserted so the pulse cannot select the ROM downloader. + device.rts = False + device.dtr = False + device.rts = True + time.sleep(0.15) + device.rts = False + time.sleep(0.2) + + +def to_pcm_frames(audio: bytes, audio_format: str = "encoded") -> list[bytes]: + if audio_format == "pcm_s16le_16000_mono": + pcm = audio + else: + result = subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-i", + "pipe:0", + "-f", + "s16le", + "-acodec", + "pcm_s16le", + "-ac", + "1", + "-ar", + "16000", + "pipe:1", + ], + input=audio, + capture_output=True, + check=False, + ) + if result.returncode != 0 or not result.stdout: + raise RuntimeError("ffmpeg_pcm_decode_failed") + pcm = result.stdout # The explicit serial end packet is the endpoint for this harness. Adding # trailing silence makes local VAD stop early, then incorrectly turns the # remaining test frames into late-frame rejection noise. - pcm = result.stdout remainder = len(pcm) % PCM_FRAME_BYTES if remainder: pcm += bytes(PCM_FRAME_BYTES - remainder) @@ -207,6 +219,41 @@ def parse_audio_stats(line: str) -> dict[str, int]: return {key: int(value) for key, value in re.findall(r"(\w+)=([0-9]+)", line)} +def display_evidence( + display_profile: str, raw_lines: list[str], results: list[TurnResult], requested_turns: int +) -> tuple[bool, int, bool]: + """Return (complete, content_snapshots, scroll_observed) without retaining display text.""" + if display_profile == "sparkbot": + rendered_text_lines = [line for line in raw_lines if "SPARKBOT_TEXT_RENDER" in line] + content_render_lines = [line for line in rendered_text_lines if "content_visible=1" in line] + complete = ( + bool(rendered_text_lines) + and all( + "generation=" in line + and "revision=" in line + and "content_height=" in line + and "viewport_height=" in line + and "overflow_width=" in line + and "manual_line_breaks=" in line + and " status=" in line + and " content=" in line + for line in rendered_text_lines + ) + and all("viewport_height=120" in line and "content_height=0" not in line for line in content_render_lines) + ) + return ( + complete, + len(content_render_lines), + any(re.search(r"overflow_width=[1-9][0-9]*", line) for line in content_render_lines), + ) + + display_draws_by_turn = [ + sum("DISPLAY_DRAW=1" in line for line in raw_lines[result.log_start : result.log_end]) for result in results + ] + complete = len(results) == requested_turns and all(count > 0 for count in display_draws_by_turn) + return complete, sum(display_draws_by_turn), False + + def prepare_turns(texts: list[str], input_tts: str, model: str, voice: str, say_voice: str) -> list[PreparedTurn]: """Generate every host utterance before opening the first device capture. @@ -216,13 +263,20 @@ def prepare_turns(texts: list[str], input_tts: str, model: str, voice: str, say_ """ prepared: list[PreparedTurn] = [] for index, text in enumerate(texts, start=1): - started = time.monotonic() - audio = synthesize(text, model, voice) if input_tts == "dashscope" else synthesize_macos_say(text, say_voice) - frames = to_pcm_frames(audio) - tts_ms = round((time.monotonic() - started) * 1000) + fixture_voice = say_voice if input_tts == "macos-say" else voice + fixture = synthesize_fixture(text, input_tts, model, fixture_voice) + frames = to_pcm_frames(fixture.audio, fixture.audio_format) if not frames: raise RuntimeError(f"turn_{index}_has_no_pcm_frames") - prepared.append(PreparedTurn(input_text=text, tts_ms=tts_ms, frames=frames)) + prepared.append( + PreparedTurn( + input_text=text, + tts_ms=fixture.total_ms, + first_packet_ms=fixture.first_packet_ms, + audio_bytes=len(fixture.audio), + frames=frames, + ) + ) return prepared @@ -268,15 +322,57 @@ def run_turn( time.sleep(0.02) device.write(packet(END)) device.flush() - cursor, end_result = log.wait_for("SERIAL_VOICE_TURN_END", turn_cursor, 5) - if "=ok" not in end_result and not result.input_endpoint_truncated: - raise RuntimeError(f"turn_end_failed:{end_result}") + # The serial task may wait for pooled PCM payloads while a long Linx + # utterance drains. Scale the endpoint window with the injected frame + # count, but cap it so a genuinely stuck turn still fails promptly. + turn_end_timeout = min(60.0, max(15.0, 10.0 + len(prepared.frames) * 0.1)) + # Auto mode may let Linx's server VAD finish the turn before the USB + # fixture's explicit END packet reaches the state machine. In that + # ordering STT is the authoritative endpoint and is already followed + # by the same TTS state flow; waiting for a later local stop would + # skip the valid ASR line and manufacture a timeout. + cursor, endpoint_marker = log.wait_for_any( + ( + "SERIAL_VOICE_TURN_END", + "SERIAL_VOICE_EVIDENCE event=capture_stopped ", + "SERIAL_VOICE_EVIDENCE event=stt_text_received ", + ), + turn_cursor, + turn_end_timeout, + ) + asr_line: str | None = None + if "event=stt_text_received" in endpoint_marker: + asr_line = endpoint_marker + if "SERIAL_VOICE_TURN_END" in endpoint_marker: + if "=ok" not in endpoint_marker and not result.input_endpoint_truncated: + raise RuntimeError(f"turn_end_failed:{endpoint_marker}") + cursor, endpoint_followup = log.wait_for_any( + ( + "SERIAL_VOICE_EVIDENCE event=capture_stopped ", + "SERIAL_VOICE_EVIDENCE event=stt_text_received ", + ), + cursor, + response_timeout, + ) + if "event=stt_text_received" in endpoint_followup: + asr_line = endpoint_followup + capture_stopped_seen = False + else: + capture_stopped_seen = True + elif asr_line is not None: + capture_stopped_seen = False + else: + # Local VAD is a valid endpoint even when the USB fixture's END + # packet is consumed after the capture callback has already run. + capture_stopped_seen = True # Local VAD can stop capture before the explicit host end packet. The # packet still terminates injection, but the real state transition is # valid from any point after this turn began. - cursor, _ = wait_evidence(log, "capture_stopped", turn_cursor, 12) - cursor, asr = wait_evidence(log, "stt_text_received", cursor, response_timeout) - result.asr_text = evidence_text(asr) + if asr_line is None: + if not capture_stopped_seen: + cursor, _ = wait_evidence(log, "capture_stopped", cursor, 12) + cursor, asr_line = wait_evidence(log, "stt_text_received", cursor, response_timeout) + result.asr_text = evidence_text(asr_line) result.asr_matches_input = normalize_transcript(result.asr_text) == normalize_transcript(result.input_text) cursor, _ = wait_evidence(log, "tts_started", cursor, response_timeout) # Linx may announce the first display sentence before its first PCM @@ -313,6 +409,11 @@ def run_turn( # Wait for that existing capture instead of sending a second PressDown. log.wait_for("SERIAL_VOICE_CAPTURE_READY", cursor, 12) result.completed = True + # The capture-ready evidence and its phase-4 display snapshot are + # emitted by separate runtime tasks. Give the logger one scheduling + # slice to collect the snapshot before taking the turn boundary; + # otherwise the next turn's range can accidentally omit phase=4. + time.sleep(0.1) except (RuntimeError, TimeoutError, serial.SerialException) as error: result.error = str(error) result.log_end = log.mark() @@ -333,9 +434,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--voice", default="longanhuan_v3") parser.add_argument( "--input-tts", - choices=("dashscope", "macos-say"), + choices=("dashscope", "aliyun-nls", "macos-say"), default="dashscope", - help="Source for injected audio. macos-say is a local fallback when external TTS is unavailable.", + help="Host TTS fixture provider. macos-say is local-only and never proves cloud availability.", ) parser.add_argument("--say-voice", default="Ting-Ting", help="macOS voice used with --input-tts macos-say.") parser.add_argument("--response-timeout", type=float, default=45) @@ -355,6 +456,12 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Require at least one subtitle wider than the safe one-line viewport to enter horizontal scrolling.", ) + parser.add_argument( + "--display-profile", + choices=("sparkbot", "pcb"), + default="sparkbot", + help="Board-specific display evidence contract used by the test firmware.", + ) parser.add_argument( "--allow-asr-mismatch", action="store_true", @@ -372,34 +479,24 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() - if args.input_tts == "dashscope" and not os.environ.get("DASHSCOPE_API_KEY"): - print("DASHSCOPE_API_KEY is required", file=sys.stderr) - return 2 if serial is None: print("pyserial is required", file=sys.stderr) return 2 - if args.input_tts == "dashscope" and (dashscope is None or SpeechSynthesizer is None): - print("dashscope is required for --input-tts dashscope", file=sys.stderr) + try: + validate_provider_environment(args.input_tts) + except TtsFixtureError as error: + print(f"input_preparation_failed:{error.code}", file=sys.stderr) return 2 - if args.input_tts == "dashscope": - dashscope.api_key = os.environ["DASHSCOPE_API_KEY"] texts = args.text or DEFAULT_TURNS try: # Prepare every utterance before the serial endpoint is opened. This is # part of the harness contract, not a production latency measurement. prepared_turns = prepare_turns(texts, args.input_tts, args.tts_model, args.voice, args.say_voice) - except (RuntimeError, subprocess.SubprocessError, TimeoutError) as error: + except (TtsFixtureError, RuntimeError, subprocess.SubprocessError, TimeoutError) as error: print(f"input_preparation_failed:{error}", file=sys.stderr) return 2 - device = serial.Serial() - device.port = args.port - device.baudrate = args.baud - device.timeout = 0.2 - device.write_timeout = 5 - device.dtr = False - device.rts = False try: - device.open() + device = open_serial(args.port, args.baud) except serial.SerialException as error: print(f"cannot open serial port: {type(error).__name__}", file=sys.stderr) return 2 @@ -408,11 +505,7 @@ def main() -> int: results: list[TurnResult] = [] try: if args.reset_before_run: - # USB-Serial/JTAG maps RTS to EN. Keeping DTR deasserted avoids - # entering the bootloader; this is an explicit test-only reset. - device.rts = True - time.sleep(0.12) - device.rts = False + reset_usb_serial_jtag(device) log.wait_for("SERIAL_VOICE_TEST_READY=1", 0, 20) # READY means the serial endpoint and I2S port exist, not that the # asynchronous local wake-model bootstrap has returned the controller @@ -455,36 +548,32 @@ def main() -> int: interaction_stats = [parse_audio_stats(line) for line in raw_lines if "INTERACTION_QUEUE_STATS" in line] interaction_keys = ("control_dropped", "best_effort_dropped", "board_dropped") serial_pcm_rejections = [line for line in raw_lines if "SERIAL_VOICE_PCM=reject" in line] - required_active_phases = (3, 4, 5, 6) + # Auto mode may receive Linx's server-VAD STT event before the local + # capture-stopped callback. In that valid path the runtime transitions + # directly from listening (4) to thinking (6), so phase 5 is optional + # only when the segment contains an authoritative STT event. + required_active_phases = (3, 4, 6, 7) rendered_text_lines = [line for line in raw_lines if "SPARKBOT_TEXT_RENDER" in line] - content_render_lines = [line for line in rendered_text_lines if "content_visible=1" in line] - display_text_trace_complete = ( - bool(rendered_text_lines) - and all( - "generation=" in line - and "revision=" in line - and "content_height=" in line - and "viewport_height=" in line - and "overflow_width=" in line - and "manual_line_breaks=" in line - and " status=" in line - and " content=" in line - for line in rendered_text_lines - ) - and all("viewport_height=120" in line and "content_height=0" not in line for line in content_render_lines) + display_text_trace_complete, display_content_snapshots, display_scroll_observed = display_evidence( + args.display_profile, raw_lines, results, len(texts) ) - display_scroll_observed = any(re.search(r"overflow_width=[1-9][0-9]*", line) for line in content_render_lines) def turn_phases_complete(marker: str) -> bool: - return len(results) == len(texts) and all( - all( - any(f"{marker}={phase} " in line for line in raw_lines[result.log_start : result.log_end]) - for phase in required_active_phases - ) - for result in results - ) + if len(results) != len(texts): + return False + for result in results: + segment = raw_lines[result.log_start : result.log_end] + if not all(any(f"{marker}={phase} " in line for line in segment) for phase in required_active_phases): + return False + if not any(f"{marker}=5 " in line for line in segment): + has_server_stt = any("SERIAL_VOICE_EVIDENCE event=stt_text_received " in line for line in segment) + if not has_server_stt: + return False + return True acceptance = { + "input_tts_preflight": bool(prepared_turns) + and all(turn.audio_bytes > 0 and len(turn.frames) > 0 for turn in prepared_turns), "audio_stats_present": all(key in audio_stats for key in required_present), "audio_flow_observed": audio_stats.get("test_in_frames", 0) > 0 and audio_stats.get("out_frames", 0) > 0, "zero_loss": all(audio_stats.get(key, -1) == 0 for key in required_zero), @@ -506,6 +595,16 @@ def turn_phases_complete(marker: str) -> bool: or (bool(results) and results[-1].terminal_guard_armed and results[-1].terminal_guard_clean), } report = { + "schema_version": 2, + "fixture": { + "provider": args.input_tts, + "model": "macos-say" if args.input_tts == "macos-say" else args.tts_model, + "voice": args.say_voice if args.input_tts == "macos-say" else args.voice, + "requests": len(prepared_turns), + "audio_bytes": sum(turn.audio_bytes for turn in prepared_turns), + "first_packet_ms_max": max((turn.first_packet_ms for turn in prepared_turns), default=0), + "total_ms_max": max((turn.tts_ms for turn in prepared_turns), default=0), + }, "requested_turns": len(texts), "completed_turns": sum(result.completed for result in results), "asr_exact_matches": sum(result.asr_matches_input for result in results), @@ -515,7 +614,7 @@ def turn_phases_complete(marker: str) -> bool: "serial_pcm_rejections": serial_pcm_rejections, "display": { "rendered_text_snapshots": len(rendered_text_lines), - "content_snapshots": len(content_render_lines), + "content_snapshots": display_content_snapshots, "scroll_observed": display_scroll_observed, }, "acceptance": acceptance, diff --git a/scripts/voice_linx_wake_injection_test.py b/scripts/voice_linx_wake_injection_test.py new file mode 100644 index 00000000..00afe123 --- /dev/null +++ b/scripts/voice_linx_wake_injection_test.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Inject a real DashScope TTS wake phrase into SparkBot's local wake detector.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +from voice_linx_serial_multiturn_test import ( + PreparedTurn, + SerialLog, + open_serial, + packet, + reset_usb_serial_jtag, + run_turn, + to_pcm_frames, +) +from voice_tts_fixture import TtsFixtureError, synthesize_fixture, validate_provider_environment + +try: + import serial +except ImportError: + serial = None + +TURN_BEGIN, PCM, TURN_END = 1, 2, 3 +WAKE_BEGIN, WAKE_END = 4, 5 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", default="/dev/cu.usbmodem14401") + parser.add_argument("--baud", type=int, default=115200) + parser.add_argument("--text", default="你好牛牛") + parser.add_argument( + "--followup-text", + action="append", + help="唤醒后继续注入一条或多条真实语音，并等待 Linx STT/TTS 完成。", + ) + parser.add_argument("--tts-model", default="cosyvoice-v3-flash") + parser.add_argument("--voice", default="longanhuan_v3") + parser.add_argument("--input-tts", choices=("dashscope", "aliyun-nls"), default="dashscope") + parser.add_argument("--timeout", type=float, default=30) + parser.add_argument( + "--reset-before-run", + action="store_true", + help="Hard-reset the board after opening the serial port, then wait for its ready sequence.", + ) + parser.add_argument("--serial-log", type=Path) + parser.add_argument("--result-json", type=Path) + args = parser.parse_args() + if args.baud <= 0 or args.timeout <= 0: + parser.error("baud 和 timeout 必须为正数") + return args + + +def wait_for(log: SerialLog, marker: str, cursor: int, timeout: float) -> int: + next_cursor, _ = log.wait_for(marker, cursor, timeout) + return next_cursor + + +def request_wake_begin(log: SerialLog, device: serial.Serial, timeout: float) -> int: + """Open the local wake-input gate without requiring a one-shot boot log.""" + deadline = time.monotonic() + timeout + cursor = log.mark() + # A USB-Serial/JTAG attach can reset SparkBot. In that case writes made + # before the serial task starts are buffered and later replayed together. + # Observe its one-shot startup marker opportunistically before sending; + # after a bounded wait, an already-running board simply proceeds to the + # request/response handshake below. + try: + cursor, _ = log.wait_for("SERIAL_VOICE_TEST_READY=1", cursor, min(8.0, timeout)) + except TimeoutError: + cursor = log.mark() + while time.monotonic() < deadline: + device.write(packet(WAKE_BEGIN)) + device.flush() + try: + cursor, line = log.wait_for("SERIAL_VOICE_WAKE_BEGIN=", cursor, min(1.0, deadline - time.monotonic())) + except TimeoutError: + # The serial task may still be starting after an explicit USB reset. + # Retry the idempotent request until the task acknowledges it. + cursor = log.mark() + continue + if "=ok" in line: + return cursor + # Code 4 means the detector is still leaving an active interaction. + # Wait for its explicit standby transition before issuing one retry; + # repeatedly writing while booting would only queue duplicate requests + # on the USB endpoint. + if "code=4" in line: + try: + cursor, _ = log.wait_for( + "SERIAL_VOICE_EVIDENCE event=standby_ready ", + cursor, + min(5.0, max(0.0, deadline - time.monotonic())), + ) + except TimeoutError: + cursor = log.mark() + else: + time.sleep(0.25) + raise TimeoutError("SERIAL_VOICE_WAKE_BEGIN=ok") + + +def main() -> int: + args = parse_args() + if serial is None: + print("pyserial is required", file=sys.stderr) + return 2 + try: + validate_provider_environment(args.input_tts) + wake_fixture = synthesize_fixture(args.text, args.input_tts, args.tts_model, args.voice) + frames = to_pcm_frames(wake_fixture.audio, wake_fixture.audio_format) + followups = [] + for text in args.followup_text or []: + fixture = synthesize_fixture(text, args.input_tts, args.tts_model, args.voice) + followups.append( + PreparedTurn( + input_text=text, + tts_ms=fixture.total_ms, + first_packet_ms=fixture.first_packet_ms, + audio_bytes=len(fixture.audio), + frames=to_pcm_frames(fixture.audio, fixture.audio_format), + ) + ) + except (TtsFixtureError, RuntimeError, OSError) as error: + print(f"input_preparation_failed:{error}", file=sys.stderr) + return 2 + if not frames: + print("input_preparation_failed:empty_pcm", file=sys.stderr) + return 2 + + try: + device = open_serial(args.port, args.baud) + except serial.SerialException as error: + print(f"cannot open serial port: {type(error).__name__}", file=sys.stderr) + return 2 + + log = SerialLog(device) + log.start() + cursor = 0 + wake_detected = False + ack_requested = False + ack_tts_started = False + ack_tts_first_audio = False + ack_tts_stopped = False + capture_started = False + completed_followups = 0 + exit_code = 1 + try: + if args.reset_before_run: + reset_usb_serial_jtag(device) + # `SERIAL_VOICE_TEST_READY=1` is emitted once per firmware boot and can + # precede a later serial attach. The request/response handshake is the + # authoritative readiness check for both fresh and already-running + # boards; it also verifies that local wake detection is in standby. + cursor = request_wake_begin(log, device, args.timeout) + + for frame in frames: + device.write(packet(PCM, frame)) + device.flush() + time.sleep(0.02) + device.write(packet(WAKE_END)) + device.flush() + cursor = wait_for(log, "SERIAL_VOICE_WAKE_END=ok", cursor, 5) + cursor = wait_for(log, "WAKE_DETECTED word=你好牛牛", cursor, args.timeout) + wake_detected = True + cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=wake_detected ", cursor, 5) + # Both Linx-compatible wake paths are valid: a silent detect opens the + # capture immediately, while a deliberate confirmation speech first + # emits ack -> tts.stop and only then opens the capture. The harness + # must wait for either path instead of timing out before sending the + # follow-up utterance. + cursor, wake_protocol = log.wait_for_any( + ( + "SERIAL_VOICE_EVIDENCE event=local_wake_detect_requested ", + "SERIAL_VOICE_EVIDENCE event=local_wake_ack_requested ", + ), + cursor, + args.timeout, + ) + if "local_wake_ack_requested" in wake_protocol: + ack_requested = True + cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=tts_started ", cursor, args.timeout) + ack_tts_started = True + cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=tts_first_audio ", cursor, args.timeout) + ack_tts_first_audio = True + cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=tts_stopped ", cursor, args.timeout) + ack_tts_stopped = True + cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=capture_started ", cursor, args.timeout) + capture_started = True + if not followups: + print(f"wake_injection_success text={args.text} frames={len(frames)}") + exit_code = 0 + return exit_code + for index, prepared in enumerate(followups, start=1): + result = run_turn( + device, + log, + index, + prepared, + response_timeout=args.timeout, + first_turn=False, + expect_terminal=False, + guard_observation_seconds=8.5, + ) + if result.error: + raise TimeoutError(f"followup_{index}:{result.error}") + completed_followups += 1 + print( + f"wake_followup_success index={index} input={prepared.input_text} " + f"asr={result.asr_text} reply={result.reply_text}" + ) + exit_code = 0 + return exit_code + except TimeoutError as error: + print(f"wake_injection_timeout:{error}", file=sys.stderr) + return 1 + except serial.SerialException as error: + print(f"serial_error:{type(error).__name__}", file=sys.stderr) + return 1 + finally: + time.sleep(1) + log.stop() + device.close() + if args.serial_log: + args.serial_log.write_text("\n".join(log.all_lines()) + "\n", encoding="utf-8") + if args.result_json: + report = { + "schema_version": 1, + "fixture": { + "provider": args.input_tts, + "model": args.tts_model, + "voice": args.voice, + "requests": 1 + len(followups), + "audio_bytes": len(wake_fixture.audio) + sum(turn.audio_bytes for turn in followups), + "first_packet_ms_max": max( + [wake_fixture.first_packet_ms, *(turn.first_packet_ms for turn in followups)] + ), + "total_ms_max": max([wake_fixture.total_ms, *(turn.tts_ms for turn in followups)]), + }, + "requested_followups": len(followups), + "completed_followups": completed_followups, + "acceptance": { + "input_tts_preflight": bool(frames), + "wake_detected": wake_detected, + "wake_ack_requested": ack_requested, + "wake_ack_tts_started": ack_tts_started, + "wake_ack_tts_first_audio": ack_tts_first_audio, + "wake_ack_tts_stopped": ack_tts_stopped, + "capture_started_after_ack": capture_started and ack_tts_stopped, + }, + "exit_code": exit_code, + } + args.result_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/voice_tts_fixture.py b/scripts/voice_tts_fixture.py new file mode 100644 index 00000000..e72d7fab --- /dev/null +++ b/scripts/voice_tts_fixture.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Provider-neutral host TTS fixtures for voice HIL input generation.""" + +from __future__ import annotations + +import importlib +import os +import shutil +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +class TtsFixtureError(RuntimeError): + """Stable, non-sensitive TTS fixture failure.""" + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +@dataclass(frozen=True) +class TtsFixtureResult: + audio: bytes + audio_format: str + provider: str + model: str + voice: str + total_ms: int + first_packet_ms: int + + +SUPPORTED_PROVIDERS = frozenset({"dashscope", "aliyun-nls", "macos-say"}) + + +def validate_provider_environment(provider: str) -> None: + if provider not in SUPPORTED_PROVIDERS: + raise TtsFixtureError("input_tts_provider_unsupported") + if provider == "dashscope": + if not os.environ.get("DASHSCOPE_API_KEY"): + raise TtsFixtureError("dashscope_api_key_missing") + try: + importlib.import_module("dashscope.audio.tts_v2") + except ImportError as error: + raise TtsFixtureError("dashscope_unavailable") from error + elif provider == "aliyun-nls": + if not os.environ.get("ALIYUN_NLS_APPKEY") or not os.environ.get("ALIYUN_NLS_TOKEN"): + raise TtsFixtureError("aliyun_nls_credentials_missing") + try: + importlib.import_module("nls") + except ImportError as error: + raise TtsFixtureError("aliyun_nls_unavailable") from error + elif shutil.which("say") is None: + raise TtsFixtureError("macos_say_unavailable") + + +def _dashscope_synthesize(text: str, model: str, voice: str) -> TtsFixtureResult: + dashscope = importlib.import_module("dashscope") + tts_v2 = importlib.import_module("dashscope.audio.tts_v2") + dashscope.api_key = os.environ["DASHSCOPE_API_KEY"] + endpoint = os.environ.get("DASHSCOPE_BASE_WEBSOCKET_API_URL") + if endpoint: + dashscope.base_websocket_api_url = endpoint + started = time.monotonic() + synthesizer = tts_v2.SpeechSynthesizer(model=model, voice=voice) + audio = synthesizer.call(text) + total_ms = round((time.monotonic() - started) * 1000) + if not isinstance(audio, (bytes, bytearray)) or not audio: + raise TtsFixtureError("empty_tts_audio") + first_packet = getattr(synthesizer, "get_first_package_delay", None) + first_packet_ms = first_packet() if callable(first_packet) else 0 + if not isinstance(first_packet_ms, (int, float)) or first_packet_ms < 0: + first_packet_ms = 0 + return TtsFixtureResult( + audio=bytes(audio), + audio_format="encoded", + provider="dashscope", + model=model, + voice=voice, + total_ms=total_ms, + first_packet_ms=round(first_packet_ms), + ) + + +def _aliyun_nls_synthesize(text: str, model: str, voice: str) -> TtsFixtureResult: + """Use the official NLS callback SDK and request raw 16 kHz PCM.""" + nls = importlib.import_module("nls") + chunks: list[bytes] = [] + failure: list[str] = [] + first_packet_ms = 0 + started = time.monotonic() + + def on_data(data: bytes, *_: Any) -> None: + nonlocal first_packet_ms + if not chunks: + first_packet_ms = round((time.monotonic() - started) * 1000) + if isinstance(data, (bytes, bytearray)): + chunks.append(bytes(data)) + + def on_error(message: object, *_: Any) -> None: + failure.append(type(message).__name__) + + synthesizer = nls.NlsSpeechSynthesizer( + url=os.environ.get("ALIYUN_NLS_URL", "wss://nls-gateway.cn-shanghai.aliyuncs.com/ws/v1"), + token=os.environ["ALIYUN_NLS_TOKEN"], + appkey=os.environ["ALIYUN_NLS_APPKEY"], + on_data=on_data, + on_error=on_error, + ) + try: + completed = synthesizer.start(text, voice=voice, aformat="pcm", sample_rate=16000) + except Exception as error: + raise TtsFixtureError("aliyun_nls_request_failed") from error + if completed is False or failure: + raise TtsFixtureError("aliyun_nls_request_failed") + audio = b"".join(chunks) + if not audio: + raise TtsFixtureError("empty_tts_audio") + return TtsFixtureResult( + audio=audio, + audio_format="pcm_s16le_16000_mono", + provider="aliyun-nls", + model=model, + voice=voice, + total_ms=round((time.monotonic() - started) * 1000), + first_packet_ms=first_packet_ms, + ) + + +def _macos_say_synthesize(text: str, voice: str) -> TtsFixtureResult: + path: Path | None = None + started = time.monotonic() + try: + with tempfile.NamedTemporaryFile(prefix="voicelife-serial-input-", suffix=".aiff", delete=False) as output: + path = Path(output.name) + subprocess.run(["say", "-v", voice, "-o", str(path), text], check=True, capture_output=True) + audio = path.read_bytes() + if not audio: + raise TtsFixtureError("empty_local_tts_audio") + return TtsFixtureResult( + audio=audio, + audio_format="encoded", + provider="macos-say", + model="macos-say", + voice=voice, + total_ms=round((time.monotonic() - started) * 1000), + first_packet_ms=0, + ) + except FileNotFoundError as error: + raise TtsFixtureError("macos_say_unavailable") from error + except subprocess.CalledProcessError as error: + raise TtsFixtureError("macos_say_failed") from error + finally: + if path is not None: + path.unlink(missing_ok=True) + + +def synthesize_fixture(text: str, provider: str, model: str, voice: str) -> TtsFixtureResult: + validate_provider_environment(provider) + if provider == "dashscope": + return _dashscope_synthesize(text, model, voice) + if provider == "aliyun-nls": + return _aliyun_nls_synthesize(text, model, voice) + return _macos_say_synthesize(text, voice) diff --git a/scripts/voice_tts_preflight.py b/scripts/voice_tts_preflight.py new file mode 100644 index 00000000..40e61bee --- /dev/null +++ b/scripts/voice_tts_preflight.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify a host TTS fixture before reserving time on a real voice device.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from voice_linx_serial_multiturn_test import to_pcm_frames +from voice_tts_fixture import TtsFixtureError, synthesize_fixture, validate_provider_environment + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-tts", choices=("dashscope", "aliyun-nls"), required=True) + parser.add_argument("--tts-model", required=True) + parser.add_argument("--voice", required=True) + parser.add_argument("--text", default="这是 VoiceLife HIL TTS 预检。") + parser.add_argument("--result-json", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + validate_provider_environment(args.input_tts) + fixture = synthesize_fixture(args.text, args.input_tts, args.tts_model, args.voice) + frames = to_pcm_frames(fixture.audio, fixture.audio_format) + except (TtsFixtureError, RuntimeError, OSError) as error: + print(f"input_preparation_failed:{error}", file=sys.stderr) + return 2 + report = { + "schema_version": 1, + "fixture": { + "provider": fixture.provider, + "model": fixture.model, + "voice": fixture.voice, + "requests": 1, + "audio_bytes": len(fixture.audio), + "first_packet_ms_max": fixture.first_packet_ms, + "total_ms_max": fixture.total_ms, + }, + "acceptance": { + "audio_present": bool(fixture.audio), + "pcm_frames_present": bool(frames), + }, + } + args.result_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return 0 if all(report["acceptance"].values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/host/linx_esp_transport_contract_test.cc b/tests/host/linx_esp_transport_contract_test.cc index 6fa87f16..560f6a07 100644 --- a/tests/host/linx_esp_transport_contract_test.cc +++ b/tests/host/linx_esp_transport_contract_test.cc @@ -50,10 +50,10 @@ int main() { Check(defaults.tx_timeout_ms == 1000, "Linx 默认同步写超时必须限制在 1 秒，避免 generation 切换拖慢本地打断"); Check(SelectLinxTextTxLane("{\"type\":\"listen\",\"state\":\"stop\"}") == LinxTextTxLane::kMediaOrdered, "listen.stop 必须排在已经入队的 PCM 之后，不能由控制队列越过尾音"); - Check(SelectLinxTextTxLane("{\"type\":\"listen\",\"state\":\"start\"}") == LinxTextTxLane::kControl && + Check(SelectLinxTextTxLane("{\"type\":\"listen\",\"state\":\"start\"}") == LinxTextTxLane::kMediaOrdered && SelectLinxTextTxLane("{\"type\":\"listen\",\"state\":\"detect\"}") == LinxTextTxLane::kControl && SelectLinxTextTxLane("{\"type\":\"abort\"}") == LinxTextTxLane::kControl, - "开始、检测和 abort 必须保留控制通道的低延迟抢占能力"); + "start 必须与 PCM 有序，detect 和 abort 仍保留控制通道的低延迟抢占能力"); LinxTxGenerationGate generation_gate; generation_gate.SetGeneration(1); diff --git a/tests/host/linx_provider_contract_test.cc b/tests/host/linx_provider_contract_test.cc index d2a99b0b..710c4888 100644 --- a/tests/host/linx_provider_contract_test.cc +++ b/tests/host/linx_provider_contract_test.cc @@ -101,7 +101,8 @@ voicelife::linx::LinxConnectionConfig Connection() { .token_ref = "secret://linx/device-token", .device_id = "device-test", .client_id = "client-test", - .agent_id = std::string("agent-test")}; + .agent_id = std::string("agent-test"), + .preferred_audio = std::nullopt}; } } // namespace @@ -116,8 +117,8 @@ int main() { Check(hello.value->find("\"transport\":\"websocket\"") != std::string::npos, "hello 必须声明 websocket transport"); Check(hello.value->find("\"mcp\":true") != std::string::npos, "hello 必须声明 MCP 能力"); Check(hello.value->find("\"sample_rate\":16000") != std::string::npos, "hello 必须声明采样率"); - Check(hello.value->find("\"play_buffer_duration\":200") != std::string::npos, - "默认播放缓冲必须保持在 200ms 实时预算内"); + Check(hello.value->find("\"play_buffer_duration\":1000") != std::string::npos, + "默认播放缓冲必须符合 Linx 文档的 1000ms 协议值"); auto larger_buffer_connection = connection; larger_buffer_connection.playback_buffer_duration_ms = 320; auto larger_buffer_hello = codec.EncodeHello(config, larger_buffer_connection); @@ -185,23 +186,26 @@ int main() { "Provider 应分别暴露请求的上行格式和 hello 协商的下行格式"); transport.EmitConnected(); Check(transport.texts.size() == 1, "重复 connected 事件不得重复发送 hello"); - Check(provider.NotifyLocalWakeWord("你好牛牛", "收到！").ok() && provider.StartCapture(config.mode).ok() && + Check(provider.NotifyLocalWakeWord("你好牛牛").ok() && provider.StartCapture(config.mode).ok() && provider.StopCapture().ok(), - "本地唤醒确认必须先 detect，再由状态机在 TTS 结束后开始采集"); + "普通本地唤醒必须先发送无确认音 detect，再由状态机开始采集"); Check(provider.Speak("测试播报").ok() && provider.Abort("user_interrupt").ok(), "detect/abort 应通过传输发送"); Check(transport.texts.size() == 6, "hello、本地 detect、listen、listen、detect、abort 应各发送一帧"); Check(transport.texts[1].find("\"type\":\"listen\"") != std::string::npos && transport.texts[1].find("\"state\":\"detect\"") != std::string::npos && transport.texts[1].find("\"text\":\"你好牛牛\"") != std::string::npos && - transport.texts[1].find("\"text_response\":\"收到！\"") != std::string::npos && + transport.texts[1].find("\"text_response\"") == std::string::npos && transport.texts[2].find("\"state\":\"start\"") != std::string::npos, - "本地唤醒 detect 必须请求确认播报；listen.start 由后续状态机控制"); + "普通本地唤醒 detect 不得请求确认播报；listen.start 由后续状态机控制"); Check(transport.texts[4].find("\"text\":\"system_prompt\"") != std::string::npos && transport.texts[4].find("\"text_response\":\"测试播报\"") != std::string::npos, "系统播报必须使用 Linx 定义的 text_response，不能伪装为用户 STT"); Check(transport.texts[1].find("\"session_id\":\"remote-linx-session\"") != std::string::npos && transport.texts[5].find("\"session_id\":\"remote-linx-session\"") != std::string::npos, "服务端 hello 分配的 session_id 必须用于后续控制消息"); + Check(transport.texts[3].find("\"state\":\"stop\"") != std::string::npos && + transport.texts[3].find("\"mode\"") == std::string::npos, + "listen.stop 必须遵循 Linx 文档，不携带 listen.start 的 mode"); const auto events_before_mismatched_session = events.size(); transport.EmitText(R"({"type":"stt","session_id":"wrong-session","text":"不应接受"})"); Check(events.size() == events_before_mismatched_session + 1 && diff --git a/tests/host/serial_voice_protocol_test.cc b/tests/host/serial_voice_protocol_test.cc index 231cfa43..dc0fbd29 100644 --- a/tests/host/serial_voice_protocol_test.cc +++ b/tests/host/serial_voice_protocol_test.cc @@ -9,6 +9,8 @@ using voicelife::runtime::detail::kSerialVoiceBegin; using voicelife::runtime::detail::kSerialVoicePcm; using voicelife::runtime::detail::kSerialVoicePcmBytes; using voicelife::runtime::detail::kSerialVoiceProtocolVersion; +using voicelife::runtime::detail::kSerialVoiceWakeBegin; +using voicelife::runtime::detail::kSerialVoiceWakeEnd; using voicelife::runtime::detail::SerialVoiceFrameHeader; using voicelife::runtime::detail::SerialVoiceMagicMatcher; using voicelife::test::Check; @@ -39,5 +41,11 @@ int main() { "超出固定 PCM 容量的声明长度必须在读取 payload 前拒绝"); Check(!IsValidSerialVoiceHeader({.version = kSerialVoiceProtocolVersion, .kind = 99, .payload_bytes = 0}), "未知 kind 不能作为合法空帧进入状态机"); + Check(IsValidSerialVoiceHeader( + {.version = kSerialVoiceProtocolVersion, .kind = kSerialVoiceWakeBegin, .payload_bytes = 0}), + "wake_begin 帧必须是合法空帧"); + Check(IsValidSerialVoiceHeader( + {.version = kSerialVoiceProtocolVersion, .kind = kSerialVoiceWakeEnd, .payload_bytes = 0}), + "wake_end 帧必须是合法空帧"); return 0; } diff --git a/tests/host/voice_interaction_controller_test.cc b/tests/host/voice_interaction_controller_test.cc index 1d20ea19..c433a6bb 100644 --- a/tests/host/voice_interaction_controller_test.cc +++ b/tests/host/voice_interaction_controller_test.cc @@ -20,13 +20,15 @@ void CheckTransition(VoiceInteractionController& controller, VoiceInteractionEve Check(transition.value->action == expected_action, "交互动作迁移错误"); } -void CompleteWakeAcknowledgement(VoiceInteractionController& controller, const char* message) { +void CompleteWakeDetect(VoiceInteractionController& controller, const char* message) { CheckTransition(controller, VoiceInteractionEvent::kWakeDetected, VoiceInteractionState::kAcknowledging, VoiceInteractionAction::kStartVoiceTurn, message); + CheckTransition(controller, VoiceInteractionEvent::kWakeDetectionAccepted, VoiceInteractionState::kAcknowledging, + VoiceInteractionAction::kNone, "detect 入队后应等待服务端问候或有界超时"); CheckTransition(controller, VoiceInteractionEvent::kTtsStarted, VoiceInteractionState::kSpeaking, - VoiceInteractionAction::kNone, "确认播报开始后应进入播报态"); + VoiceInteractionAction::kNone, "本地唤醒问候 TTS 开始后才进入播报态"); CheckTransition(controller, VoiceInteractionEvent::kTtsStopped, VoiceInteractionState::kOpeningCapture, - VoiceInteractionAction::kStartCapture, "确认播报结束后应事务式请求采集"); + VoiceInteractionAction::kStartCapture, "本地唤醒问候结束后才请求采集"); CheckTransition(controller, VoiceInteractionEvent::kCaptureStarted, VoiceInteractionState::kListening, VoiceInteractionAction::kNone, "只有采集成功确认后才能显示聆听中"); } @@ -67,6 +69,20 @@ int main() { VoiceInteractionState::kListening, VoiceInteractionAction::kNone, "确认超时后的采集成功确认才进入聆听"); + VoiceInteractionController detect_only_controller; + CheckTransition(detect_only_controller, VoiceInteractionEvent::kBootCompleted, VoiceInteractionState::kStandby, + VoiceInteractionAction::kRestoreStandby, "标准 detect 唤醒用例应先进入待机"); + CheckTransition(detect_only_controller, VoiceInteractionEvent::kWakeDetected, VoiceInteractionState::kAcknowledging, + VoiceInteractionAction::kStartVoiceTurn, "标准 detect 唤醒仍应先提交唤醒词"); + CheckTransition(detect_only_controller, VoiceInteractionEvent::kWakeDetectionAccepted, + VoiceInteractionState::kAcknowledging, VoiceInteractionAction::kNone, + "detect 入队成功后不得立即请求采集"); + CheckTransition(detect_only_controller, VoiceInteractionEvent::kAcknowledgementTimedOut, + VoiceInteractionState::kOpeningCapture, VoiceInteractionAction::kStartCapture, + "没有服务端问候时必须在有界超时后请求采集"); + CheckTransition(detect_only_controller, VoiceInteractionEvent::kCaptureStarted, VoiceInteractionState::kListening, + VoiceInteractionAction::kNone, "标准 detect 采集成功后才进入聆听"); + VoiceInteractionController acknowledgement_audio_timeout_controller; CheckTransition(acknowledgement_audio_timeout_controller, VoiceInteractionEvent::kBootCompleted, VoiceInteractionState::kStandby, VoiceInteractionAction::kRestoreStandby, @@ -80,7 +96,7 @@ int main() { CheckTransition(acknowledgement_audio_timeout_controller, VoiceInteractionEvent::kAcknowledgementTimedOut, VoiceInteractionState::kOpeningCapture, VoiceInteractionAction::kStartCapture, "远端 TTS 无首段 PCM 时必须中止等待并开始采集"); - CompleteWakeAcknowledgement(controller, "本地唤醒必须先进入确认播报阶段"); + CompleteWakeDetect(controller, "本地唤醒必须先完成 detect 再开始采集"); CheckTransition(controller, VoiceInteractionEvent::kIntentReceived, VoiceInteractionState::kThinking, VoiceInteractionAction::kNone, "识别文本或工具调用后应显示思考"); CheckTransition(controller, VoiceInteractionEvent::kTtsStarted, VoiceInteractionState::kSpeaking, @@ -93,14 +109,14 @@ int main() { VoiceInteractionController terminal_controller; CheckTransition(terminal_controller, VoiceInteractionEvent::kBootCompleted, VoiceInteractionState::kStandby, VoiceInteractionAction::kRestoreStandby, "终结型回复测试应先进入待机"); - CompleteWakeAcknowledgement(terminal_controller, "终结型回复应从完整唤醒确认开始"); + CompleteWakeDetect(terminal_controller, "终结型回复应从完整 detect 唤醒开始"); CheckTransition(terminal_controller, VoiceInteractionEvent::kTtsStarted, VoiceInteractionState::kSpeaking, VoiceInteractionAction::kNone, "终结型回复应允许进入播报状态"); CheckTransition(terminal_controller, VoiceInteractionEvent::kTerminalResponseCompleted, VoiceInteractionState::kStandby, VoiceInteractionAction::kRestoreStandby, "绑定码等终结型回复播报后应直接回待机，不进入 follow-up 聆听"); - CompleteWakeAcknowledgement(controller, "新一轮唤醒应可完整开始"); + CompleteWakeDetect(controller, "新一轮唤醒应可完整开始"); CheckTransition(controller, VoiceInteractionEvent::kTtsStarted, VoiceInteractionState::kSpeaking, VoiceInteractionAction::kNone, "无需先收到文本也允许服务器直接开始 TTS"); CheckTransition(controller, VoiceInteractionEvent::kTtsStopped, VoiceInteractionState::kOpeningCapture, @@ -115,7 +131,7 @@ int main() { VoiceInteractionController restart_during_finalization; CheckTransition(restart_during_finalization, VoiceInteractionEvent::kBootCompleted, VoiceInteractionState::kStandby, VoiceInteractionAction::kRestoreStandby, "重开语音用例应先完成启动"); - CompleteWakeAcknowledgement(restart_during_finalization, "重开语音用例应先完成确认再聆听"); + CompleteWakeDetect(restart_during_finalization, "重开语音用例应先完成 detect 再聆听"); CheckTransition(restart_during_finalization, VoiceInteractionEvent::kPressUp, VoiceInteractionState::kFinalizing, VoiceInteractionAction::kStopVoiceTurn, "松开后应进入最终识别等待"); CheckTransition(restart_during_finalization, VoiceInteractionEvent::kPressDown, @@ -125,7 +141,7 @@ int main() { VoiceInteractionState::kListening, VoiceInteractionAction::kNone, "旧回合取消后只有成功采集确认才能进入新聆听"); - CompleteWakeAcknowledgement(controller, "按住说打断路径前应可进入一轮语音"); + CompleteWakeDetect(controller, "按住说打断路径前应可进入一轮语音"); CheckTransition(controller, VoiceInteractionEvent::kTtsStarted, VoiceInteractionState::kSpeaking, VoiceInteractionAction::kNone, "按住说打断路径应可进入播报状态"); CheckTransition(controller, VoiceInteractionEvent::kPressDown, VoiceInteractionState::kInterrupting, @@ -154,7 +170,7 @@ int main() { CheckTransition(controller, VoiceInteractionEvent::kTransportConnected, VoiceInteractionState::kStandby, VoiceInteractionAction::kNone, "后台重连完成不得扰动空闲显示"); - CompleteWakeAcknowledgement(controller, "待机唤醒仍应完成确认并开始云端语音"); + CompleteWakeDetect(controller, "待机唤醒仍应完成 detect 并开始云端语音"); CheckTransition(controller, VoiceInteractionEvent::kTtsStarted, VoiceInteractionState::kSpeaking, VoiceInteractionAction::kNone, "打断回归路径应允许直接播报"); CheckTransition(controller, VoiceInteractionEvent::kWakeDetected, VoiceInteractionState::kInterrupting, @@ -162,14 +178,14 @@ int main() { CheckTransition(controller, VoiceInteractionEvent::kInterruptCompleted, VoiceInteractionState::kStandby, VoiceInteractionAction::kRestoreStandby, "播报打断后应回到待机再等待下一次唤醒"); - CompleteWakeAcknowledgement(controller, "待机中唤醒应完成确认并开始新一轮云端语音"); + CompleteWakeDetect(controller, "待机中唤醒应完成 detect 并开始新一轮云端语音"); CheckTransition(controller, VoiceInteractionEvent::kWakeDetected, VoiceInteractionState::kStandby, VoiceInteractionAction::kStopVoiceTurn, "聆听中再次唤醒应关闭当前音频通道"); VoiceInteractionController interrupt_ack_controller; CheckTransition(interrupt_ack_controller, VoiceInteractionEvent::kBootCompleted, VoiceInteractionState::kStandby, VoiceInteractionAction::kRestoreStandby, "打断确认用例应先进入待机"); - CompleteWakeAcknowledgement(interrupt_ack_controller, "打断确认用例应先开始一轮语音"); + CompleteWakeDetect(interrupt_ack_controller, "打断确认用例应先开始一轮语音"); CheckTransition(interrupt_ack_controller, VoiceInteractionEvent::kTtsStarted, VoiceInteractionState::kSpeaking, VoiceInteractionAction::kNone, "打断确认用例应进入播报"); CheckTransition(interrupt_ack_controller, VoiceInteractionEvent::kInterruptAndAcknowledge, @@ -188,7 +204,7 @@ int main() { "失败必须中止远端轮次后恢复本地待机"); CheckTransition(controller, VoiceInteractionEvent::kStandbyReady, VoiceInteractionState::kStandby, VoiceInteractionAction::kNone, "本地待机恢复后应清除错误状态"); - CompleteWakeAcknowledgement(controller, "错误恢复后仍应可重新唤醒"); + CompleteWakeDetect(controller, "错误恢复后仍应可重新唤醒"); CheckTransition(controller, VoiceInteractionEvent::kInterruptRequested, VoiceInteractionState::kInterrupting, VoiceInteractionAction::kInterruptSession, "重新唤醒后仍应支持打断"); CheckTransition(controller, VoiceInteractionEvent::kStandbyReady, VoiceInteractionState::kStandby, @@ -205,7 +221,7 @@ int main() { CheckTransition(multi_turn_controller, VoiceInteractionEvent::kBootCompleted, VoiceInteractionState::kStandby, VoiceInteractionAction::kRestoreStandby, "多轮测试应先进入待机"); for (int turn = 0; turn < 24; ++turn) { - CompleteWakeAcknowledgement(multi_turn_controller, "每轮都必须能从待机开始完整语音会话"); + CompleteWakeDetect(multi_turn_controller, "每轮都必须能从待机开始完整语音会话"); if (turn % 3 == 0) { CheckTransition(multi_turn_controller, VoiceInteractionEvent::kEndpointDetected, VoiceInteractionState::kFinalizing, VoiceInteractionAction::kStopVoiceTurn, diff --git a/tests/host/voice_session_contract_test.cc b/tests/host/voice_session_contract_test.cc index 40d556b2..155a8b18 100644 --- a/tests/host/voice_session_contract_test.cc +++ b/tests/host/voice_session_contract_test.cc @@ -102,6 +102,7 @@ class FakeProvider final : public voicelife::voice::SpeechProviderAdapter { } Status StartCapture(voicelife::voice::VoiceMode) override { ++starts; + calls.push_back("listen.start"); return start_result; } Status StopCapture() override { @@ -124,6 +125,7 @@ class FakeProvider final : public voicelife::voice::SpeechProviderAdapter { } Status NotifyLocalWakeWord(std::string_view wake_word, std::string_view text_response = {}) override { ++wake_notifications; + calls.push_back("listen.detect"); last_wake_word = std::string(wake_word); last_wake_response = std::string(text_response); return wake_notification_result; @@ -169,6 +171,7 @@ class FakeProvider final : public voicelife::voice::SpeechProviderAdapter { int speaks = 0; int wake_notifications = 0; int disconnects = 0; + std::vector calls; std::string last_wake_word; std::string last_wake_response; }; @@ -237,19 +240,19 @@ int main() { Check(session.state() == voicelife::voice::VoiceSessionState::kReady, "启动后应进入 ready"); Check(session.NotifyLocalWakeWord("你好牛牛", "收到！").ok() && provider.wake_notifications == 1 && provider.last_wake_word == "你好牛牛" && provider.last_wake_response == "收到！", - "本地唤醒确认必须只通过 Provider 请求受控 TTS"); + "普通本地唤醒必须通过 Provider 请求受控确认 TTS"); provider.Emit(voicelife::voice::VoiceEvent{.kind = voicelife::voice::VoiceEventKind::kTtsStarted, .generation = session.generation(), .text = {}, .aborted = false}); Check(session.state() == voicelife::voice::VoiceSessionState::kSpeaking, - "本地唤醒确认的真实 TTS start 才能进入 speaking"); + "唤醒确认的真实 TTS start 才能进入 speaking"); provider.Emit(voicelife::voice::VoiceEvent{.kind = voicelife::voice::VoiceEventKind::kTtsStopped, .generation = session.generation(), .text = {}, .aborted = false}); Check(session.state() == voicelife::voice::VoiceSessionState::kReady, - "本地唤醒确认 TTS 结束后会话必须允许开始真实聆听"); + "唤醒确认 TTS 结束后会话必须允许开始真实聆听"); session.ReportToolCallStarted(); session.ReportToolResult("event=创建会议", true); Check(session.state() == voicelife::voice::VoiceSessionState::kReady && provider.audio_frames == 0 && @@ -395,6 +398,44 @@ int main() { "停止会话应清理输入回调，避免资源关闭后的迟到帧"); Check(evidence_count >= 4, "会话生命周期应产出可关联的证据事件"); + FakeInput wake_input; + FakeOutput wake_output; + FakeProvider wake_provider; + std::vector wake_evidence; + voicelife::voice::VoiceSession wake_session(wake_input, wake_output, wake_provider, + [&wake_evidence](const auto& item) { wake_evidence.push_back(item); }); + auto wake_config = Config(); + wake_config.mode = voicelife::voice::VoiceMode::kAuto; + Check(wake_session.Start(wake_config).ok() && wake_session.NotifyLocalWakeWord("你好牛牛").ok(), + "普通唤醒应先提交 detect"); + Check(wake_provider.calls == std::vector{"listen.detect"} && wake_provider.starts == 0 && + wake_input.starts == 0, + "detect 后尚未进入协议监听时不得打开物理输入"); + Check(wake_session.BeginProviderCapture().ok() && + wake_provider.calls == std::vector{"listen.detect", "listen.start"} && + wake_provider.starts == 1 && wake_input.starts == 0, + "普通唤醒必须按 detect -> listen.start 顺序发送且暂不打开物理输入"); + const uint64_t wake_generation = wake_session.generation(); + Check(wake_input.EmitCapture(Frame(wake_generation, 0)).code == ErrorCode::kUnavailable, + "问候 TTS 阶段的物理 PCM 不得进入 Provider"); + wake_provider.Emit(voicelife::voice::VoiceEvent{.kind = voicelife::voice::VoiceEventKind::kAsrText, + .generation = wake_generation, + .text = "你好牛牛", + .aborted = false}); + Check(!wake_evidence.empty() && wake_evidence.back().event == "wake_echo_suppressed", + "服务端回传唤醒词必须只抑制一次，不得武装错误回复"); + wake_provider.Emit(voicelife::voice::VoiceEvent{.kind = voicelife::voice::VoiceEventKind::kTtsStarted, + .generation = wake_generation, + .text = {}, + .aborted = false}); + wake_provider.Emit(voicelife::voice::VoiceEvent{.kind = voicelife::voice::VoiceEventKind::kTtsStopped, + .generation = wake_generation, + .text = {}, + .aborted = false}); + Check(wake_session.BeginCapture().ok() && wake_provider.starts == 1 && wake_input.starts == 1, + "问候结束后应只开启物理采集，不重复发送 listen.start"); + Check(wake_session.EndCapture().ok(), "普通唤醒后的真实采集应可正常结束"); + FakeInput acknowledged_wake_input; FakeOutput acknowledged_wake_output; FakeProvider acknowledged_wake_provider; @@ -536,6 +577,14 @@ int main() { stop_capture_failure_provider.generation_ == stop_capture_failure.generation(), "本地已停止而远端停止失败时回 ready 并使旧代次失效，不得卡死在 capturing"); + FakeInput duplicate_end_input; + FakeOutput duplicate_end_output; + FakeProvider duplicate_end_provider; + voicelife::voice::VoiceSession duplicate_end(duplicate_end_input, duplicate_end_output, duplicate_end_provider); + Check(duplicate_end.Start(Config()).ok() && duplicate_end.BeginCapture().ok() && duplicate_end.EndCapture().ok() && + duplicate_end.EndCapture().ok() && duplicate_end_input.stops == 1 && duplicate_end_provider.stops == 1, + "重复结束采集必须幂等成功且只发送一次本地/远端 stop"); + FakeInput disconnect_failure_input; FakeOutput disconnect_failure_output; FakeProvider disconnect_failure_provider; diff --git a/tests/host/wake_gate_audio_input_test.cc b/tests/host/wake_gate_audio_input_test.cc index e3283b6d..5da1be8b 100644 --- a/tests/host/wake_gate_audio_input_test.cc +++ b/tests/host/wake_gate_audio_input_test.cc @@ -25,6 +25,10 @@ class FakeInput final : public voicelife::voice::AudioInputPort { ++stops; return Status::Ok(); } + Status DiscardPendingInput() override { + ++discarded; + return Status::Ok(); + } void Close() override { ++closes; } Status Emit(voicelife::voice::AudioFrame frame) { return sink_ ? sink_(std::move(frame)) : Status::Error(ErrorCode::kUnavailable, "采集回调未绑定"); @@ -33,6 +37,7 @@ class FakeInput final : public voicelife::voice::AudioInputPort { int opens = 0; int starts = 0; int stops = 0; + int discarded = 0; int closes = 0; private: @@ -101,9 +106,15 @@ int main() { Check(wake_events == 1 && forwarded == 0, "检测事件不能把待机 PCM 误送云端"); Check(gate.StartCapture(voicelife::voice::VoiceMode::kRealtime).ok(), "唤醒后应能切换到云端采集"); - Check(physical.starts == 1 && detector.stops == 0, "唤醒命中后检测器已自停，切换上行不应重复停止检测器"); + Check(physical.starts == 1 && physical.stops == 0 && physical.discarded == 1 && detector.stops == 0, + "唤醒切换上行必须清除唤醒前 PCM，不能重复停止已自停检测器或重启物理采集"); + Check(physical.Emit(Frame()).ok() && detector.frames == 1 && forwarded == 0, + "唤醒边界迟到 PCM 不得转发 VoiceSession"); + for (int index = 0; index < 3; ++index) { + Check(physical.Emit(Frame()).ok(), "边界窗口内的 PCM 必须可被丢弃"); + } Check(physical.Emit(Frame()).ok() && detector.frames == 1 && forwarded == 1, - "上行状态 PCM 必须只转发 VoiceSession"); + "边界窗口之后的 PCM 必须转发 VoiceSession"); Check(gate.StopCapture().ok() && !gate.standby(), "停止上行不得在播报或最终识别期间隐式恢复本地唤醒"); Check(detector.starts == 1 && physical.starts == 1, "停止上行不得重启物理采集或检测器"); Check(physical.Emit(Frame()).ok() && detector.frames == 1 && forwarded == 1, diff --git a/tests/python/test_e2e_artifact_tools.py b/tests/python/test_e2e_artifact_tools.py new file mode 100644 index 00000000..25f468d8 --- /dev/null +++ b/tests/python/test_e2e_artifact_tools.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" + + +def load(name: str, path: Path) -> object: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +EVIDENCE = load("e2e_evidence", SCRIPTS / "e2e_evidence.py") +CHECK = load("check_e2e_artifacts", SCRIPTS / "check_e2e_artifacts.py") +SUMMARY = load("render_e2e_summary", SCRIPTS / "render_e2e_summary.py") + + +class E2EArtifactToolsTest(unittest.TestCase): + def evidence(self) -> dict[str, object]: + return { + "schema_version": 1, + "run_id": "a" * 32, + "correlation_id": "b" * 32, + "scope": "runner_contract_only", + "layer": "host", + "journey": "lifecycle-example", + "profile": "host", + "started_at": "2026-08-21T00:00:00.000Z", + "finished_at": "2026-08-21T00:00:01.000Z", + "duration_ms": 1000, + "status": "passed", + "failure_category": None, + "failed_phase": None, + "message_code": "run_passed", + "stages": [ + {"name": phase, "status": "passed", "code": "phase_passed"} + for phase in ("prepare", "run", "assert", "collect", "cleanup") + ], + "assertions": [], + "metrics": {"resource_count": 1}, + "cleanup": {"status": "passed", "error_codes": []}, + "hardware_verified": False, + "hil": None, + } + + def test_validates_evidence_and_renders_only_public_fields(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "evidence-a.json").write_text(json.dumps(self.evidence()), encoding="utf-8") + self.assertEqual(CHECK.validate_directory(root), (1, 1)) + summary = SUMMARY.render(root) + self.assertIn("lifecycle-example", summary) + self.assertNotIn("run_id", summary) + + def test_rejects_raw_logs_and_sensitive_json(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "serial.log").write_text("raw", encoding="utf-8") + with self.assertRaises(ValueError): + CHECK.validate_directory(root) + (root / "serial.log").unlink() + (root / "other.json").write_text(json.dumps({"token": "canary"}), encoding="utf-8") + with self.assertRaises(ValueError): + CHECK.validate_directory(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_e2e_evidence.py b/tests/python/test_e2e_evidence.py index f058f606..8fed2f3d 100644 --- a/tests/python/test_e2e_evidence.py +++ b/tests/python/test_e2e_evidence.py @@ -178,6 +178,73 @@ def test_accepts_strict_hil_pairing_evidence(self) -> None: EVIDENCE.validate_evidence(document) self.assertEqual(json.loads(EVIDENCE.canonical_json(document)), document) + def voice_document(self) -> dict[str, object]: + document = self.document() + document.update( + { + "scope": "hil_voice", + "layer": "hil", + "journey": "voice", + "profile": "sparkbot", + "hardware_verified": True, + "assertions": [ + {"name": "voice_turns_complete", "passed": True, "code": "ok"}, + {"name": "voice_acceptance_clean", "passed": True, "code": "ok"}, + ], + "metrics": { + "input_tts_requests": 3, + "input_tts_audio_bytes": 32000, + "input_tts_first_packet_ms_max": 180, + "input_tts_total_ms_max": 500, + "wake_completed": 1, + "requested_turns": 3, + "completed_turns": 3, + "asr_exact_matches": 3, + "test_in_frames": 120, + "out_frames": 240, + "in_drop": 0, + "out_reject": 0, + "short_write": 0, + "in_i2s_err": 0, + "out_i2s_err": 0, + "serial_pcm_rejections": 0, + "display_content_snapshots": 6, + }, + "hil": { + "firmware_sha256": "a" * 64, + "gateway_commit": "b" * 40, + "device_fingerprint": "c" * 16, + "readiness_markers": ["provisioned", "wifi_ready", "sntp_synced", "ready"], + "input_tts_provider": "dashscope", + "input_tts_model": "qwen-audio-3.0-tts-flash", + "input_tts_voice": "longanlingxi", + "wake_stage": "passed", + }, + } + ) + return document + + def test_accepts_strict_hil_voice_evidence_without_raw_conversation(self) -> None: + document = self.voice_document() + EVIDENCE.validate_evidence(document) + self.assertEqual(json.loads(EVIDENCE.canonical_json(document)), document) + + def test_accepts_pcb_hil_voice_evidence(self) -> None: + document = self.voice_document() + document["profile"] = "pcb" + document["hil"]["wake_stage"] = "not_applicable" + document["metrics"]["wake_completed"] = 0 + EVIDENCE.validate_evidence(document) + + def test_voice_evidence_rejects_pairing_fields_and_unsafe_model(self) -> None: + pairing_fields = self.voice_document() + pairing_fields["hil"]["pairing_markers"] = ["pending"] + unsafe_model = self.voice_document() + unsafe_model["hil"]["input_tts_model"] = "model with spaces" + for document in (pairing_fields, unsafe_model): + with self.subTest(document=document), self.assertRaises(EVIDENCE.EvidenceValidationError): + EVIDENCE.validate_evidence(document) + def test_hil_evidence_rejects_missing_gates_false_success_claim_and_sensitive_values(self) -> None: missing = self.hil_document() missing["hil"]["readiness_markers"] = ["ready"] diff --git a/tests/python/test_e2e_hil_adapter.py b/tests/python/test_e2e_hil_adapter.py index 5986e560..08950c3c 100644 --- a/tests/python/test_e2e_hil_adapter.py +++ b/tests/python/test_e2e_hil_adapter.py @@ -2,6 +2,7 @@ import hashlib import json +import os import sys import tempfile import types @@ -38,9 +39,21 @@ def __init__( self.device_revoked = False self.serial_open = False self.reset_count = 0 + self.build_profiles: list[str] = [] def inspect(self, descriptor: object, temporary_directory: Path) -> object: self.calls.append("inspect") + if getattr(descriptor, "profile", "") == "pcb": + return [ + HIL.Partition("nvs", 1, 2, 0x9000, 0x6000, 0), + HIL.Partition("otadata", 1, 0, 0xF000, 0x2000, 0), + HIL.Partition("phy_init", 1, 1, 0x11000, 0x1000, 0), + HIL.Partition("ota_0", 0, 0x10, 0x20000, 0x3E0000, 0), + HIL.Partition("ota_1", 0, 0x11, 0x400000, 0x3E0000, 0), + HIL.Partition("voicelife", 1, 0x82, 0x7E0000, 0x200000, 0), + HIL.Partition("linx_secrets", 1, 2, 0xA00000, 0x10000, 0), + HIL.Partition("model", 1, 0x82, 0xA10000, 0x300000, 0), + ] return [ HIL.Partition("nvs", 1, 2, 0x9000, 0x4000, 0), HIL.Partition("otadata", 1, 0, 0xD000, 0x2000, 0), @@ -49,15 +62,18 @@ def inspect(self, descriptor: object, temporary_directory: Path) -> object: HIL.Partition("linx_secrets", 1, 2, 0x2E0000, 0x10000, 0), HIL.Partition("assets", 1, 0x82, 0x300000, 0x100000, 0), HIL.Partition("model", 1, 0x82, 0x400000, 0x300000, 0), + HIL.Partition("voicelife", 1, 0x81, 0x700000, 0x900000, 0), ] def build(self, descriptor: object) -> Path: self.calls.append("build") + self.build_profiles.append(str(getattr(descriptor, "firmware_profile", ""))) return Path("/safe/build") def image(self, build_directory: Path, descriptor: object, partitions: object) -> object: self.calls.append("image") - return HIL.ApplicationImage(Path("/safe/voicelife.bin"), 0x10000, 8, "a" * 64) + offset = 0x20000 if getattr(descriptor, "profile", "") == "pcb" else 0x10000 + return HIL.ApplicationImage(Path("/safe/voicelife.bin"), offset, 8, "a" * 64) def flash(self, descriptor: object, image: object, timeout_s: float) -> None: self.calls.append("flash") @@ -95,10 +111,20 @@ def recover(self, descriptor: object) -> None: class HilPairingAdapterTest(unittest.TestCase): - def descriptor_file(self, directory: str) -> Path: + def test_real_hardware_prepares_sqlite_component_before_build(self) -> None: + with ( + mock.patch.object(HIL, "SQLITE_COMPONENT_FILES", (Path("/definitely-missing-sqlite.c"),)), + mock.patch.object(HIL.subprocess, "run") as run, + ): + HIL.ensure_sqlite_component() + run.assert_called_once() + command = run.call_args.args[0] + self.assertEqual(command[-1], str(HIL.ROOT / "scripts" / "prepare_sqlite.py")) + + def descriptor_file(self, directory: str, profile: str = "sparkbot") -> Path: path = Path(directory) / "device.json" path.write_text( - json.dumps({"schema_version": 1, "name": "bench-a", "port": "/dev/cu.test-a", "profile": "sparkbot"}), + json.dumps({"schema_version": 1, "name": "bench-a", "port": "/dev/cu.test-a", "profile": profile}), encoding="utf-8", ) return path @@ -175,6 +201,14 @@ def test_profile_mismatch_prevents_flash(self) -> None: self.assertNotIn("flash", hardware.calls) self.assertFalse(adapter.lease_held) + def test_lease_conflict_is_classified_separately_from_device_failure(self) -> None: + hardware = FakeHardware() + with mock.patch.object(HIL.DeviceLease, "acquire", side_effect=HIL.HilLeaseUnavailable): + adapter, result = self.execute(hardware) + self.assertEqual(result.failure_category, RUNNER.FailureCategory.LEASE) + self.assertEqual(result.message_code, "device_lease_unavailable") + self.assertFalse(adapter.lease_held) + def test_registration_failure_revokes_run_scoped_device_and_releases_lease(self) -> None: hardware = FakeHardware() hardware.register = mock.Mock( @@ -229,6 +263,22 @@ def test_real_hardware_classifies_missing_command_as_infrastructure(self) -> Non self.assertEqual(raised.exception.category, RUNNER.FailureCategory.INFRASTRUCTURE) self.assertEqual(raised.exception.message_code, "hil_command_unavailable") + def test_real_hardware_classifies_remote_service_failure_as_external(self) -> None: + hardware = HIL.RealHilHardware( + "runner@example.test", "/srv/voicelife", "https://gateway.example.test", "user-test" + ) + with ( + mock.patch.object( + hardware, + "_run", + side_effect=RUNNER.RunnerFailure(RUNNER.FailureCategory.INFRASTRUCTURE, "hil_command_failed"), + ), + self.assertRaises(RUNNER.RunnerFailure) as raised, + ): + hardware._remote("safe script") + self.assertEqual(raised.exception.category, RUNNER.FailureCategory.EXTERNAL) + self.assertEqual(raised.exception.message_code, "external_service_unavailable") + def test_real_hardware_passes_serial_path_as_string(self) -> None: serial_port = mock.Mock() serial_port.__enter__ = mock.Mock(return_value=serial_port) @@ -247,5 +297,151 @@ def test_real_hardware_passes_serial_path_as_string(self) -> None: serial_module.Serial.assert_called_once_with("/dev/cu.test", 115200, timeout=0.2, write_timeout=2) +class HilVoiceAdapterTest(HilPairingAdapterTest): + def voice_config(self, profile: str = "sparkbot") -> object: + return RUNNER.RunnerConfig( + layer="hil", + journey="voice", + profile=profile, + hard_timeout_s=5.0, + phase_timeout_s=3.0, + cleanup_timeout_s=0.5, + ) + + def execute_voice(self, hardware: FakeHardware, profile: str = "sparkbot") -> tuple[HIL.HilVoiceAdapter, object]: + with tempfile.TemporaryDirectory() as directory: + adapter = HIL.HilVoiceAdapter( + self.descriptor_file(directory, profile), + Path(directory) / "leases", + hardware=hardware, + input_tts="dashscope", + tts_model="qwen-audio-3.0-tts-flash", + voice="longanlingxi", + texts=["你好"], + expect_terminal=False, + response_timeout=5.0, + ) + result = RUNNER.run_e2e(self.voice_config(profile), adapter) + return adapter, result + + def test_dashscope_voice_journey_uses_voice_profile_and_sanitized_metrics(self) -> None: + hardware = FakeHardware() + fixture = { + "provider": "dashscope", + "model": "qwen-audio-3.0-tts-flash", + "voice": "longanlingxi", + "requests": 1, + "audio_bytes": 4096, + "first_packet_ms_max": 120, + "total_ms_max": 360, + } + voice_result = { + "schema_version": 2, + "fixture": fixture, + "requested_turns": 1, + "completed_turns": 1, + "asr_exact_matches": 1, + "audio_stats": { + "test_in_frames": 4, + "out_frames": 8, + "in_drop": 0, + "out_reject": 0, + "short_write": 0, + "in_i2s_err": 0, + "out_i2s_err": 0, + }, + "serial_pcm_rejections": [], + "display": {"content_snapshots": 2}, + "acceptance": {key: True for key in HIL.HilVoiceAdapter.CONVERSATION_ACCEPTANCE_KEYS}, + } + wake_result = { + "schema_version": 1, + "fixture": fixture, + "requested_followups": 0, + "completed_followups": 0, + "acceptance": {key: True for key in HIL.HilVoiceAdapter.WAKE_ACCEPTANCE_KEYS}, + "exit_code": 0, + } + preflight_result = { + "schema_version": 1, + "fixture": fixture, + "acceptance": {key: True for key in HIL.HilVoiceAdapter.PREFLIGHT_ACCEPTANCE_KEYS}, + } + + commands: list[list[str]] = [] + + def run_voice(command: list[str], **kwargs: object) -> object: + commands.append(command) + result_path = Path(command[command.index("--result-json") + 1]) + if "voice_tts_preflight.py" in command[1]: + payload = preflight_result + elif "voice_linx_wake_injection_test.py" in command[1]: + payload = wake_result + else: + payload = voice_result + result_path.write_text(json.dumps(payload), encoding="utf-8") + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + with ( + mock.patch.dict(os.environ, {"DASHSCOPE_API_KEY": "test-key"}, clear=False), + mock.patch.object(HIL, "validate_provider_environment"), + mock.patch.object(HIL.shutil, "which", return_value="/usr/bin/ffmpeg"), + mock.patch.object(HIL.subprocess, "run", side_effect=run_voice), + ): + for profile, firmware_profile in ( + ("sparkbot", "esp32s3-esp-sparkbot-serial-voice"), + ("pcb", "esp32s3-voicelife-pcb-serial-voice"), + ): + with self.subTest(profile=profile): + adapter, result = self.execute_voice(hardware, profile) + self.assertEqual(result.exit_code, RUNNER.ExitCode.SUCCESS) + self.assertEqual(hardware.build_profiles[-1], firmware_profile) + self.assertEqual(commands[-1][commands[-1].index("--display-profile") + 1], profile) + self.assertEqual(result.collected["scope"], "hil_voice") + self.assertTrue(result.collected["hardware_verified"]) + self.assertEqual(result.collected["metrics"]["asr_exact_matches"], 1) + self.assertEqual(result.collected["input_tts_model"], "qwen-audio-3.0-tts-flash") + self.assertEqual( + result.collected["wake_stage"], "passed" if profile == "sparkbot" else "not_applicable" + ) + self.assertNotIn("你好", repr(result.collected)) + self.assertFalse(adapter.lease_held) + + self.assertEqual( + [assertion.name for assertion in result.assertions], + [ + "voice_input_tts_preflight_clean", + "voice_input_fixture_clean", + "voice_wake_sequence_clean", + "voice_turns_complete", + "voice_state_flow_clean", + "voice_display_flow_clean", + "voice_wake_guard_clean", + "voice_acceptance_clean", + ], + ) + + def test_voice_acceptance_contract_fails_closed_when_a_check_disappears(self) -> None: + incomplete = {key: True for key in HIL.HilVoiceAdapter.CONVERSATION_ACCEPTANCE_KEYS} + incomplete.pop("zero_loss") + self.assertFalse( + HIL.HilVoiceAdapter._acceptance_valid(incomplete, HIL.HilVoiceAdapter.CONVERSATION_ACCEPTANCE_KEYS) + ) + + def test_voice_script_exit_categories_are_stable(self) -> None: + self.assertEqual( + HIL.HilVoiceAdapter._classify_voice_exit(1, "").category, + RUNNER.FailureCategory.PRODUCT, + ) + self.assertEqual( + HIL.HilVoiceAdapter._classify_voice_exit(2, "cannot open serial port").category, + RUNNER.FailureCategory.DEVICE, + ) + self.assertEqual( + HIL.HilVoiceAdapter._classify_voice_exit(2, "input_preparation_failed:timeout").category, + RUNNER.FailureCategory.EXTERNAL, + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_e2e_hil_device.py b/tests/python/test_e2e_hil_device.py index f5d9df12..cfc22a53 100644 --- a/tests/python/test_e2e_hil_device.py +++ b/tests/python/test_e2e_hil_device.py @@ -6,6 +6,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "scripts")) @@ -69,6 +70,7 @@ def sparkbot_layout(self) -> list[object]: self.partition("linx_secrets", 1, 2, 0x2E0000, 0x10000), self.partition("assets", 1, 0x82, 0x300000, 0x100000), self.partition("model", 1, 0x82, 0x400000, 0x300000), + self.partition("voicelife", 1, 0x81, 0x700000, 0x900000), ] def pcb_layout(self) -> list[object]: @@ -119,6 +121,23 @@ def test_application_image_must_match_profile_offset_and_partition_size(self) -> with self.assertRaises(HIL.HilProfileMismatch): HIL.load_application_image(build, descriptor, self.sparkbot_layout()) + def test_application_image_rejects_build_layout_different_from_board(self) -> None: + descriptor = self.descriptor("sparkbot") + with tempfile.TemporaryDirectory() as directory: + build = Path(directory) + (build / "voicelife.bin").write_bytes(b"firmware") + (build / "flasher_args.json").write_text( + json.dumps({"flash_files": {"0x10000": "voicelife.bin"}}), encoding="utf-8" + ) + table = build / "partition_table" / "partition-table.bin" + table.parent.mkdir() + table.write_bytes(b"partition-table") + with ( + patch.object(HIL, "parse_partition_table", return_value=self.pcb_layout()), + self.assertRaises(HIL.HilProfileMismatch), + ): + HIL.load_application_image(build, descriptor, self.sparkbot_layout()) + def test_flash_plan_writes_and_verifies_only_the_validated_application(self) -> None: with tempfile.TemporaryDirectory() as directory: image_path = Path(directory) / "voicelife.bin" diff --git a/tests/python/test_e2e_runner.py b/tests/python/test_e2e_runner.py index 36bec39b..147addb6 100644 --- a/tests/python/test_e2e_runner.py +++ b/tests/python/test_e2e_runner.py @@ -115,6 +115,7 @@ def test_failure_categories_have_stable_exit_codes(self) -> None: self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.INFRASTRUCTURE), 10) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.PRODUCT), 20) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.DEVICE), 30) + self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.LEASE), 31) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.EXTERNAL), 40) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.TIMEOUT), 60) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.INTERRUPTED), 70) diff --git a/tests/python/test_firmware.py b/tests/python/test_firmware.py index 122822e4..75ddf0be 100644 --- a/tests/python/test_firmware.py +++ b/tests/python/test_firmware.py @@ -129,6 +129,17 @@ def test_im_pcb_profile_verifies_cross_signed_cloudflare_chain(self) -> None: self.assertIn("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_CROSS_SIGNED_VERIFY=y", profile["sdkconfig"]) + def test_pcb_serial_voice_profile_keeps_im_and_enables_test_harness(self) -> None: + profile_path = ROOT / "config" / "profiles" / "esp32s3-voicelife-pcb-serial-voice.json" + profile = json.loads(profile_path.read_text(encoding="utf-8")) + + self.assertEqual(profile["adapters"]["im"]["driver"], "voicelife-gateway") + self.assertIn("CONFIG_VOICELIFE_SERIAL_VOICE_TEST=y", profile["sdkconfig"]) + self.assertIn( + 'CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="config/partitions/voicelife-pcb.csv"', profile["sdkconfig"] + ) + self.assertNotIn("CONFIG_VOICELIFE_BOARD_ESP_SPARKBOT=y", profile["sdkconfig"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_run_e2e.py b/tests/python/test_run_e2e.py index 7e6b5447..b59d303b 100644 --- a/tests/python/test_run_e2e.py +++ b/tests/python/test_run_e2e.py @@ -8,6 +8,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" @@ -107,6 +108,23 @@ def remaining(self) -> float: self.assertEqual(raised.exception.category, RUNNER.FailureCategory.INFRASTRUCTURE) self.assertEqual(raised.exception.message_code, "host_recovery_journey_failed") + def test_recovery_details_stay_out_of_public_artifacts(self) -> None: + class Context: + run_id = "a" * 32 + temporary_directory = Path("/tmp/voicelife-e2e-test") + cleanup = mock.Mock() + + with tempfile.TemporaryDirectory() as directory, mock.patch.object(ADAPTERS.subprocess, "Popen") as popen: + artifact_directory = Path(directory) / "artifacts" + adapter = ADAPTERS.HostImGatewayRecoveryE2EAdapter(artifact_directory) + adapter.prepare(Context()) + + environment = popen.call_args.kwargs["env"] + detail_path = Path(environment["E2E_RECOVERY_EVIDENCE"]) + self.assertEqual(detail_path.parent.parent, Context.temporary_directory) + self.assertNotEqual(detail_path.parent, artifact_directory) + self.assertFalse(artifact_directory.exists()) + class RunE2eCliTest(unittest.TestCase): def run_cli(self, *arguments: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: @@ -153,6 +171,17 @@ def test_host_and_hil_cli_write_valid_sanitized_evidence(self) -> None: self.assertFalse(document["hardware_verified"]) self.assertNotIn(str(evidence_paths[0]), result.stdout) + def test_contract_failure_writes_failed_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + result = self.run_cli( + *self.base_args(Path(directory)), env={**os.environ, "VOICELIFE_E2E_CONTRACT_FAILURE": "1"} + ) + self.assertEqual(result.returncode, 20, result.stderr) + document = json.loads(next(Path(directory).glob("evidence-*.json")).read_text(encoding="utf-8")) + EVIDENCE.validate_evidence(document) + self.assertEqual(document["failure_category"], "product") + self.assertEqual(document["failed_phase"], "assert") + def test_cli_rejects_nonzero_retries_and_hil_host_profile(self) -> None: with tempfile.TemporaryDirectory() as directory: retry_args = self.base_args(Path(directory)) @@ -374,6 +403,43 @@ def test_build_adapter_registers_real_hil_without_host_fallback(self) -> None: with self.assertRaises(ValueError): RUN_E2E.build_adapter("host", "im-pairing", args) + def test_build_adapter_registers_voice_for_supported_profiles(self) -> None: + with tempfile.TemporaryDirectory() as directory: + descriptor = Path(directory) / "device.json" + for profile in ("sparkbot", "pcb"): + descriptor.write_text( + json.dumps({"schema_version": 1, "name": "bench-a", "port": "/dev/cu.test", "profile": profile}), + encoding="utf-8", + ) + args = RUN_E2E.parse_args( + [ + "--layer", + "hil", + "--journey", + "voice", + "--profile", + profile, + "--artifact-dir", + directory, + "--timeout", + "2", + "--device", + str(descriptor), + "--server", + "runner@example.test", + "--server-dir", + "/srv/voicelife", + "--gateway-origin", + "https://gateway.example.test", + "--user-id", + "user-test", + ] + ) + with self.subTest(profile=profile): + self.assertEqual( + type(RUN_E2E.build_adapter(args.layer, args.journey, args)).__name__, "HilVoiceAdapter" + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_voice_linx_serial_multiturn_test.py b/tests/python/test_voice_linx_serial_multiturn_test.py new file mode 100644 index 00000000..460710a9 --- /dev/null +++ b/tests/python/test_voice_linx_serial_multiturn_test.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "voice_linx_serial_multiturn_test.py" +sys.path.insert(0, str(ROOT / "scripts")) + + +def load() -> object: + spec = importlib.util.spec_from_file_location("voice_linx_serial_multiturn_test", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULE = load() + + +class DisplayEvidenceTest(unittest.TestCase): + def test_sparkbot_requires_complete_text_trace_and_observes_scroll(self) -> None: + line = ( + "SPARKBOT_TEXT_RENDER generation=1 revision=2 content_height=16 viewport_height=120 " + "overflow_width=4 manual_line_breaks=0 status=说话 content=你好 content_visible=1" + ) + self.assertEqual(MODULE.display_evidence("sparkbot", [line], [], 0), (True, 1, True)) + + def test_pcb_requires_a_draw_for_every_turn_after_the_turn_starts(self) -> None: + results = [ + MODULE.TurnResult(index=1, input_text="", log_start=1, log_end=3), + MODULE.TurnResult(index=2, input_text="", log_start=3, log_end=5), + ] + lines = ["DISPLAY_DRAW=1 boot", "state", "DISPLAY_DRAW=1 first", "state", "DISPLAY_DRAW=1 second"] + self.assertEqual(MODULE.display_evidence("pcb", lines, results, 2), (True, 2, False)) + self.assertEqual(MODULE.display_evidence("pcb", lines, results, 3), (False, 2, False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_voice_tts_fixture.py b/tests/python/test_voice_tts_fixture.py new file mode 100644 index 00000000..fadb640e --- /dev/null +++ b/tests/python/test_voice_tts_fixture.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import os +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) + +import voice_linx_serial_multiturn_test as MULTITURN # noqa: E402 +import voice_tts_fixture as FIXTURE # noqa: E402 + + +class TtsFixtureTest(unittest.TestCase): + def test_dashscope_returns_sanitized_metrics(self) -> None: + synthesizer = mock.Mock() + synthesizer.call.return_value = b"encoded-audio" + synthesizer.get_first_package_delay.return_value = 123.4 + tts_v2 = types.SimpleNamespace(SpeechSynthesizer=mock.Mock(return_value=synthesizer)) + dashscope = types.SimpleNamespace(api_key=None) + + def import_module(name: str) -> object: + return tts_v2 if name == "dashscope.audio.tts_v2" else dashscope + + with ( + mock.patch.dict(os.environ, {"DASHSCOPE_API_KEY": "secret"}, clear=False), + mock.patch.object(FIXTURE.importlib, "import_module", side_effect=import_module), + ): + result = FIXTURE.synthesize_fixture("你好", "dashscope", "model-a", "voice-a") + self.assertEqual(result.audio, b"encoded-audio") + self.assertEqual(result.provider, "dashscope") + self.assertEqual(result.first_packet_ms, 123) + self.assertNotIn("secret", repr(result)) + + def test_aliyun_nls_collects_raw_pcm_without_ffmpeg(self) -> None: + constructor: dict[str, object] = {} + + class FakeSynthesizer: + def __init__(self, **kwargs: object) -> None: + constructor.update(kwargs) + self.on_data = kwargs["on_data"] + + def start(self, text: str, **kwargs: object) -> bool: + self.on_data(bytes(MULTITURN.PCM_FRAME_BYTES)) + return True + + module = types.SimpleNamespace(NlsSpeechSynthesizer=FakeSynthesizer) + with ( + mock.patch.dict( + os.environ, + {"ALIYUN_NLS_APPKEY": "app", "ALIYUN_NLS_TOKEN": "token"}, + clear=False, + ), + mock.patch.object(FIXTURE.importlib, "import_module", return_value=module), + mock.patch.object(MULTITURN.subprocess, "run") as ffmpeg, + ): + result = FIXTURE.synthesize_fixture("你好", "aliyun-nls", "nls-v1", "xiaoyun") + frames = MULTITURN.to_pcm_frames(result.audio, result.audio_format) + self.assertEqual(len(frames), 1) + self.assertEqual(constructor["url"], "wss://nls-gateway.cn-shanghai.aliyuncs.com/ws/v1") + ffmpeg.assert_not_called() + + def test_missing_provider_credentials_have_stable_codes(self) -> None: + with mock.patch.dict(os.environ, {}, clear=True), self.assertRaises(FIXTURE.TtsFixtureError) as raised: + FIXTURE.validate_provider_environment("aliyun-nls") + self.assertEqual(raised.exception.code, "aliyun_nls_credentials_missing") + + +if __name__ == "__main__": + unittest.main()