From 7b8e3048187f6b26212a5bfc8956319f21afc570 Mon Sep 17 00:00:00 2001 From: riaz37 Date: Mon, 15 Jun 2026 14:52:18 +0600 Subject: [PATCH 1/2] feat(e2e): add contract/live tiers and publish live gate This introduces a two-tier E2E setup (contract vs live), adds stable test hooks in the example app, and wires live E2E as a pre-publish gate. --- .changeset/e2e-integration-roadmap.md | 6 + .changeset/integration-hardening.md | 2 +- .github/workflows/ci.yml | 4 +- .github/workflows/nightly.yml | 7 +- .github/workflows/publish.yml | 40 ++++++ AGENTS.md | 39 +++++- .../app/{ => (main)}/about/page.tsx | 0 apps/example-nextjs/app/(main)/layout.tsx | 6 + apps/example-nextjs/app/{ => (main)}/page.tsx | 0 apps/example-nextjs/app/demo/layout.tsx | 48 +++++++ apps/example-nextjs/app/demo/page.tsx | 28 ++++ .../app/guidekit-test-bridge.tsx | 87 +++++++++++++ apps/example-nextjs/app/layout.tsx | 14 +- apps/example-nextjs/app/providers.tsx | 5 + e2e/{ => contract}/accessibility.spec.ts | 0 e2e/{ => contract}/agent-flows.spec.ts | 2 +- e2e/contract/agent-form.spec.ts | 20 +++ e2e/contract/agent-read-page.spec.ts | 27 ++++ e2e/contract/agent-tour.spec.ts | 23 ++++ e2e/contract/cognitive.spec.ts | 20 +++ e2e/contract/contract-platform.spec.ts | 37 ++++++ e2e/contract/custom-actions.spec.ts | 32 +++++ e2e/contract/hallucination-guard.spec.ts | 26 ++++ e2e/contract/highlight-dismiss.spec.ts | 33 +++++ e2e/contract/knowledge-retrieval.spec.ts | 23 ++++ e2e/{ => contract}/navigation.spec.ts | 0 e2e/contract/proxy-voice-keys.spec.ts | 39 ++++++ e2e/contract/session-recovery.spec.ts | 49 +++++++ e2e/{ => contract}/voice.spec.ts | 12 +- e2e/{ => contract}/widget.spec.ts | 0 e2e/env.ts | 49 +++++++ e2e/fixtures/live-helpers.ts | 85 ++++++++++++ e2e/fixtures/mock-llm-proxy.ts | 45 ++++++- e2e/fixtures/voice-e2e.ts | 19 +++ e2e/fixtures/voice-mocks.ts | 1 + e2e/live-llm.spec.ts | 36 ------ e2e/live/agent-tools.spec.ts | 52 ++++++++ e2e/live/custom-actions.spec.ts | 28 ++++ e2e/live/form.spec.ts | 28 ++++ e2e/live/llm.spec.ts | 54 ++++++++ e2e/live/platform.spec.ts | 50 ++++++++ e2e/live/proxy.spec.ts | 103 +++++++++++++++ e2e/live/voice.spec.ts | 42 ++++++ package.json | 6 +- packages/core/src/core/runtime-init.ts | 28 +++- packages/core/src/core/voice-config.test.ts | 75 +++++++++++ packages/core/src/core/voice-config.ts | 121 ++++++++++++++++++ packages/core/src/llm/proxy-adapter.ts | 6 +- packages/core/src/types/index.ts | 8 +- packages/react/src/widget/index.tsx | 17 ++- playwright.config.ts | 59 ++++----- scripts/ci-check.sh | 4 +- 52 files changed, 1434 insertions(+), 111 deletions(-) create mode 100644 .changeset/e2e-integration-roadmap.md rename apps/example-nextjs/app/{ => (main)}/about/page.tsx (100%) create mode 100644 apps/example-nextjs/app/(main)/layout.tsx rename apps/example-nextjs/app/{ => (main)}/page.tsx (100%) create mode 100644 apps/example-nextjs/app/demo/layout.tsx create mode 100644 apps/example-nextjs/app/demo/page.tsx create mode 100644 apps/example-nextjs/app/guidekit-test-bridge.tsx rename e2e/{ => contract}/accessibility.spec.ts (100%) rename e2e/{ => contract}/agent-flows.spec.ts (99%) create mode 100644 e2e/contract/agent-form.spec.ts create mode 100644 e2e/contract/agent-read-page.spec.ts create mode 100644 e2e/contract/agent-tour.spec.ts create mode 100644 e2e/contract/cognitive.spec.ts create mode 100644 e2e/contract/contract-platform.spec.ts create mode 100644 e2e/contract/custom-actions.spec.ts create mode 100644 e2e/contract/hallucination-guard.spec.ts create mode 100644 e2e/contract/highlight-dismiss.spec.ts create mode 100644 e2e/contract/knowledge-retrieval.spec.ts rename e2e/{ => contract}/navigation.spec.ts (100%) create mode 100644 e2e/contract/proxy-voice-keys.spec.ts create mode 100644 e2e/contract/session-recovery.spec.ts rename e2e/{ => contract}/voice.spec.ts (81%) rename e2e/{ => contract}/widget.spec.ts (100%) create mode 100644 e2e/env.ts create mode 100644 e2e/fixtures/live-helpers.ts create mode 100644 e2e/fixtures/voice-e2e.ts delete mode 100644 e2e/live-llm.spec.ts create mode 100644 e2e/live/agent-tools.spec.ts create mode 100644 e2e/live/custom-actions.spec.ts create mode 100644 e2e/live/form.spec.ts create mode 100644 e2e/live/llm.spec.ts create mode 100644 e2e/live/platform.spec.ts create mode 100644 e2e/live/proxy.spec.ts create mode 100644 e2e/live/voice.spec.ts create mode 100644 packages/core/src/core/voice-config.test.ts create mode 100644 packages/core/src/core/voice-config.ts diff --git a/.changeset/e2e-integration-roadmap.md b/.changeset/e2e-integration-roadmap.md new file mode 100644 index 0000000..05703a4 --- /dev/null +++ b/.changeset/e2e-integration-roadmap.md @@ -0,0 +1,6 @@ +--- +"@guidekit/core": patch +"@guidekit/react": patch +--- + +Expand E2E integration roadmap: contract/live test tiers, full tool coverage, publish live gate, proxy voice credential resolution, and widget test hooks. diff --git a/.changeset/integration-hardening.md b/.changeset/integration-hardening.md index ea7f28b..ffa3c2a 100644 --- a/.changeset/integration-hardening.md +++ b/.changeset/integration-hardening.md @@ -5,4 +5,4 @@ "@guidekit/cli": patch --- -Harden real-app integration: session token recovery after server restart, voice widget transcript streaming, continuous mic listening, doctor VAD checks, and voice E2E smoke tests. +Harden real-app integration: session token recovery after server restart, voice widget transcript streaming, continuous mic listening, doctor VAD checks, voice E2E smoke tests, and full live-tier Playwright suite (agent tools, platform mode, proxy API, multi-turn). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da8e018..3ef991d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,11 +69,11 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Run E2E tests + - name: Run E2E contract tests env: GUIDEKIT_SECRET: guidekit-example-e2e-secret-32-chars LLM_API_KEY: e2e-dummy-llm-key-for-contract-tests - run: pnpm test:e2e + run: pnpm test:e2e:contract - name: Upload test results if: failure() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b772d07..8346680 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,12 +36,13 @@ jobs: run: pnpm size:check - name: Install Playwright browsers - run: npx playwright install --with-deps + run: npx playwright install --with-deps chromium - - name: E2E tests + - name: E2E contract tests env: GUIDEKIT_SECRET: guidekit-example-e2e-secret-32-chars - run: npx playwright test + LLM_API_KEY: e2e-dummy-llm-key-for-contract-tests + run: pnpm test:e2e:contract - name: Dependency audit run: pnpm audit --prod diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8afe180..fcd76ef 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,8 +14,48 @@ concurrency: cancel-in-progress: true jobs: + live-e2e: + name: Live E2E (pre-publish) + if: ${{ !inputs.dry_run }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build all packages + run: pnpm build + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + + - name: Run live E2E suite + env: + LIVE_LLM: '1' + GUIDEKIT_SECRET: guidekit-example-e2e-secret-32-chars + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + run: pnpm test:e2e:live:full + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: publish-live-e2e-report + path: playwright-report/ + retention-days: 14 + publish: name: Publish @guidekit/* to npm + needs: live-e2e + if: ${{ always() && (inputs.dry_run || needs.live-e2e.result == 'success') }} runs-on: ubuntu-latest environment: npm-publish permissions: diff --git a/AGENTS.md b/AGENTS.md index 884332a..d300f7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,9 +62,42 @@ guidekit/ ### Testing expectations -- Unit tests: Vitest (`pnpm test:unit`) -- E2E: Playwright (`pnpm test:e2e`) — agent flows in `e2e/` -- New pipeline or cognitive behavior needs unit coverage; integration changes should touch example app or E2E when user-facing +- Unit tests: Vitest (`pnpm test:unit`) — all packages +- Contract E2E: Playwright (`pnpm test:e2e:contract`) — mocked LLM + Web Speech voice, runs on every PR +- Live E2E: Playwright (`pnpm test:e2e:live`) — real Gemini via proxy; **publish gate only** (`LIVE_LLM=1` + `LLM_API_KEY`) +- New pipeline or cognitive behavior needs unit coverage; user-facing integration changes should touch example app or contract E2E + +### E2E layout + +``` +e2e/ +├── contract/ # CI + pnpm check (no API key) +├── live/ # Pre-publish only +├── fixtures/ # LLM mocks, Web Speech mocks, helpers +└── env.ts # .env.local + LIVE_LLM detection +``` + +Voice E2E always mocks the browser Web Speech API — no Deepgram/ElevenLabs in Playwright. + +### E2E coverage matrix (user-facing flows) + +| Flow | Contract | Live | +|------|:--------:|:----:| +| Widget UI / a11y | yes | — | +| Proxy health / token / LLM | yes | yes | +| Text chat + streaming | mocked | yes | +| Multi-turn memory | — | yes | +| Agent tools (scroll, highlight, navigate, tour) | yes | yes | +| Platform Mode (RAG, plugin, cognitive page) | yes | yes | +| Session recovery 401 | yes | yes | +| Voice (Web Speech mock → LLM) | yes | yes | +| Custom actions / form / readPage / dismiss | yes | partial | +| STT/TTS proxy key minting | yes | — | +| Hallucination guard bus event | yes | — | + +Commands: `pnpm test:e2e:contract` (CI), `pnpm test:e2e:live` (local), `pnpm test:e2e:live:full` (publish gate). + +Before release, run `pnpm test:e2e:live:full` twice locally and confirm both pass (flake budget). Publish workflow uploads Playwright artifacts on live E2E failure. ## Commands diff --git a/apps/example-nextjs/app/about/page.tsx b/apps/example-nextjs/app/(main)/about/page.tsx similarity index 100% rename from apps/example-nextjs/app/about/page.tsx rename to apps/example-nextjs/app/(main)/about/page.tsx diff --git a/apps/example-nextjs/app/(main)/layout.tsx b/apps/example-nextjs/app/(main)/layout.tsx new file mode 100644 index 0000000..ed4bed0 --- /dev/null +++ b/apps/example-nextjs/app/(main)/layout.tsx @@ -0,0 +1,6 @@ +import type { ReactNode } from 'react'; +import { Providers } from '../providers'; + +export default function MainLayout({ children }: { children: ReactNode }) { + return {children}; +} diff --git a/apps/example-nextjs/app/page.tsx b/apps/example-nextjs/app/(main)/page.tsx similarity index 100% rename from apps/example-nextjs/app/page.tsx rename to apps/example-nextjs/app/(main)/page.tsx diff --git a/apps/example-nextjs/app/demo/layout.tsx b/apps/example-nextjs/app/demo/layout.tsx new file mode 100644 index 0000000..2c054e7 --- /dev/null +++ b/apps/example-nextjs/app/demo/layout.tsx @@ -0,0 +1,48 @@ +'use client'; + +import dynamic from 'next/dynamic'; +import type { ReactNode } from 'react'; +import { + platformDemoPlugin, + platformKnowledgeDocuments, +} from '../../lib/guidekit-platform'; +import { GuideKitDemoActions } from '../guidekit-demo-actions'; +import { GuideKitTestBridge } from '../guidekit-test-bridge'; + +const GuideKitProvider = dynamic( + () => import('@guidekit/react').then((mod) => mod.GuideKitProvider), + { ssr: false }, +); + +/** Demo layout with cognitive engine enabled for E2E contract tests. */ +export default function DemoLayout({ children }: { children: ReactNode }) { + return ( + + + + {children} + + ); +} diff --git a/apps/example-nextjs/app/demo/page.tsx b/apps/example-nextjs/app/demo/page.tsx new file mode 100644 index 0000000..db853f9 --- /dev/null +++ b/apps/example-nextjs/app/demo/page.tsx @@ -0,0 +1,28 @@ +export default function DemoPage() { + return ( +
+

GuideKit Cognitive Demo

+

+ This page runs Platform Mode with cognitive={true} for E2E contract + tests. Voice is text-only here for stability. +

+ +
+

Overview

+

Ask the assistant about page sections or request guided navigation.

+
+ +
+

Contact Form

+
+ + + + + +