diff --git a/.changeset/e2e-integration-roadmap.md b/.changeset/e2e-integration-roadmap.md new file mode 100644 index 0000000..05703a4 --- /dev/null +++ b/.changeset/e2e-integration-roadmap.md @@ -0,0 +1,6 @@ +--- +"@guidekit/core": patch +"@guidekit/react": patch +--- + +Expand E2E integration roadmap: contract/live test tiers, full tool coverage, publish live gate, proxy voice credential resolution, and widget test hooks. diff --git a/.changeset/integration-hardening.md b/.changeset/integration-hardening.md index ea7f28b..ffa3c2a 100644 --- a/.changeset/integration-hardening.md +++ b/.changeset/integration-hardening.md @@ -5,4 +5,4 @@ "@guidekit/cli": patch --- -Harden real-app integration: session token recovery after server restart, voice widget transcript streaming, continuous mic listening, doctor VAD checks, and voice E2E smoke tests. +Harden real-app integration: session token recovery after server restart, voice widget transcript streaming, continuous mic listening, doctor VAD checks, voice E2E smoke tests, and full live-tier Playwright suite (agent tools, platform mode, proxy API, multi-turn). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da8e018..3ef991d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,11 +69,11 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Run E2E tests + - name: Run E2E contract tests env: GUIDEKIT_SECRET: guidekit-example-e2e-secret-32-chars LLM_API_KEY: e2e-dummy-llm-key-for-contract-tests - run: pnpm test:e2e + run: pnpm test:e2e:contract - name: Upload test results if: failure() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b772d07..8346680 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,12 +36,13 @@ jobs: run: pnpm size:check - name: Install Playwright browsers - run: npx playwright install --with-deps + run: npx playwright install --with-deps chromium - - name: E2E tests + - name: E2E contract tests env: GUIDEKIT_SECRET: guidekit-example-e2e-secret-32-chars - run: npx playwright test + LLM_API_KEY: e2e-dummy-llm-key-for-contract-tests + run: pnpm test:e2e:contract - name: Dependency audit run: pnpm audit --prod diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8afe180..6efc30d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,8 +14,46 @@ concurrency: cancel-in-progress: true jobs: + live-e2e: + name: Live E2E (pre-publish) + if: ${{ !inputs.dry_run }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + + - name: Run live E2E suite + env: + LIVE_LLM: '1' + GUIDEKIT_SECRET: guidekit-example-e2e-secret-32-chars + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + SKIP_NPM_DRY_RUN: '1' + run: pnpm check:release + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: publish-live-e2e-report + path: playwright-report/ + retention-days: 14 + publish: name: Publish @guidekit/* to npm + needs: live-e2e + if: ${{ always() && (inputs.dry_run || needs.live-e2e.result == 'success') }} runs-on: ubuntu-latest environment: npm-publish permissions: diff --git a/AGENTS.md b/AGENTS.md index 884332a..c92bbfa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,9 +62,52 @@ guidekit/ ### Testing expectations -- Unit tests: Vitest (`pnpm test:unit`) -- E2E: Playwright (`pnpm test:e2e`) — agent flows in `e2e/` -- New pipeline or cognitive behavior needs unit coverage; integration changes should touch example app or E2E when user-facing +- Unit tests: Vitest (`pnpm test:unit`) — all packages +- Contract E2E: Playwright (`pnpm test:e2e:contract`) — mocked LLM + Web Speech voice, runs on every PR +- Live E2E: Playwright (`pnpm test:e2e:live`) — real Gemini via proxy; **publish gate only** (`LIVE_LLM=1` + `LLM_API_KEY`) +- New pipeline or cognitive behavior needs unit coverage; user-facing integration changes should touch example app or contract E2E + +### E2E layout + +``` +e2e/ +├── contract/ # CI + pnpm check (no API key) +├── live/ # Pre-publish only +├── fixtures/ # LLM mocks, Web Speech mocks, helpers +└── env.ts # .env.local + LIVE_LLM detection +``` + +Voice E2E always mocks the browser Web Speech API — no Deepgram/ElevenLabs in Playwright. + +### E2E coverage matrix (user-facing flows) + +| Flow | Contract | Live | +|------|:--------:|:----:| +| Widget UI / a11y | yes | — | +| Proxy health / token / LLM | yes | yes | +| Text chat + streaming | mocked | yes | +| Multi-turn memory | — | yes | +| Agent tools (scroll, highlight, navigate, tour, clickElement) | yes | yes | +| Platform Mode (RAG, plugin, cognitive page) | yes | yes | +| Session recovery 401 | yes | yes | +| Voice (Web Speech mock → LLM) | yes | yes | +| Custom actions / form / readPage / dismiss | yes | partial | +| STT/TTS proxy key minting | yes | — | +| Hallucination guard bus event | yes | — | +| Vanilla IIFE widget | yes | — | + +Commands: `pnpm test:e2e:contract` (CI), `pnpm test:e2e:live` (local), `pnpm test:e2e:live:full` (publish gate). + +Before release, run `pnpm check:release` (runs live suite twice for the flake budget). Publish workflow uploads Playwright artifacts on failure. + +### Release gate (production readiness) + +Run `pnpm check:release` before publishing. It includes: + +- `pnpm check` (build, typecheck, lint, unit, size, contract E2E) +- Package artifact verification (`scripts/verify-published-packages.mjs`) +- CLI subprocess smoke (`packages/cli/src/cli.smoke.test.ts`) +- Live E2E twice (`pnpm test:e2e:live:full` ×2) ## Commands @@ -74,6 +117,7 @@ pnpm skills:sync # Link skills/guidekit for Codex/Cursor discovery pnpm dev # Start docs + example apps pnpm build # Build all packages pnpm check # Full CI parity (build, typecheck, lint, test) +pnpm check:release # Production publish gate (includes live E2E x2) pnpm publish:packages:dry-run # Dry-run npm publish locally pnpm stats # Package LOC + core facade size pnpm llms:generate # Regenerate llms.txt agent index diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a8f2f0..7be863c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,6 +26,16 @@ pnpm lint # ESLint pnpm test # Unit tests (Vitest) ``` +### Production readiness gate (before publishing) + +Use the full release gate locally (it runs the live E2E suite twice): + +```bash +pnpm check:release +``` + +This requires a real `LLM_API_KEY` (or `apps/example-nextjs/.env.local`) and will skip npm publish dry-run unless `NODE_AUTH_TOKEN`/`NPM_TOKEN` is present. + ## Development Workflow 1. **Fork** the repository and create a feature branch from `main`. diff --git a/apps/example-nextjs/app/about/page.tsx b/apps/example-nextjs/app/(main)/about/page.tsx similarity index 100% rename from apps/example-nextjs/app/about/page.tsx rename to apps/example-nextjs/app/(main)/about/page.tsx diff --git a/apps/example-nextjs/app/(main)/layout.tsx b/apps/example-nextjs/app/(main)/layout.tsx new file mode 100644 index 0000000..ed4bed0 --- /dev/null +++ b/apps/example-nextjs/app/(main)/layout.tsx @@ -0,0 +1,6 @@ +import type { ReactNode } from 'react'; +import { Providers } from '../providers'; + +export default function MainLayout({ children }: { children: ReactNode }) { + return {children}; +} diff --git a/apps/example-nextjs/app/page.tsx b/apps/example-nextjs/app/(main)/page.tsx similarity index 100% rename from apps/example-nextjs/app/page.tsx rename to apps/example-nextjs/app/(main)/page.tsx diff --git a/apps/example-nextjs/app/demo/layout.tsx b/apps/example-nextjs/app/demo/layout.tsx new file mode 100644 index 0000000..2c054e7 --- /dev/null +++ b/apps/example-nextjs/app/demo/layout.tsx @@ -0,0 +1,48 @@ +'use client'; + +import dynamic from 'next/dynamic'; +import type { ReactNode } from 'react'; +import { + platformDemoPlugin, + platformKnowledgeDocuments, +} from '../../lib/guidekit-platform'; +import { GuideKitDemoActions } from '../guidekit-demo-actions'; +import { GuideKitTestBridge } from '../guidekit-test-bridge'; + +const GuideKitProvider = dynamic( + () => import('@guidekit/react').then((mod) => mod.GuideKitProvider), + { ssr: false }, +); + +/** Demo layout with cognitive engine enabled for E2E contract tests. */ +export default function DemoLayout({ children }: { children: ReactNode }) { + return ( + + + + {children} + + ); +} diff --git a/apps/example-nextjs/app/demo/page.tsx b/apps/example-nextjs/app/demo/page.tsx new file mode 100644 index 0000000..db853f9 --- /dev/null +++ b/apps/example-nextjs/app/demo/page.tsx @@ -0,0 +1,28 @@ +export default function DemoPage() { + return ( +
+

GuideKit Cognitive Demo

+

+ This page runs Platform Mode with cognitive={true} for E2E contract + tests. Voice is text-only here for stability. +

+ +
+

Overview

+

Ask the assistant about page sections or request guided navigation.

+
+ +
+

Contact Form

+
+ + + + + +