From e504e7687cbfb5b7d7c4d5637a7e8f92d495cb10 Mon Sep 17 00:00:00 2001 From: riaz37 Date: Tue, 16 Jun 2026 17:27:08 +0600 Subject: [PATCH] feat: production hardening for telemetry, proxy security, and ops docs Ship stable pipeline telemetry with DevTools visibility, tighten server proxy auth boundaries, and document rate limits, failure modes, and Redis session storage for production operators. --- .changeset/production-hardening.md | 7 ++ apps/docs/app/docs/_meta.ts | 1 + apps/docs/app/docs/architecture/page.mdx | 17 +++- apps/docs/app/docs/devtools/page.mdx | 12 ++- apps/docs/app/docs/observability/page.mdx | 73 ++++++++++++++ apps/docs/app/docs/server/page.mdx | 77 ++++++++++++++- apps/docs/app/docs/troubleshooting/page.mdx | 63 ++++++++++++ apps/example-nextjs/lib/guidekit-routes.ts | 17 +++- .../lib/guidekit-session-store.ts | 33 +++++++ apps/example-nextjs/next.config.mjs | 2 + e2e/contract/custom-actions.spec.ts | 2 +- e2e/contract/proxy-security.spec.ts | 69 +++++++++++++ packages/core/src/index.ts | 2 +- packages/core/src/pipeline/orchestrator.ts | 17 +++- packages/core/src/telemetry/index.ts | 63 ++++++++++-- packages/react/README.md | 2 + packages/react/src/devtools.tsx | 97 ++++++++++++++++++- packages/server/src/auth.ts | 27 +++++- packages/server/src/proxy/llm.test.ts | 92 ++++++++++++++++++ packages/server/src/proxy/llm.ts | 40 +++++++- packages/server/src/proxy/voice.test.ts | 67 +++++++++++++ packages/server/src/proxy/voice.ts | 25 ++++- scripts/ci-check.sh | 4 + scripts/release-check.sh | 3 + 24 files changed, 784 insertions(+), 28 deletions(-) create mode 100644 .changeset/production-hardening.md create mode 100644 apps/docs/app/docs/observability/page.mdx create mode 100644 apps/example-nextjs/lib/guidekit-session-store.ts create mode 100644 e2e/contract/proxy-security.spec.ts create mode 100644 packages/server/src/proxy/voice.test.ts diff --git a/.changeset/production-hardening.md b/.changeset/production-hardening.md new file mode 100644 index 0000000..197fa1f --- /dev/null +++ b/.changeset/production-hardening.md @@ -0,0 +1,7 @@ +--- +"@guidekit/core": patch +"@guidekit/react": patch +"@guidekit/server": patch +--- + +Production hardening: stable pipeline telemetry export and DevTools Telemetry tab; LLM/voice proxy permission and origin checks with request validation; example app Redis session-store path and operational docs (rate limits, failure modes, observability). diff --git a/apps/docs/app/docs/_meta.ts b/apps/docs/app/docs/_meta.ts index 6dcaed5..d70d789 100644 --- a/apps/docs/app/docs/_meta.ts +++ b/apps/docs/app/docs/_meta.ts @@ -10,6 +10,7 @@ const meta: MetaRecord = { 'proactive-triggers': 'Proactive Triggers', 'platform-mode': 'Platform Mode', privacy: 'Privacy & Security', + observability: 'Observability', server: 'Server SDK', compatibility: 'Compatibility Tiers', troubleshooting: 'Troubleshooting', diff --git a/apps/docs/app/docs/architecture/page.mdx b/apps/docs/app/docs/architecture/page.mdx index 34dd204..79140c1 100644 --- a/apps/docs/app/docs/architecture/page.mdx +++ b/apps/docs/app/docs/architecture/page.mdx @@ -94,9 +94,18 @@ Client → JWT to server middleware → Server looks up provider keys → Proxy ``` - JWT tokens do **NOT** contain API keys (base64-decodable) -- Provider keys stay server-side in an in-memory session store +- Provider keys stay server-side in a `SessionStore` (in-memory for dev; **Redis for multi-instance production** — see [Server SDK](/docs/server#session-storage)) - Tokens refresh at 80% of TTL, multi-tab coordination via BroadcastChannel - Signing secret rotation: accepts array `[newSecret, oldSecret]` +- Proxy routes enforce **permissions** (`llm` / `stt` / `tts`) and optional **`allowedOrigins`** (JWT `aud` → `Origin` header) + +### Production security checklist + +1. **Proxy mode only** — never ship provider API keys to the browser ([Privacy & Security](/docs/privacy)). +2. **Session store** — `RedisSessionStore` when running more than one instance ([compatibility](/docs/compatibility#multi-instance-deployments)). +3. **Rate limits** — tune `createGuideKitHandler` `rateLimit` ([Server SDK](/docs/server#rate-limiting)). +4. **Origin allowlist** — set `allowedOrigins` when minting tokens in production. +5. **Observability** — use pipeline telemetry to debug latency and token-heavy turns ([Observability](/docs/observability)). ### Privacy @@ -133,6 +142,12 @@ class GuideKitError extends Error { Error types: `AuthenticationError`, `ConfigurationError`, `NetworkError`, `TimeoutError`, `RateLimitError`, `PermissionError`, `BrowserSupportError`, `ContentFilterError`, `ResourceExhaustedError`, `InitializationError`. +See [Troubleshooting](/docs/troubleshooting#recoverable-vs-non-recoverable-errors) for how `recoverable` maps to user-facing retry behavior. + +## Observability + +Each user message records per-stage pipeline spans (latency, token metadata). Use [Observability](/docs/observability) and the DevTools **Telemetry** tab during development. + ## EventBus Typed pub/sub with namespace support: diff --git a/apps/docs/app/docs/devtools/page.mdx b/apps/docs/app/docs/devtools/page.mdx index b1d1fdf..a1f9114 100644 --- a/apps/docs/app/docs/devtools/page.mdx +++ b/apps/docs/app/docs/devtools/page.mdx @@ -21,7 +21,7 @@ function App() { ## Features -The DevTools panel has four tabs: +The DevTools panel has five tabs: ### State Tab @@ -59,6 +59,16 @@ Shows current rate limiter state: - TTS characters: current / limit per session - Visual meters showing usage +### Telemetry Tab + +Shows per-message pipeline spans (stage timings and metadata) recorded by `@guidekit/core`: + +- Stage spans: `scan`, `enrich`, `retrieve`, `context`, `cognize`, `llm`, `validate`, `render` +- Duration per span (`durationMs`) +- Attributes (like `conversationId`, `totalTokens`, `toolCallsExecuted`, `rounds`) + +Use **Copy JSON** to paste spans into an issue, or to correlate client timings with server logs. + ## Appearance The DevTools panel: diff --git a/apps/docs/app/docs/observability/page.mdx b/apps/docs/app/docs/observability/page.mdx new file mode 100644 index 0000000..ad94c82 --- /dev/null +++ b/apps/docs/app/docs/observability/page.mdx @@ -0,0 +1,73 @@ +# Observability + +GuideKit ships **client-side pipeline telemetry** so you can understand: + +- **Latency**: which stages are slow (DOM scan vs LLM vs validation) +- **Cost signals**: token usage metadata attached to the LLM stage +- **Failures**: correlate stage timing with `EventBus` errors and server proxy logs + +## What gets recorded + +Each user message runs the v2 pipeline stages: + +`scan → enrich → retrieve → context → cognize → llm → validate → render` + +For each stage, GuideKit records one span with: + +- `name`: stable span name (example: `guidekit.pipeline.llm`) +- `stage`: pipeline stage id +- `startTimeEpochMs` / `endTimeEpochMs`: wall-clock timestamps for log correlation +- `durationMs`: monotonic duration in milliseconds +- `attributes`: stage metadata (primitives only) + +## Where to see telemetry + +### DevTools (recommended) + +In development, enable the DevTools panel and open the **Telemetry** tab: + +```tsx +import { GuideKitDevTools } from '@guidekit/react/devtools'; + +{process.env.NODE_ENV === 'development' && } +``` + +The Telemetry tab shows stage timings and provides **Copy JSON** for sharing or analysis. + +### Programmatic export + +If you own a custom UI, you can read the last message spans from the core instance: + +```ts +const spans = core.getTelemetrySpans(); +``` + +This returns an array of exported spans with epoch timestamps and attributes. + +## Attribute contract (stable keys) + +GuideKit guarantees these attribute keys when applicable: + +- **`stage`**: pipeline stage id +- **`conversationId`**: unique per user message +- **`totalTokens`**: total tokens used for the completed message (attached as stages complete) +- **`toolCallsExecuted`**: number of executed tool calls for the message +- **`rounds`**: LLM/tool rounds performed + +Additional attributes may be added over time, but existing keys are stable. + +## Debugging workflow + +1. Reproduce the slow or failed message in dev with `options={{ debug: true }}`. +2. Open **GuideKit DevTools → Telemetry** and note the slowest `stage` (`llm` vs `scan` vs `validate`). +3. Check **Events** for `error`, `auth:token-refreshed`, or `validation:complete`. +4. Correlate `conversationId` and `startTimeEpochMs` with server proxy logs. + +For proxy/auth failures (401, 403, 429), see [Troubleshooting](/docs/troubleshooting#production--proxy-failures). + +## See also + +- [Architecture](/docs/architecture) — security model and production checklist +- [Server SDK](/docs/server) — rate limits, session store, origin allowlist +- [DevTools](/docs/devtools) — Telemetry tab reference + diff --git a/apps/docs/app/docs/server/page.mdx b/apps/docs/app/docs/server/page.mdx index fdc5c51..6db0dc1 100644 --- a/apps/docs/app/docs/server/page.mdx +++ b/apps/docs/app/docs/server/page.mdx @@ -57,6 +57,35 @@ const keys = await getSessionKeys('my-session-id'); import { RedisSessionStore } from '@guidekit/server/redis'; ``` +### In-memory vs Redis (production guidance) + +- **InMemorySessionStore**: good for local dev and single-instance deployments. + - Not suitable for **multi-instance** or **serverless** deployments where requests can land on different processes. +- **RedisSessionStore**: recommended for production when you run multiple instances or need durability across restarts. + +The reference example app (`apps/example-nextjs`) uses in-memory storage by default. Set `REDIS_URL` and install `ioredis` in that app to exercise the Redis path locally. + +Example: + +```typescript +import Redis from 'ioredis'; +import { RedisSessionStore } from '@guidekit/server/redis'; +import { createNextAppRouterRoutes } from '@guidekit/server/next'; + +const redis = new Redis(process.env.REDIS_URL!); +const sessionStore = new RedisSessionStore({ redis }); + +const routes = createNextAppRouterRoutes({ + signingSecret: process.env.GUIDEKIT_SECRET!, + sessionStore, + createTokenOptions: () => ({ + llmApiKey: process.env.LLM_API_KEY!, + expiresIn: '15m', + allowedOrigins: ['https://yourapp.com'], + }), +}); +``` + ## Framework-agnostic handler For non-Next.js runtimes, use `createGuideKitHandler`: @@ -92,7 +121,53 @@ All proxy routes require `Authorization: Bearer ` except `/token` ## Rate limiting -`createGuideKitHandler` applies a sliding-window rate limiter per session ID and IP (default: 60 req/min). +`createGuideKitHandler` applies a **sliding-window** rate limiter keyed by **session ID** (when authenticated) or **client IP**. + +### Defaults + +| Setting | Default | Meaning | +|---------|---------|---------| +| `windowMs` | `60_000` | 1-minute window | +| `maxRequests` | `60` | Max requests per window per key | + +When exceeded, the handler returns **429** with a `Retry-After` header (seconds). + +### Production recommendations + +| Deployment | Suggested `maxRequests` | Notes | +|------------|-------------------------|-------| +| Single-tenant internal app | `60` (default) | Fine for most dashboards | +| Public consumer app | `30–40` | Tighten if you see abuse | +| High-traffic B2B | `80–120` | Monitor 429 rates; add CDN/WAF in front | + +Tune via handler options: + +```typescript +const handler = createGuideKitHandler({ + signingSecret: process.env.GUIDEKIT_SECRET!, + rateLimit: { + windowMs: 60_000, + maxRequests: 40, + }, + createTokenOptions: () => ({ llmApiKey: process.env.LLM_API_KEY!, expiresIn: '15m' }), +}); +``` + +The example app reads `GUIDEKIT_RATE_LIMIT_WINDOW_MS` and `GUIDEKIT_RATE_LIMIT_MAX` for local tuning. + +### Origin allowlist (recommended in production) + +Set `allowedOrigins` when minting tokens. Proxy routes (`/llm`, `/stt`, `/tts`) reject requests when the token includes an `aud` claim and the `Origin` header does not match. + +```typescript +createTokenOptions: () => ({ + llmApiKey: process.env.LLM_API_KEY!, + expiresIn: '15m', + allowedOrigins: ['https://yourapp.com'], +}), +``` + +In the example app, set `GUIDEKIT_ALLOWED_ORIGINS=https://yourapp.com,https://staging.yourapp.com`. ## `createSessionToken` diff --git a/apps/docs/app/docs/troubleshooting/page.mdx b/apps/docs/app/docs/troubleshooting/page.mdx index 0176e60..be4635b 100644 --- a/apps/docs/app/docs/troubleshooting/page.mdx +++ b/apps/docs/app/docs/troubleshooting/page.mdx @@ -131,6 +131,69 @@ import { createGuideKit } from '@guidekit/core'; Visual/rendering exports live under `@guidekit/core/rendering` to keep the vanilla bundle smaller. +## Production / Proxy Failures + +### Session expired or server restarted (401) + +**Symptom:** LLM proxy returns `401` with *"Session expired or server restarted — request a new token"*. + +**Cause:** Provider keys live in the server `SessionStore`, keyed by `sessionId`. After a deploy/restart (in-memory store) or TTL expiry, the JWT may still decode but keys are gone. + +**Fix:** +1. Ensure the client refreshes tokens (GuideKit refreshes at ~80% TTL automatically). +2. For multi-instance/serverless, use [`RedisSessionStore`](/docs/server#in-memory-vs-redis-production-guidance). +3. See [Session recovery](/docs/troubleshooting#session-recovery-client-side) below. + +### Rate limit exceeded (429) + +**Symptom:** Proxy returns `429` with `Retry-After`. + +**Fix:** Back off and retry after the header value. Tune `rateLimit.maxRequests` on `createGuideKitHandler` — see [Server SDK rate limiting](/docs/server#rate-limiting). + +### Origin not allowed (403) + +**Symptom:** Proxy returns `403` with *"Origin not allowed"*. + +**Fix:** Ensure `allowedOrigins` on `createSessionToken` includes your app's origin (scheme + host + port). Browser `fetch` from the app must send a matching `Origin` header. + +### Permission denied (403) + +**Symptom:** STT/TTS/LLM proxy returns `403` with *Permission "llm" not granted* (or `stt` / `tts`). + +**Fix:** Mint tokens with the required `permissions` array, e.g. `['stt', 'tts', 'llm']` (default). + +## Recoverable vs non-recoverable errors + +Every `GuideKitError` includes `recoverable`: + +| `recoverable` | Meaning | Examples | +|---------------|---------|----------| +| `true` | Safe to retry or continue | `SEND_IN_FLIGHT`, `INPUT_TOO_LONG`, rate limits, network blips | +| `false` | Fix configuration or auth | Missing LLM config, invalid provider, fatal init errors | + +**Client pattern:** + +```tsx +core.bus.on('error', (err) => { + if (err.recoverable) { + // show toast, allow retry + return; + } + // log to monitoring, show support message +}); +``` + +Use [Observability](/docs/observability) (DevTools Telemetry tab or `core.getTelemetrySpans()`) to see which pipeline stage failed before the error surfaced. + +### Session recovery (client-side) + +When the LLM proxy returns **401**, the SDK should fetch a new token and retry. Contract coverage: `e2e/contract/session-recovery.spec.ts`. + +If recovery loops: +1. Confirm `/api/guidekit/token` returns 200 and `LLM_API_KEY` is set server-side. +2. Confirm all app instances share the same `SessionStore` (Redis in production). +3. Enable `options={{ debug: true }}` and watch `auth:token-refreshed` events. + ## Error Codes GuideKit uses structured error codes for all failures. Each `GuideKitError` includes: diff --git a/apps/example-nextjs/lib/guidekit-routes.ts b/apps/example-nextjs/lib/guidekit-routes.ts index b9d53ed..543c6a2 100644 --- a/apps/example-nextjs/lib/guidekit-routes.ts +++ b/apps/example-nextjs/lib/guidekit-routes.ts @@ -1,12 +1,25 @@ -import { createNextAppRouterRoutes, getSharedSessionStore } from '@guidekit/server/next'; +import { createNextAppRouterRoutes } from '@guidekit/server/next'; +import { guidekitSessionStore } from './guidekit-session-store'; + +function parseAllowedOrigins(): string[] | undefined { + const raw = process.env.GUIDEKIT_ALLOWED_ORIGINS; + if (!raw) return undefined; + const origins = raw.split(',').map((s) => s.trim()).filter(Boolean); + return origins.length > 0 ? origins : undefined; +} export const guidekitRoutes = createNextAppRouterRoutes({ signingSecret: process.env.GUIDEKIT_SECRET!, - sessionStore: getSharedSessionStore(), + sessionStore: guidekitSessionStore, + rateLimit: { + windowMs: Number(process.env.GUIDEKIT_RATE_LIMIT_WINDOW_MS ?? 60_000), + maxRequests: Number(process.env.GUIDEKIT_RATE_LIMIT_MAX ?? 60), + }, createTokenOptions: () => ({ llmApiKey: process.env.LLM_API_KEY!, sttApiKey: process.env.STT_API_KEY, ttsApiKey: process.env.TTS_API_KEY, expiresIn: '15m', + allowedOrigins: parseAllowedOrigins(), }), }); diff --git a/apps/example-nextjs/lib/guidekit-session-store.ts b/apps/example-nextjs/lib/guidekit-session-store.ts new file mode 100644 index 0000000..4e6b22d --- /dev/null +++ b/apps/example-nextjs/lib/guidekit-session-store.ts @@ -0,0 +1,33 @@ +import { createRequire } from 'node:module'; +import type { SessionStore } from '@guidekit/server'; +import { getSharedSessionStore } from '@guidekit/server/next'; +import { RedisSessionStore, type RedisLike } from '@guidekit/server/redis'; + +const require = createRequire(import.meta.url); + +/** + * Process-wide session store for the example app. + * + * - Default: in-memory (single instance, local dev) + * - Production: set REDIS_URL and install `ioredis` for horizontal scaling + */ +function buildSessionStore(): SessionStore { + const redisUrl = process.env.REDIS_URL; + if (!redisUrl) { + return getSharedSessionStore(); + } + + try { + // Optional peer — webpack must not statically resolve unless installed. + const redisPkg = ['iored', 'is'].join(''); + const Redis = require(redisPkg) as new (url: string) => RedisLike; + return new RedisSessionStore({ redis: new Redis(redisUrl) }); + } catch { + console.warn( + '[GuideKit example] REDIS_URL is set but ioredis is not installed. Falling back to in-memory session store.', + ); + return getSharedSessionStore(); + } +} + +export const guidekitSessionStore: SessionStore = buildSessionStore(); diff --git a/apps/example-nextjs/next.config.mjs b/apps/example-nextjs/next.config.mjs index 8f73668..57dbc44 100644 --- a/apps/example-nextjs/next.config.mjs +++ b/apps/example-nextjs/next.config.mjs @@ -2,6 +2,8 @@ const nextConfig = { // Transpile workspace packages so Next.js can process their TypeScript/ESM transpilePackages: ['@guidekit/core', '@guidekit/react', '@guidekit/server', '@guidekit/vad'], + // Optional when REDIS_URL is set (see lib/guidekit-session-store.ts) + serverExternalPackages: ['ioredis'], // Disable React Strict Mode to avoid double-mount issues with Shadow DOM // (attachShadow can only be called once per element) reactStrictMode: false, diff --git a/e2e/contract/custom-actions.spec.ts b/e2e/contract/custom-actions.spec.ts index 9d71282..cb4bef9 100644 --- a/e2e/contract/custom-actions.spec.ts +++ b/e2e/contract/custom-actions.spec.ts @@ -23,7 +23,7 @@ test.describe('Custom actions', () => { const input = await openWidgetInput(page); await input.fill('Show alert'); - await page.getByTestId('guidekit-send').click(); + await input.press('Enter'); await expect(page.locator('.gk-message[data-role="assistant"]').last()).not.toHaveText('', { timeout: 20_000, diff --git a/e2e/contract/proxy-security.spec.ts b/e2e/contract/proxy-security.spec.ts new file mode 100644 index 0000000..4ee9974 --- /dev/null +++ b/e2e/contract/proxy-security.spec.ts @@ -0,0 +1,69 @@ +import { test, expect } from '@playwright/test'; + +/** + * Contract tests for proxy security boundaries (no live LLM required). + */ +test.describe('Proxy security boundaries', () => { + test('LLM proxy rejects missing Authorization', async ({ request }) => { + const res = await request.post('/api/guidekit/llm', { + data: { + provider: 'gemini', + systemPrompt: 'test', + contents: [], + userMessage: 'hello', + }, + }); + expect(res.status()).toBe(401); + const body = await res.json(); + expect(body.error).toMatch(/Authorization|token/i); + }); + + test('LLM proxy rejects invalid JSON body', async ({ request }) => { + const tokenRes = await request.post('/api/guidekit/token'); + expect(tokenRes.ok()).toBeTruthy(); + const { token } = (await tokenRes.json()) as { token: string }; + + const res = await request.post('/api/guidekit/llm', { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + data: 'not-json', + }); + expect(res.status()).toBe(400); + }); + + test('LLM proxy rejects empty systemPrompt', async ({ request }) => { + const tokenRes = await request.post('/api/guidekit/token'); + const { token } = (await tokenRes.json()) as { token: string }; + + const res = await request.post('/api/guidekit/llm', { + headers: { Authorization: `Bearer ${token}` }, + data: { + provider: 'gemini', + systemPrompt: '', + contents: [], + }, + }); + expect(res.status()).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/systemPrompt/i); + }); + + test('STT proxy rejects missing Authorization', async ({ request }) => { + const res = await request.post('/api/guidekit/stt'); + expect(res.status()).toBe(401); + }); + + test('invalid bearer token is rejected on LLM proxy', async ({ request }) => { + const res = await request.post('/api/guidekit/llm', { + headers: { Authorization: 'Bearer not-a-valid-jwt' }, + data: { + provider: 'gemini', + systemPrompt: 'test', + contents: [], + }, + }); + expect(res.status()).toBe(401); + }); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 542c197..9d17b43 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -37,7 +37,7 @@ export type { // Telemetry (optional subpath) export { PipelineTelemetry } from './telemetry/index.js'; -export type { TelemetrySpan } from './telemetry/index.js'; +export type { TelemetrySpan, TelemetrySpanExport } from './telemetry/index.js'; // Context engine export { TokenBudgetManager, heuristicCount } from './context/token-budget.js'; diff --git a/packages/core/src/pipeline/orchestrator.ts b/packages/core/src/pipeline/orchestrator.ts index dce9f37..66a5a59 100644 --- a/packages/core/src/pipeline/orchestrator.ts +++ b/packages/core/src/pipeline/orchestrator.ts @@ -112,6 +112,7 @@ export class PipelineOrchestrator { let ctx = createInitialContext(message); try { + deps.telemetry?.clear(); deps.setStreaming(true, ''); deps.notifyListeners(); deps.setAgentState({ status: 'processing', transcript: ctx.userMessage }); @@ -123,7 +124,9 @@ export class PipelineOrchestrator { }); for (const stage of PIPELINE_STAGES) { - const span = deps.telemetry?.startSpan(stage); + const span = deps.telemetry?.startSpan(stage, { + conversationId: ctx.conversationId, + }); if (stage === 'llm') { for await (const chunk of streamLLMChunks(ctx, deps, llmOrchestrator)) { @@ -151,18 +154,24 @@ export class PipelineOrchestrator { }; } if (span) { - span.attributes = { - ...span.attributes, + deps.telemetry?.setAttributes(span, { totalTokens: ctx.totalTokens, toolCallsExecuted: ctx.toolCallsExecuted, rounds: ctx.rounds, - }; + }); deps.telemetry?.endSpan(span); } continue; } ctx = await runStage(stage, ctx, deps); + if (span) { + deps.telemetry?.setAttributes(span, { + totalTokens: ctx.totalTokens, + toolCallsExecuted: ctx.toolCallsExecuted, + rounds: ctx.rounds, + }); + } if (stage === 'validate') { deps.bus.emit('validation:complete', { confidence: ctx.validation?.confidence, diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index a6b784e..4fc3f4c 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -9,13 +9,40 @@ import type { PipelineStage } from '../pipeline/types.js'; export interface TelemetrySpan { name: string; stage: PipelineStage; + /** + * Monotonic timestamps from `performance.now()` (ms). + * Use these for reliable duration calculations. + */ startTime: number; endTime?: number; attributes?: Record; } +export interface TelemetrySpanExport { + /** Stable span name. */ + name: string; + /** Pipeline stage this span corresponds to. */ + stage: PipelineStage; + /** Epoch wall-clock start time (ms since Unix epoch). */ + startTimeEpochMs: number; + /** Epoch wall-clock end time (ms since Unix epoch). */ + endTimeEpochMs?: number; + /** Duration in milliseconds (monotonic). */ + durationMs?: number; + /** Arbitrary span attributes (must be JSON-serializable primitives). */ + attributes: Record; +} + export class PipelineTelemetry { private spans: TelemetrySpan[] = []; + /** + * Offset to convert monotonic `performance.now()` to epoch ms. + * Computed once so all spans share the same mapping. + */ + private readonly epochOffsetMs: number = + typeof performance !== 'undefined' + ? Date.now() - performance.now() + : Date.now(); startSpan(stage: PipelineStage, attributes?: Record): TelemetrySpan { const span: TelemetrySpan = { @@ -32,6 +59,13 @@ export class PipelineTelemetry { span.endTime = performance.now(); } + setAttributes( + span: TelemetrySpan, + attributes: Record, + ): void { + span.attributes = { ...(span.attributes ?? {}), ...attributes }; + } + getSpans(): TelemetrySpan[] { return this.spans.slice(); } @@ -40,14 +74,25 @@ export class PipelineTelemetry { this.spans = []; } - /** Export as OTEL-like JSON for observability backends. */ - toJSON(): Array> { - return this.spans.map((s) => ({ - name: s.name, - startTimeUnixNano: Math.round(s.startTime * 1e6), - endTimeUnixNano: s.endTime ? Math.round(s.endTime * 1e6) : undefined, - attributes: { stage: s.stage, ...s.attributes }, - durationMs: s.endTime ? s.endTime - s.startTime : undefined, - })); + /** + * Export a stable, backend-friendly JSON format. + * + * Notes: + * - We include epoch timestamps for correlation with server logs. + * - Durations use monotonic time to avoid clock drift issues. + */ + toJSON(): TelemetrySpanExport[] { + return this.spans.map((s) => { + const startEpoch = this.epochOffsetMs + s.startTime; + const endEpoch = s.endTime !== undefined ? this.epochOffsetMs + s.endTime : undefined; + return { + name: s.name, + stage: s.stage, + startTimeEpochMs: startEpoch, + endTimeEpochMs: endEpoch, + durationMs: s.endTime ? s.endTime - s.startTime : undefined, + attributes: { stage: s.stage, ...(s.attributes ?? {}) }, + }; + }); } } diff --git a/packages/react/README.md b/packages/react/README.md index 6897cf9..4e6dc45 100644 --- a/packages/react/README.md +++ b/packages/react/README.md @@ -88,6 +88,8 @@ Development-only component for inspecting SDK state, events, and context. import { GuideKitDevTools } from '@guidekit/react/devtools'; ``` +DevTools includes a **Telemetry** tab for per-message pipeline stage timings (useful for debugging latency and token-heavy turns). + ### `@guidekit/react/testing` Test utilities for mocking the provider in unit tests. diff --git a/packages/react/src/devtools.tsx b/packages/react/src/devtools.tsx index 199be90..661449c 100644 --- a/packages/react/src/devtools.tsx +++ b/packages/react/src/devtools.tsx @@ -31,7 +31,7 @@ import { GuideKitContext } from './_context.js'; // Types // --------------------------------------------------------------------------- -type TabId = 'state' | 'events' | 'sections' | 'ratelimits'; +type TabId = 'state' | 'events' | 'sections' | 'ratelimits' | 'telemetry'; interface EventLogEntry { id: number; @@ -51,9 +51,10 @@ const TAB_LABELS: Record = { events: 'Events', sections: 'Sections', ratelimits: 'Rate Limits', + telemetry: 'Telemetry', }; -const TAB_ORDER: TabId[] = ['state', 'events', 'sections', 'ratelimits']; +const TAB_ORDER: TabId[] = ['state', 'events', 'sections', 'ratelimits', 'telemetry']; // --------------------------------------------------------------------------- // Styles (inline — devtools live outside Shadow DOM) @@ -633,6 +634,97 @@ function RateLimitsTab({ core }: { core: GuideKitCore }) { ); } +// --------------------------------------------------------------------------- +// Tab: Telemetry +// --------------------------------------------------------------------------- + +function TelemetryTab({ core }: { core: GuideKitCore }) { + const [exported, setExported] = useState( + core.getTelemetrySpans?.() ?? [], + ); + + useEffect(() => { + const id = setInterval(() => { + setExported(core.getTelemetrySpans?.() ?? []); + }, 500); + return () => clearInterval(id); + }, [core]); + + const spans = Array.isArray(exported) ? exported : []; + + const handleCopy = useCallback(async () => { + try { + await navigator.clipboard.writeText(JSON.stringify(spans, null, 2)); + } catch { + // ignore clipboard failures in devtools + } + }, [spans]); + + const handleClear = useCallback(() => { + // Clearing happens inside core telemetry; we can only force a refresh view here. + setExported(core.getTelemetrySpans?.() ?? []); + }, [core]); + + if (spans.length === 0) { + return ( +
+
No telemetry spans yet.
+
+ Spans are recorded per message and cleared at the start of each send. +
+
+ ); + } + + return ( +
+
+ {spans.length} spans +
+ + +
+
+ +
+ {spans.map((s, idx) => { + const name = typeof s === 'object' && s ? (s as any).name : 'span'; + const stage = typeof s === 'object' && s ? (s as any).stage : ''; + const duration = + typeof s === 'object' && s ? (s as any).durationMs : undefined; + const attrs = + typeof s === 'object' && s ? (s as any).attributes : undefined; + + return ( +
+ + {String(stage || name)} + + + {typeof duration === 'number' ? `${duration.toFixed(1)}ms` : ''} + + {attrs && ( + {truncateJSON(attrs, 220)} + )} +
+ ); + })} +
+
+ ); +} + // --------------------------------------------------------------------------- // Main Component: GuideKitDevTools // --------------------------------------------------------------------------- @@ -733,6 +825,7 @@ function DevToolsInner({ core: coreProp }: { core?: GuideKitCore }) { {activeTab === 'events' && } {activeTab === 'sections' && } {activeTab === 'ratelimits' && } + {activeTab === 'telemetry' && } diff --git a/packages/server/src/auth.ts b/packages/server/src/auth.ts index 28a9b8a..d425078 100644 --- a/packages/server/src/auth.ts +++ b/packages/server/src/auth.ts @@ -307,12 +307,31 @@ export async function validateSessionToken( const { payload } = await jwtVerify(token, encodeSecret(secret), verifyOptions); + // Validate required claims defensively (never trust casts). + const sessionId = payload.sessionId; + const exp = payload.exp; + const iat = payload.iat; + const permissions = payload.permissions; + + if (typeof sessionId !== 'string' || sessionId.trim().length === 0) { + return { valid: false, error: 'Token payload missing required sessionId.' }; + } + if (typeof exp !== 'number' || !Number.isFinite(exp)) { + return { valid: false, error: 'Token payload missing required exp.' }; + } + if (typeof iat !== 'number' || !Number.isFinite(iat)) { + return { valid: false, error: 'Token payload missing required iat.' }; + } + if (!Array.isArray(permissions) || !permissions.every((p) => typeof p === 'string')) { + return { valid: false, error: 'Token payload missing required permissions.' }; + } + const tokenPayload: TokenPayload = { - sessionId: payload.sessionId as string, - expiresAt: payload.exp as number, + sessionId, + expiresAt: exp, audience: normalizeAudience(payload.aud), - permissions: (payload.permissions as string[]) ?? [], - iat: payload.iat as number, + permissions, + iat, }; if (payload.userId !== undefined) { diff --git a/packages/server/src/proxy/llm.test.ts b/packages/server/src/proxy/llm.test.ts index 1b3f4fa..d855339 100644 --- a/packages/server/src/proxy/llm.test.ts +++ b/packages/server/src/proxy/llm.test.ts @@ -131,4 +131,96 @@ describe('handleLLMProxy SSE contract', () => { const calledUrl = (globalThis.fetch as ReturnType).mock.calls[0]?.[0] as string; expect(calledUrl).toBe('https://api.anthropic.com/v1/messages'); }); + + it('rejects requests when llm permission is not granted', async () => { + globalThis.fetch = vi.fn() as unknown as typeof fetch; + + const store = new InMemorySessionStore(); + const { token } = await createSessionToken({ + signingSecret: TEST_SECRET, + sessionId: 'no-llm-perm', + llmApiKey: 'test-llm-key', + permissions: ['stt', 'tts'], + sessionStore: store, + }); + + const request = new Request('http://localhost/api/guidekit/llm', { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + provider: 'gemini', + systemPrompt: 'test', + contents: [], + userMessage: 'hello', + }), + }); + + const res = await handleLLMProxy(request, { + signingSecret: TEST_SECRET, + sessionStore: store, + }); + + expect(res.status).toBe(403); + const body = await res.json(); + expect(body.error).toContain('Permission "llm"'); + }); + + it('enforces allowedOrigins when token includes aud', async () => { + globalThis.fetch = vi.fn() as unknown as typeof fetch; + + const store = new InMemorySessionStore(); + const { token } = await createSessionToken({ + signingSecret: TEST_SECRET, + sessionId: 'origin-check', + llmApiKey: 'test-llm-key', + allowedOrigins: ['https://app.example.com'], + sessionStore: store, + }); + + const request = new Request('http://localhost/api/guidekit/llm', { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + Origin: 'https://evil.example.com', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + provider: 'gemini', + systemPrompt: 'test', + contents: [], + userMessage: 'hello', + }), + }); + + const res = await handleLLMProxy(request, { + signingSecret: TEST_SECRET, + sessionStore: store, + }); + + expect(res.status).toBe(403); + const body = await res.json(); + expect(body.error).toContain('Origin'); + }); + + it('validates request body shape', async () => { + globalThis.fetch = vi.fn() as unknown as typeof fetch; + + const { request, store } = await authedProxyRequest({ + provider: 'gemini', + systemPrompt: '', + contents: 'not-an-array', + }); + + const res = await handleLLMProxy(request, { + signingSecret: TEST_SECRET, + sessionStore: store, + }); + + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toContain('Invalid body'); + }); }); diff --git a/packages/server/src/proxy/llm.ts b/packages/server/src/proxy/llm.ts index c133a70..36aea61 100644 --- a/packages/server/src/proxy/llm.ts +++ b/packages/server/src/proxy/llm.ts @@ -25,6 +25,25 @@ export interface LLMProxyOptions { defaultModel?: string; } +function requestOrigin(request: Request): string | null { + const origin = request.headers.get('Origin'); + return origin && origin.trim().length > 0 ? origin : null; +} + +function enforceAllowedOrigins( + allowedOrigins: string[] | undefined, + origin: string | null, +): Response | null { + if (!allowedOrigins || allowedOrigins.length === 0) return null; + if (!origin) { + return jsonResponse({ error: 'Missing Origin header' }, 403); + } + if (!allowedOrigins.includes(origin)) { + return jsonResponse({ error: 'Origin not allowed' }, 403); + } + return null; +} + function extractBearerToken(request: Request): string | null { const auth = request.headers.get('Authorization'); if (!auth?.startsWith('Bearer ')) return null; @@ -54,6 +73,15 @@ export async function handleLLMProxy( return jsonResponse({ error: validation.error ?? 'Invalid token' }, 401); } + const origin = requestOrigin(request); + const originCheck = enforceAllowedOrigins(validation.payload.audience, origin); + if (originCheck) return originCheck; + + const permissions = validation.payload.permissions ?? ['stt', 'tts', 'llm']; + if (!permissions.includes('llm')) { + return jsonResponse({ error: 'Permission "llm" not granted' }, 403); + } + const keys = await options.sessionStore.get(validation.payload.sessionId); if (!keys?.llmApiKey) { return jsonResponse( @@ -69,6 +97,16 @@ export async function handleLLMProxy( return jsonResponse({ error: 'Invalid JSON body' }, 400); } + if (typeof body.systemPrompt !== 'string' || body.systemPrompt.length === 0) { + return jsonResponse({ error: 'Invalid body: systemPrompt must be a non-empty string' }, 400); + } + if (!Array.isArray(body.contents)) { + return jsonResponse({ error: 'Invalid body: contents must be an array' }, 400); + } + if (body.userMessage !== undefined && typeof body.userMessage !== 'string') { + return jsonResponse({ error: 'Invalid body: userMessage must be a string' }, 400); + } + const provider = body.provider ?? options.defaultProvider ?? 'gemini'; const apiKey = keys.llmApiKey; @@ -92,7 +130,7 @@ export async function handleLLMProxy( function jsonResponse(data: unknown, status: number): Response { return new Response(JSON.stringify(data), { status, - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json', 'X-Content-Type-Options': 'nosniff' }, }); } diff --git a/packages/server/src/proxy/voice.test.ts b/packages/server/src/proxy/voice.test.ts new file mode 100644 index 0000000..d09ffa2 --- /dev/null +++ b/packages/server/src/proxy/voice.test.ts @@ -0,0 +1,67 @@ +/** + * @vitest-environment node + */ +import { describe, it, expect } from 'vitest'; +import { handleVoiceProxy } from './voice.js'; +import { createSessionToken } from '../auth.js'; +import { InMemorySessionStore } from '../session-store.js'; + +const TEST_SECRET = 'test-secret-that-is-long-enough-for-hmac-256-bits!!'; + +describe('handleVoiceProxy', () => { + it('rejects when permission is not granted', async () => { + const store = new InMemorySessionStore(); + const { token } = await createSessionToken({ + signingSecret: TEST_SECRET, + sessionId: 'voice-no-perm', + sttApiKey: 'dg-key', + permissions: ['llm'], + sessionStore: store, + }); + + const req = new Request('http://localhost/api/guidekit/stt', { + method: 'POST', + headers: { Authorization: `Bearer ${token}` }, + }); + + const res = await handleVoiceProxy(req, { + signingSecret: TEST_SECRET, + sessionStore: store, + kind: 'stt', + }); + + expect(res.status).toBe(403); + const body = await res.json(); + expect(body.error).toContain('Permission "stt"'); + }); + + it('enforces allowedOrigins when token includes aud', async () => { + const store = new InMemorySessionStore(); + const { token } = await createSessionToken({ + signingSecret: TEST_SECRET, + sessionId: 'voice-origin', + sttApiKey: 'dg-key', + allowedOrigins: ['https://app.example.com'], + sessionStore: store, + }); + + const req = new Request('http://localhost/api/guidekit/stt', { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + Origin: 'https://evil.example.com', + }, + }); + + const res = await handleVoiceProxy(req, { + signingSecret: TEST_SECRET, + sessionStore: store, + kind: 'stt', + }); + + expect(res.status).toBe(403); + const body = await res.json(); + expect(body.error).toContain('Origin'); + }); +}); + diff --git a/packages/server/src/proxy/voice.ts b/packages/server/src/proxy/voice.ts index 0bce907..2e822e7 100644 --- a/packages/server/src/proxy/voice.ts +++ b/packages/server/src/proxy/voice.ts @@ -16,6 +16,25 @@ export interface VoiceProxyOptions { kind: VoiceProxyKind; } +function requestOrigin(request: Request): string | null { + const origin = request.headers.get('Origin'); + return origin && origin.trim().length > 0 ? origin : null; +} + +function enforceAllowedOrigins( + allowedOrigins: string[] | undefined, + origin: string | null, +): Response | null { + if (!allowedOrigins || allowedOrigins.length === 0) return null; + if (!origin) { + return jsonResponse({ error: 'Missing Origin header' }, 403); + } + if (!allowedOrigins.includes(origin)) { + return jsonResponse({ error: 'Origin not allowed' }, 403); + } + return null; +} + function extractBearerToken(request: Request): string | null { const auth = request.headers.get('Authorization'); if (!auth?.startsWith('Bearer ')) return null; @@ -25,7 +44,7 @@ function extractBearerToken(request: Request): string | null { function jsonResponse(data: unknown, status: number): Response { return new Response(JSON.stringify(data), { status, - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json', 'X-Content-Type-Options': 'nosniff' }, }); } @@ -51,6 +70,10 @@ export async function handleVoiceProxy( return jsonResponse({ error: validation.error ?? 'Invalid token' }, 401); } + const origin = requestOrigin(request); + const originCheck = enforceAllowedOrigins(validation.payload.audience, origin); + if (originCheck) return originCheck; + const permissions = validation.payload.permissions ?? ['stt', 'tts', 'llm']; if (!permissions.includes(options.kind)) { return jsonResponse({ error: `Permission "${options.kind}" not granted` }, 403); diff --git a/scripts/ci-check.sh b/scripts/ci-check.sh index 4846805..4754997 100755 --- a/scripts/ci-check.sh +++ b/scripts/ci-check.sh @@ -22,6 +22,10 @@ pnpm test:unit echo "==> pnpm size:check" pnpm size:check +echo "==> playwright install (contract browsers)" +# Contract E2E uses chromium headless shell. Install if missing (cached in CI). +pnpm exec playwright install chromium-headless-shell + echo "==> pnpm test:e2e:contract" GUIDEKIT_SECRET="${GUIDEKIT_SECRET:-guidekit-example-e2e-secret-32-chars}" \ LLM_API_KEY="${LLM_API_KEY:-e2e-dummy-llm-key-for-contract-tests}" \ diff --git a/scripts/release-check.sh b/scripts/release-check.sh index b7877a2..eabaa16 100755 --- a/scripts/release-check.sh +++ b/scripts/release-check.sh @@ -13,6 +13,9 @@ node scripts/verify-published-packages.mjs echo "==> CLI subprocess smoke" pnpm test:unit -- packages/cli/src/cli.smoke.test.ts +echo "==> playwright install (live + contract browsers)" +pnpm exec playwright install chromium-headless-shell + echo "==> live E2E run 1" LIVE_LLM=1 pnpm test:e2e:live:full