Production hardening: telemetry, proxy security, and ops docs

.changeset/production-hardening.md

@@ -0,0 +1,7 @@
+---
+"@guidekit/core": patch
+"@guidekit/react": patch
+"@guidekit/server": patch
+---
+
+Production hardening: stable pipeline telemetry export and DevTools Telemetry tab; LLM/voice proxy permission and origin checks with request validation; example app Redis session-store path and operational docs (rate limits, failure modes, observability).

apps/docs/app/docs/_meta.ts

@@ -10,6 +10,7 @@ const meta: MetaRecord = {
   'proactive-triggers': 'Proactive Triggers',
   'platform-mode': 'Platform Mode',
   privacy: 'Privacy & Security',
+  observability: 'Observability',
   server: 'Server SDK',
   compatibility: 'Compatibility Tiers',
   troubleshooting: 'Troubleshooting',

apps/docs/app/docs/architecture/page.mdx

@@ -94,9 +94,18 @@ Client → JWT to server middleware → Server looks up provider keys → Proxy
 ```
 
 - JWT tokens do **NOT** contain API keys (base64-decodable)
-- Provider keys stay server-side in an in-memory session store
+- Provider keys stay server-side in a `SessionStore` (in-memory for dev; **Redis for multi-instance production** — see [Server SDK](/docs/server#session-storage))
 - Tokens refresh at 80% of TTL, multi-tab coordination via BroadcastChannel
 - Signing secret rotation: accepts array `[newSecret, oldSecret]`
+- Proxy routes enforce **permissions** (`llm` / `stt` / `tts`) and optional **`allowedOrigins`** (JWT `aud` → `Origin` header)
+
+### Production security checklist
+
+1. **Proxy mode only** — never ship provider API keys to the browser ([Privacy & Security](/docs/privacy)).
+2. **Session store** — `RedisSessionStore` when running more than one instance ([compatibility](/docs/compatibility#multi-instance-deployments)).
+3. **Rate limits** — tune `createGuideKitHandler` `rateLimit` ([Server SDK](/docs/server#rate-limiting)).
+4. **Origin allowlist** — set `allowedOrigins` when minting tokens in production.
+5. **Observability** — use pipeline telemetry to debug latency and token-heavy turns ([Observability](/docs/observability)).
 
 ### Privacy
 
@@ -133,6 +142,12 @@ class GuideKitError extends Error {
 
 Error types: `AuthenticationError`, `ConfigurationError`, `NetworkError`, `TimeoutError`, `RateLimitError`, `PermissionError`, `BrowserSupportError`, `ContentFilterError`, `ResourceExhaustedError`, `InitializationError`.
 
+See [Troubleshooting](/docs/troubleshooting#recoverable-vs-non-recoverable-errors) for how `recoverable` maps to user-facing retry behavior.
+
+## Observability
+
+Each user message records per-stage pipeline spans (latency, token metadata). Use [Observability](/docs/observability) and the DevTools **Telemetry** tab during development.
+
 ## EventBus
 
 Typed pub/sub with namespace support:

apps/docs/app/docs/devtools/page.mdx

@@ -21,7 +21,7 @@ function App() {
 
 ## Features
 
-The DevTools panel has four tabs:
+The DevTools panel has five tabs:
 
 ### State Tab
 
@@ -59,6 +59,16 @@ Shows current rate limiter state:
 - TTS characters: current / limit per session
 - Visual meters showing usage
 
+### Telemetry Tab
+
+Shows per-message pipeline spans (stage timings and metadata) recorded by `@guidekit/core`:
+
+- Stage spans: `scan`, `enrich`, `retrieve`, `context`, `cognize`, `llm`, `validate`, `render`
+- Duration per span (`durationMs`)
+- Attributes (like `conversationId`, `totalTokens`, `toolCallsExecuted`, `rounds`)
+
+Use **Copy JSON** to paste spans into an issue, or to correlate client timings with server logs.
+
 ## Appearance
 
 The DevTools panel:

apps/docs/app/docs/observability/page.mdx

@@ -0,0 +1,73 @@
+# Observability
+
+GuideKit ships **client-side pipeline telemetry** so you can understand:
+
+- **Latency**: which stages are slow (DOM scan vs LLM vs validation)
+- **Cost signals**: token usage metadata attached to the LLM stage
+- **Failures**: correlate stage timing with `EventBus` errors and server proxy logs
+
+## What gets recorded
+
+Each user message runs the v2 pipeline stages:
+
+`scan → enrich → retrieve → context → cognize → llm → validate → render`
+
+For each stage, GuideKit records one span with:
+
+- `name`: stable span name (example: `guidekit.pipeline.llm`)
+- `stage`: pipeline stage id
+- `startTimeEpochMs` / `endTimeEpochMs`: wall-clock timestamps for log correlation
+- `durationMs`: monotonic duration in milliseconds
+- `attributes`: stage metadata (primitives only)
+
+## Where to see telemetry
+
+### DevTools (recommended)
+
+In development, enable the DevTools panel and open the **Telemetry** tab:
+
+```tsx
+import { GuideKitDevTools } from '@guidekit/react/devtools';
+
+{process.env.NODE_ENV === 'development' && <GuideKitDevTools />}
+```
+
+The Telemetry tab shows stage timings and provides **Copy JSON** for sharing or analysis.
+
+### Programmatic export
+
+If you own a custom UI, you can read the last message spans from the core instance:
+
+```ts
+const spans = core.getTelemetrySpans();
+```
+
+This returns an array of exported spans with epoch timestamps and attributes.
+
+## Attribute contract (stable keys)
+
+GuideKit guarantees these attribute keys when applicable:
+
+- **`stage`**: pipeline stage id
+- **`conversationId`**: unique per user message
+- **`totalTokens`**: total tokens used for the completed message (attached as stages complete)
+- **`toolCallsExecuted`**: number of executed tool calls for the message
+- **`rounds`**: LLM/tool rounds performed
+
+Additional attributes may be added over time, but existing keys are stable.
+
+## Debugging workflow
+
+1. Reproduce the slow or failed message in dev with `options={{ debug: true }}`.
+2. Open **GuideKit DevTools → Telemetry** and note the slowest `stage` (`llm` vs `scan` vs `validate`).
+3. Check **Events** for `error`, `auth:token-refreshed`, or `validation:complete`.
+4. Correlate `conversationId` and `startTimeEpochMs` with server proxy logs.
+
+For proxy/auth failures (401, 403, 429), see [Troubleshooting](/docs/troubleshooting#production--proxy-failures).
+
+## See also
+
+- [Architecture](/docs/architecture) — security model and production checklist
+- [Server SDK](/docs/server) — rate limits, session store, origin allowlist
+- [DevTools](/docs/devtools) — Telemetry tab reference
+

apps/docs/app/docs/server/page.mdx

@@ -57,6 +57,35 @@ const keys = await getSessionKeys('my-session-id');
 import { RedisSessionStore } from '@guidekit/server/redis';
 ```
 
+### In-memory vs Redis (production guidance)
+
+- **InMemorySessionStore**: good for local dev and single-instance deployments.
+  - Not suitable for **multi-instance** or **serverless** deployments where requests can land on different processes.
+- **RedisSessionStore**: recommended for production when you run multiple instances or need durability across restarts.
+
+The reference example app (`apps/example-nextjs`) uses in-memory storage by default. Set `REDIS_URL` and install `ioredis` in that app to exercise the Redis path locally.
+
+Example:
+
+```typescript
+import Redis from 'ioredis';
+import { RedisSessionStore } from '@guidekit/server/redis';
+import { createNextAppRouterRoutes } from '@guidekit/server/next';
+
+const redis = new Redis(process.env.REDIS_URL!);
+const sessionStore = new RedisSessionStore({ redis });
+
+const routes = createNextAppRouterRoutes({
+  signingSecret: process.env.GUIDEKIT_SECRET!,
+  sessionStore,
+  createTokenOptions: () => ({
+    llmApiKey: process.env.LLM_API_KEY!,
+    expiresIn: '15m',
+    allowedOrigins: ['https://yourapp.com'],
+  }),
+});
+```
+
 ## Framework-agnostic handler
 
 For non-Next.js runtimes, use `createGuideKitHandler`:
@@ -92,7 +121,53 @@ All proxy routes require `Authorization: Bearer <session-token>` except `/token`
 
 ## Rate limiting
 
-`createGuideKitHandler` applies a sliding-window rate limiter per session ID and IP (default: 60 req/min).
+`createGuideKitHandler` applies a **sliding-window** rate limiter keyed by **session ID** (when authenticated) or **client IP**.
+
+### Defaults
+
+| Setting | Default | Meaning |
+|---------|---------|---------|
+| `windowMs` | `60_000` | 1-minute window |
+| `maxRequests` | `60` | Max requests per window per key |
+
+When exceeded, the handler returns **429** with a `Retry-After` header (seconds).
+
+### Production recommendations
+
+| Deployment | Suggested `maxRequests` | Notes |
+|------------|-------------------------|-------|
+| Single-tenant internal app | `60` (default) | Fine for most dashboards |
+| Public consumer app | `30–40` | Tighten if you see abuse |
+| High-traffic B2B | `80–120` | Monitor 429 rates; add CDN/WAF in front |
+
+Tune via handler options:
+
+```typescript
+const handler = createGuideKitHandler({
+  signingSecret: process.env.GUIDEKIT_SECRET!,
+  rateLimit: {
+    windowMs: 60_000,
+    maxRequests: 40,
+  },
+  createTokenOptions: () => ({ llmApiKey: process.env.LLM_API_KEY!, expiresIn: '15m' }),
+});
+```
+
+The example app reads `GUIDEKIT_RATE_LIMIT_WINDOW_MS` and `GUIDEKIT_RATE_LIMIT_MAX` for local tuning.
+
+### Origin allowlist (recommended in production)
+
+Set `allowedOrigins` when minting tokens. Proxy routes (`/llm`, `/stt`, `/tts`) reject requests when the token includes an `aud` claim and the `Origin` header does not match.
+
+```typescript
+createTokenOptions: () => ({
+  llmApiKey: process.env.LLM_API_KEY!,
+  expiresIn: '15m',
+  allowedOrigins: ['https://yourapp.com'],
+}),
+```
+
+In the example app, set `GUIDEKIT_ALLOWED_ORIGINS=https://yourapp.com,https://staging.yourapp.com`.
 
 ## `createSessionToken`
 

apps/docs/app/docs/troubleshooting/page.mdx

@@ -131,6 +131,69 @@ import { createGuideKit } from '@guidekit/core';
 
 Visual/rendering exports live under `@guidekit/core/rendering` to keep the vanilla bundle smaller.
 
+## Production / Proxy Failures
+
+### Session expired or server restarted (401)
+
+**Symptom:** LLM proxy returns `401` with *"Session expired or server restarted — request a new token"*.
+
+**Cause:** Provider keys live in the server `SessionStore`, keyed by `sessionId`. After a deploy/restart (in-memory store) or TTL expiry, the JWT may still decode but keys are gone.
+
+**Fix:**
+1. Ensure the client refreshes tokens (GuideKit refreshes at ~80% TTL automatically).
+2. For multi-instance/serverless, use [`RedisSessionStore`](/docs/server#in-memory-vs-redis-production-guidance).
+3. See [Session recovery](/docs/troubleshooting#session-recovery-client-side) below.
+
+### Rate limit exceeded (429)
+
+**Symptom:** Proxy returns `429` with `Retry-After`.
+
+**Fix:** Back off and retry after the header value. Tune `rateLimit.maxRequests` on `createGuideKitHandler` — see [Server SDK rate limiting](/docs/server#rate-limiting).
+
+### Origin not allowed (403)
+
+**Symptom:** Proxy returns `403` with *"Origin not allowed"*.
+
+**Fix:** Ensure `allowedOrigins` on `createSessionToken` includes your app's origin (scheme + host + port). Browser `fetch` from the app must send a matching `Origin` header.
+
+### Permission denied (403)
+
+**Symptom:** STT/TTS/LLM proxy returns `403` with *Permission "llm" not granted* (or `stt` / `tts`).
+
+**Fix:** Mint tokens with the required `permissions` array, e.g. `['stt', 'tts', 'llm']` (default).
+
+## Recoverable vs non-recoverable errors
+
+Every `GuideKitError` includes `recoverable`:
+
+| `recoverable` | Meaning | Examples |
+|---------------|---------|----------|
+| `true` | Safe to retry or continue | `SEND_IN_FLIGHT`, `INPUT_TOO_LONG`, rate limits, network blips |
+| `false` | Fix configuration or auth | Missing LLM config, invalid provider, fatal init errors |
+
+**Client pattern:**
+
+```tsx
+core.bus.on('error', (err) => {
+  if (err.recoverable) {
+    // show toast, allow retry
+    return;
+  }
+  // log to monitoring, show support message
+});
+```
+
+Use [Observability](/docs/observability) (DevTools Telemetry tab or `core.getTelemetrySpans()`) to see which pipeline stage failed before the error surfaced.
+
+### Session recovery (client-side)
+
+When the LLM proxy returns **401**, the SDK should fetch a new token and retry. Contract coverage: `e2e/contract/session-recovery.spec.ts`.
+
+If recovery loops:
+1. Confirm `/api/guidekit/token` returns 200 and `LLM_API_KEY` is set server-side.
+2. Confirm all app instances share the same `SessionStore` (Redis in production).
+3. Enable `options={{ debug: true }}` and watch `auth:token-refreshed` events.
+
 ## Error Codes
 
 GuideKit uses structured error codes for all failures. Each `GuideKitError` includes:

apps/example-nextjs/lib/guidekit-routes.ts

@@ -1,12 +1,25 @@
-import { createNextAppRouterRoutes, getSharedSessionStore } from '@guidekit/server/next';
+import { createNextAppRouterRoutes } from '@guidekit/server/next';
+import { guidekitSessionStore } from './guidekit-session-store';
+
+function parseAllowedOrigins(): string[] | undefined {
+  const raw = process.env.GUIDEKIT_ALLOWED_ORIGINS;
+  if (!raw) return undefined;
+  const origins = raw.split(',').map((s) => s.trim()).filter(Boolean);
+  return origins.length > 0 ? origins : undefined;
+}
 
 export const guidekitRoutes = createNextAppRouterRoutes({
   signingSecret: process.env.GUIDEKIT_SECRET!,
-  sessionStore: getSharedSessionStore(),
+  sessionStore: guidekitSessionStore,
+  rateLimit: {
+    windowMs: Number(process.env.GUIDEKIT_RATE_LIMIT_WINDOW_MS ?? 60_000),
+    maxRequests: Number(process.env.GUIDEKIT_RATE_LIMIT_MAX ?? 60),
+  },
   createTokenOptions: () => ({
     llmApiKey: process.env.LLM_API_KEY!,
     sttApiKey: process.env.STT_API_KEY,
     ttsApiKey: process.env.TTS_API_KEY,
     expiresIn: '15m',
+    allowedOrigins: parseAllowedOrigins(),
   }),
 });

apps/example-nextjs/lib/guidekit-session-store.ts

@@ -0,0 +1,33 @@
+import { createRequire } from 'node:module';
+import type { SessionStore } from '@guidekit/server';
+import { getSharedSessionStore } from '@guidekit/server/next';
+import { RedisSessionStore, type RedisLike } from '@guidekit/server/redis';
+
+const require = createRequire(import.meta.url);
+
+/**
+ * Process-wide session store for the example app.
+ *
+ * - Default: in-memory (single instance, local dev)
+ * - Production: set REDIS_URL and install `ioredis` for horizontal scaling
+ */
+function buildSessionStore(): SessionStore {
+  const redisUrl = process.env.REDIS_URL;
+  if (!redisUrl) {
+    return getSharedSessionStore();
+  }
+
+  try {
+    // Optional peer — webpack must not statically resolve unless installed.
+    const redisPkg = ['iored', 'is'].join('');
+    const Redis = require(redisPkg) as new (url: string) => RedisLike;
+    return new RedisSessionStore({ redis: new Redis(redisUrl) });
+  } catch {
+    console.warn(
+      '[GuideKit example] REDIS_URL is set but ioredis is not installed. Falling back to in-memory session store.',
+    );
+    return getSharedSessionStore();
+  }
+}
+
+export const guidekitSessionStore: SessionStore = buildSessionStore();

apps/example-nextjs/next.config.mjs

@@ -2,6 +2,8 @@
 const nextConfig = {
   // Transpile workspace packages so Next.js can process their TypeScript/ESM
   transpilePackages: ['@guidekit/core', '@guidekit/react', '@guidekit/server', '@guidekit/vad'],
+  // Optional when REDIS_URL is set (see lib/guidekit-session-store.ts)
+  serverExternalPackages: ['ioredis'],
   // Disable React Strict Mode to avoid double-mount issues with Shadow DOM
   // (attachShadow can only be called once per element)
   reactStrictMode: false,

e2e/contract/custom-actions.spec.ts

@@ -23,7 +23,7 @@ test.describe('Custom actions', () => {
 
     const input = await openWidgetInput(page);
     await input.fill('Show alert');
-    await page.getByTestId('guidekit-send').click();
+    await input.press('Enter');
 
     await expect(page.locator('.gk-message[data-role="assistant"]').last()).not.toHaveText('', {
       timeout: 20_000,