diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 6ddaac279..d8ad6da8c 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -348,7 +348,10 @@ interface RecordedAnalyticsEvent { function makeRecordingAnalytics() { const events: Array = []; + let epoch = 1; const layer = Layer.mock(AnalyticsService.AnalyticsService)({ + status: Effect.succeed({ available: true, consent: "product" as const }), + collectionEpoch: Effect.sync(() => epoch), record: (event, properties) => Effect.sync(() => { events.push({ event, ...(properties ? { properties } : {}) }); @@ -360,6 +363,10 @@ function makeRecordingAnalytics() { layer, reset: () => { events.length = 0; + epoch = 1; + }, + changeEpoch: () => { + epoch += 1; }, eventsByName: (event: string) => events.filter((entry) => entry.event === event), }; @@ -4171,6 +4178,46 @@ turnAnalytics.layer("ProviderServiceLive turn analytics", (it) => { }), ); + it.effect("does not export turn usage across a collection epoch change", () => + Effect.gen(function* () { + recordedTurnAnalytics.reset(); + const provider = yield* ProviderService.ProviderService; + const threadId = asThreadId("thread-turn-analytics-consent-change"); + yield* provider.startSession(threadId, { + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + threadId, + runtimeMode: "approval-required", + }); + const turn = yield* provider.sendTurn({ threadId, input: "test", attachments: [] }); + recordedTurnAnalytics.changeEpoch(); + const received = yield* Stream.take(provider.streamEvents, 1).pipe( + Stream.runDrain, + Effect.forkChild, + ); + yield* Effect.yieldNow; + primaryAnalyticsCodex.emit({ + type: "turn.aborted", + eventId: asEventId("evt-consent-change"), + provider: CODEX_DRIVER, + createdAt: "2026-01-01T00:00:00.000Z", + threadId, + turnId: turn.turnId, + payload: { + reason: "Interrupted", + tokenUsage: { + usageStatus: "partial", + usageScope: "main_agent", + inputTokens: 120, + outputTokens: 30, + }, + }, + }); + yield* Fiber.join(received); + assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0); + }), + ); + it.effect("records known token counts for an interrupted turn", () => Effect.gen(function* () { recordedTurnAnalytics.reset(); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 088279cf9..cfeef33be 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -111,6 +111,7 @@ export interface ProviderServiceLiveOptions { } interface TurnAnalyticsMetadata { + readonly collectionEpoch?: number; readonly requestId: number; readonly provider: ProviderDriverKind; readonly startedAtMs: number; @@ -387,8 +388,23 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( state.sessions.delete(input.sessionKey); } + const terminal = { ...input.completion.terminalProperties }; + if (!metadata) { + // A terminal without its start cannot establish a consent-covered token + // interval (including bounded-map eviction). Keep coverage, not totals. + for (const key of [ + "inputTokens", + "outputTokens", + "cachedInputTokens", + "cacheCreationTokens", + "reasoningTokens", + ]) + delete terminal[key]; + terminal.usageStatus = "unavailable"; + } return { - ...input.completion.terminalProperties, + ...terminal, + collectionEpoch: metadata ? metadata.collectionEpoch : terminal.collectionEpoch, ...(metadata?.model ? { model: metadata.model } : {}), ...(metadata?.effort ? { effort: metadata.effort } : {}), ...(metadata?.interactionMode ? { interactionMode: metadata.interactionMode } : {}), @@ -403,9 +419,22 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const recordCompletedTurnProperties = ( properties: ReadonlyArray>>, ) => - Effect.forEach(properties, (entry) => analytics.record("provider.turn.completed", entry), { - discard: true, - }); + Effect.forEach( + properties, + (entry) => + Effect.gen(function* () { + const { collectionEpoch, ...properties } = entry; + if ( + collectionEpoch === undefined || + collectionEpoch !== (yield* analytics.collectionEpoch) + ) + return; + yield* analytics.record("provider.turn.completed", properties); + }), + { + discard: true, + }, + ); const clearTurnAnalyticsSession = (providerInstanceId: ProviderInstanceId, threadId: ThreadId) => Effect.gen(function* () { @@ -434,6 +463,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( readonly runtimeMode: string | undefined; }) { const startedAtMs = DateTime.toEpochMillis(yield* DateTime.now); + const analyticsStatus = yield* analytics.status; + const collectionEpoch = + analyticsStatus.available && + (analyticsStatus.consent === "product" || analyticsStatus.consent === "diagnostic") + ? yield* analytics.collectionEpoch + : undefined; turnAnalyticsRequestId += 1; const requestId = turnAnalyticsRequestId; const effort = turnEffort(input.modelSelection); @@ -445,6 +480,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( deferredCompletionsByTurnId: new Map(), }; const metadata: TurnAnalyticsMetadata = { + ...(collectionEpoch === undefined ? {} : { collectionEpoch }), provider: input.provider, startedAtMs, mixedModels: false, @@ -569,6 +605,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ) { if (!event.turnId) return; const observedAtMs = DateTime.toEpochMillis(yield* DateTime.now); + const analyticsStatus = yield* analytics.status; + const collectionEpoch = + analyticsStatus.available && + (analyticsStatus.consent === "product" || analyticsStatus.consent === "diagnostic") + ? yield* analytics.collectionEpoch + : undefined; yield* Ref.update(turnAnalytics, (state) => { const completionKey = turnAnalyticsCompletionKey( source.instanceId, @@ -590,6 +632,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const current = session.activeByTurnId.get(String(event.turnId)); const metadata: TurnAnalyticsMetadata = { ...(current?.metadata ?? { + ...(collectionEpoch === undefined ? {} : { collectionEpoch }), requestId: ++turnAnalyticsRequestId, provider: source.provider, startedAtMs: observedAtMs, @@ -641,6 +684,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ) { if (!event.turnId) return; const completedAtMs = DateTime.toEpochMillis(yield* DateTime.now); + const analyticsStatus = yield* analytics.status; + const collectionEpoch = + analyticsStatus.available && + (analyticsStatus.consent === "product" || analyticsStatus.consent === "diagnostic") + ? yield* analytics.collectionEpoch + : undefined; const tokenUsage = event.payload.tokenUsage; const completion: DeferredTurnAnalyticsCompletion = { completionKey: turnAnalyticsCompletionKey( @@ -650,6 +699,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), completedAtMs, terminalProperties: { + collectionEpoch, provider: source.provider, terminalStatus: event.type === "turn.completed" diff --git a/apps/server/src/telemetry/AnalyticsUiAdapter.ts b/apps/server/src/telemetry/AnalyticsUiAdapter.ts index a1d2abfda..956d9a24a 100644 --- a/apps/server/src/telemetry/AnalyticsUiAdapter.ts +++ b/apps/server/src/telemetry/AnalyticsUiAdapter.ts @@ -28,6 +28,20 @@ export function makeAnalyticsUiAdapter(analytics: AnalyticsService["Service"]) { ); const record = Effect.fn("AnalyticsUiAdapter.record")(function* (event: ScientAnalyticsUiEvent) { + if ( + event.name === "panel.viewed" || + event.name === "settings.viewed" || + event.name === "usage.viewed" || + event.name === "feature.viewed" || + event.name === "usage.availability" + ) { + const current = yield* status; + if ( + event.collectionContext === undefined || + event.collectionContext !== current.collectionContext + ) + return { accepted: false } as const; + } if (event.name.startsWith("scient.operation.")) { const current = yield* status; if ( diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index b3bda100c..0f330c37a 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -9191,6 +9191,7 @@ function ChatViewContent(props: ChatViewProps) { > dispatch({ _tag: "SetOpen", open }), []); + useScientAnalyticsView( + state.open + ? { + name: "feature.viewed", + properties: { + feature: + state.openIntent?.kind === "add-project" + ? "project-picker" + : state.openIntent?.kind === "new-thread-in" + ? "new-thread" + : "search", + }, + } + : null, + ); const toggleMode = useCallback( (mode: SearchOverlayMode) => dispatch({ _tag: "ToggleMode", mode }), [], diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index 785a2d32d..b5ef35bfe 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -62,6 +62,8 @@ import { useEnvironmentQuery } from "~/state/query"; import { COLLAPSED_SIDEBAR_TITLEBAR_INSET_CLASS } from "~/workspaceTitlebar"; import { ScientRightPanelSurfaceIcon } from "~/scient/rightPanel/ScientRightPanelSurfaceIcon"; import { scientRightPanelSurfaceTitle } from "~/scient/rightPanel/surfaces"; +import { useScientAnalyticsView } from "~/scient/analytics/client"; +import { panelCategory } from "~/scient/analytics/viewCategories"; import { PreviewPanelShell, type PreviewPanelMode } from "./preview/PreviewPanelShell"; import { FaviconImage } from "./preview/PreviewFaviconIcon"; @@ -803,6 +805,18 @@ function PullRequestSurfaceIcon({ } export function RightPanelTabs(props: RightPanelTabsProps) { + const analyticsSurface = + props.open === false + ? undefined + : props.surfaces.find((surface) => surface.id === props.activeSurfaceId); + useScientAnalyticsView( + analyticsSurface + ? { name: "panel.viewed", properties: { category: panelCategory(analyticsSurface) } } + : null, + analyticsSurface?.kind === "pull-request" && analyticsSurface.environmentId + ? (analyticsSurface.environmentId as EnvironmentId) + : props.environmentId, + ); const ownsDesktopTitleBar = isElectron && props.mode === "inline"; const browserProfiles = useBrowserDefaults().profiles; const { resolvedTheme } = useTheme(); diff --git a/apps/web/src/components/usage/UsagePage.tsx b/apps/web/src/components/usage/UsagePage.tsx index deb05f266..13e80f7d5 100644 --- a/apps/web/src/components/usage/UsagePage.tsx +++ b/apps/web/src/components/usage/UsagePage.tsx @@ -1,4 +1,5 @@ import { RefreshIcon } from "~/components/ui/refresh-icon"; +import { useRecordScientAnalytics, useScientAnalyticsView } from "~/scient/analytics/client"; import { useAtomValue } from "@effect/atom-react"; import { USAGE_CONTRACT_VERSION, @@ -108,12 +109,38 @@ export function UsagePage() { const [selectedEnvironmentIds, setSelectedEnvironmentIds] = useState | null>(null); const { days: windowDays, window } = windowSelection; + const recordAnalytics = useRecordScientAnalytics(); + useScientAnalyticsView({ + name: "usage.viewed", + properties: { + metric, + window: String(windowDays), + breakdown: showingLimits ? "other" : breakdown, + }, + }); const isPast24Hours = windowDays === 1; const { merged, environments, selectedEnvironments, isPending, isPartial, refresh } = useUsage( window, selectedEnvironmentIds, ); const presentations = useAtomValue(environmentPresentations.presentationsAtom); + useScientAnalyticsView( + showingLimits || + isPending || + isRefreshing || + selectedEnvironments.some((entry) => entry.isPending) + ? null + : { + name: "usage.availability", + properties: { + state: !selectedEnvironments.some((entry) => entry.summary !== null) + ? "unavailable" + : selectedEnvironments.some((entry) => entry.error !== null || entry.summary === null) + ? "partial" + : "available", + }, + }, + ); const refreshProviders = useAtomCommand(serverEnvironment.refreshProviders, { reportFailure: false, }); @@ -164,6 +191,7 @@ export function UsagePage() { }; const refreshWindow = () => { if (refreshingRef.current) return; + recordAnalytics({ name: "usage.refresh.requested", properties: {} }); if (showingLimits) { refreshingRef.current = true; diff --git a/apps/web/src/routes/__root.tsx b/apps/web/src/routes/__root.tsx index d3baa6e8f..eda7a1f31 100644 --- a/apps/web/src/routes/__root.tsx +++ b/apps/web/src/routes/__root.tsx @@ -23,6 +23,7 @@ import { SshPasswordPromptDialog } from "../components/desktop/SshPasswordPrompt import { DesktopAppActivationCoordinator } from "../components/desktop/DesktopAppActivationCoordinator"; import { ProviderUpdateLaunchNotification } from "../components/ProviderUpdateLaunchNotification"; import { AnalyticsSharingNotice } from "../scient/analytics/AnalyticsSharingNotice"; +import { SettingsAnalyticsObserver } from "../scient/analytics/SettingsAnalyticsObserver"; import { SlowRpcRequestToastCoordinator } from "../components/SlowRpcRequestToastCoordinator"; import { ThemeEditorHost } from "../components/settings/ThemeEditorHost"; import { useCopyToClipboard } from "../hooks/useCopyToClipboard"; @@ -180,6 +181,7 @@ function RootRouteView() { {primaryEnvironmentAuthenticated ? : null} {primaryEnvironmentAuthenticated ? : null} {primaryEnvironmentAuthenticated ? : null} + {primaryEnvironmentAuthenticated ? : null} {appShell} {/* Above the router: a theme draft is judged by walking the app, so the editor has to survive navigation away from settings. */} diff --git a/apps/web/src/scient/analytics/AnalyticsPrivacySettings.test.tsx b/apps/web/src/scient/analytics/AnalyticsPrivacySettings.test.tsx index 026e3d331..e479c9068 100644 --- a/apps/web/src/scient/analytics/AnalyticsPrivacySettings.test.tsx +++ b/apps/web/src/scient/analytics/AnalyticsPrivacySettings.test.tsx @@ -178,7 +178,10 @@ describe("analytics sharing controls", () => { await render(); await details(); const dialog = document.querySelector('[role="dialog"]'); - expect(dialog?.textContent).toContain("Share which features you use"); + expect(dialog?.textContent).toContain( + "Share which features and provider/model categories you use", + ); + expect(dialog?.textContent).toContain("reported token counts"); expect(dialog?.querySelector("dl")).toBeNull(); expect(dialog?.textContent).toContain( "Analytics never includes prompts, responses, file contents", diff --git a/apps/web/src/scient/analytics/AnalyticsSharingInfo.tsx b/apps/web/src/scient/analytics/AnalyticsSharingInfo.tsx index cf3f151cd..e90e49f49 100644 --- a/apps/web/src/scient/analytics/AnalyticsSharingInfo.tsx +++ b/apps/web/src/scient/analytics/AnalyticsSharingInfo.tsx @@ -13,9 +13,9 @@ export function AnalyticsSharingInfo({ consent }: { consent: ScientAnalyticsCons What’s shared?

- Share which features you use, whether operations succeed or fail, basic performance - information, and counters that help check analytics delivery. Turn sharing off in - Settings to stop sending analytics. + Share which features and provider/model categories you use, reported token counts, + whether operations succeed or fail, basic performance information, and counters that + help check analytics delivery. Turn sharing off in Settings to stop sending analytics.

{consent === "essential" || consent === "product" ? (

diff --git a/apps/web/src/scient/analytics/SettingsAnalyticsObserver.tsx b/apps/web/src/scient/analytics/SettingsAnalyticsObserver.tsx new file mode 100644 index 000000000..aa8b17b97 --- /dev/null +++ b/apps/web/src/scient/analytics/SettingsAnalyticsObserver.tsx @@ -0,0 +1,11 @@ +import { useLocation } from "@tanstack/react-router"; +import { useScientAnalyticsView } from "./client"; +import { settingsCategory } from "./viewCategories"; + +export function SettingsAnalyticsObserver() { + const section = settingsCategory(useLocation().pathname); + useScientAnalyticsView( + section === null ? null : { name: "settings.viewed", properties: { section } }, + ); + return null; +} diff --git a/apps/web/src/scient/analytics/client.ts b/apps/web/src/scient/analytics/client.ts index 2b8ef4f6e..352c733c7 100644 --- a/apps/web/src/scient/analytics/client.ts +++ b/apps/web/src/scient/analytics/client.ts @@ -11,7 +11,7 @@ import type { ScientAnalyticsUiEvent, EnvironmentId, } from "@t3tools/contracts"; -import { useCallback, useEffect } from "react"; +import { useCallback, useEffect, useEffectEvent } from "react"; import * as Option from "effect/Option"; import { runtime } from "../../lib/runtime"; @@ -29,6 +29,27 @@ const gate = createAnalyticsClientGate({ runtime.runPromise(recordEnvironmentScientAnalyticsEvent({ prepared, event })), }); +/** Observe current view after consent discovery, not historical clicks. */ +export function useScientAnalyticsView( + event: ScientAnalyticsUiEvent | null, + explicitEnvironmentId?: EnvironmentId | null, +) { + const primary = usePrimaryEnvironmentId(); + const environmentId = explicitEnvironmentId === undefined ? primary : explicitEnvironmentId; + const prepared = usePreparedConnection(environmentId); + const key = JSON.stringify(event); + const read = useEffectEvent(() => (document.visibilityState === "hidden" ? null : event)); + useEffect(() => { + if (Option.isNone(prepared)) return; + const observation = gate.observeView(prepared.value, () => read()); + document.addEventListener("visibilitychange", observation.refresh); + return () => { + observation.dispose(); + document.removeEventListener("visibilitychange", observation.refresh); + }; + }, [prepared, key]); +} + export function beginScientUiOperation( environmentId: EnvironmentId, kind: ScientUiOperationKind, diff --git a/apps/web/src/scient/analytics/clientGate.test.ts b/apps/web/src/scient/analytics/clientGate.test.ts index 30653dba3..12ec04c9a 100644 --- a/apps/web/src/scient/analytics/clientGate.test.ts +++ b/apps/web/src/scient/analytics/clientGate.test.ts @@ -11,6 +11,81 @@ const tick = async () => { }; describe("UI analytics gate", () => { + it("observes only the current visible view after discovery, with no historic replay", async () => { + let resolve!: (status: ScientAnalyticsStatus) => void; + const record = vi.fn(async () => undefined); + const gate = createAnalyticsClientGate({ + status: () => + new Promise((done) => { + resolve = done; + }), + record, + }); + const connection = {}; + let visible = true; + const view = gate.observeView(connection, () => + visible ? { name: "panel.viewed", properties: { category: "browser" } } : null, + ); + await tick(); + visible = false; + resolve({ available: true, consent: "product", collectionContext: "first" }); + await tick(); + expect(record).not.toHaveBeenCalled(); + visible = true; + view.refresh(); + view.refresh(); + await tick(); + expect(record).toHaveBeenCalledOnce(); + gate.beginControl(connection); + gate.endControl(connection, { available: true, consent: "off" }); + gate.beginControl(connection); + gate.endControl(connection, { + available: true, + consent: "product", + collectionContext: "second", + }); + await tick(); + expect(record).toHaveBeenCalledTimes(2); + expect(record.mock.calls[1]).toEqual([ + connection, + { name: "panel.viewed", properties: { category: "browser" }, collectionContext: "second" }, + ]); + view.dispose(); + }); + it("does not send views disposed before discovery or strict-mode effect restart", async () => { + const record = vi.fn(async () => undefined); + const status = vi.fn(async () => ({ + available: true, + consent: "product" as const, + collectionContext: "current", + })); + const gate = createAnalyticsClientGate({ status, record }); + const connection = {}; + gate.observeView(connection, () => event).dispose(); + const active = gate.observeView(connection, () => event); + await tick(); + expect(record).toHaveBeenCalledOnce(); + active.dispose(); + }); + it("does no event work for Off or Essential views and bounds observers", async () => { + for (const consent of ["off", "essential"] as const) { + const record = vi.fn(async () => undefined); + const gate = createAnalyticsClientGate({ + status: async () => ({ available: true, consent, collectionContext: "current" }), + record, + }); + const connection = {}; + const views = Array.from({ length: 100 }, () => + gate.observeView(connection, () => ({ + name: "settings.viewed", + properties: { section: "general" }, + })), + ); + await tick(); + expect(record).not.toHaveBeenCalled(); + for (const view of views) view.dispose(); + } + }); it("coalesces discovery and does no per-event HTTP work while Off", async () => { const status = vi.fn(async (): Promise => ({ available: true, diff --git a/apps/web/src/scient/analytics/clientGate.ts b/apps/web/src/scient/analytics/clientGate.ts index 8dcd3527d..005150c33 100644 --- a/apps/web/src/scient/analytics/clientGate.ts +++ b/apps/web/src/scient/analytics/clientGate.ts @@ -24,6 +24,7 @@ export function createAnalyticsClientGate(transport: generation: number; inFlight: number; surfaces: Set; + views: Set<() => void>; }; const states = new WeakMap(); const now = transport.now ?? Date.now; @@ -39,6 +40,7 @@ export function createAnalyticsClientGate(transport: generation: 0, inFlight: 0, surfaces: new Set(), + views: new Set(), }; states.set(connection, state); } @@ -56,6 +58,7 @@ export function createAnalyticsClientGate(transport: state.status = status; state.checkedAt = now(); state.retryStatusAt = null; + for (const view of state.views) view(); }; const readStatus = async (connection: Connection) => { const state = stateFor(connection); @@ -137,6 +140,40 @@ export function createAnalyticsClientGate(transport: return true; }; return { + observeView(connection: Connection, read: () => ScientAnalyticsUiEvent | null) { + const state = stateFor(connection); + let reportedContext: string | undefined; + let disposed = false; + const refresh = () => { + if (disposed) return; + const event = read(); + if (event === null) { + reportedContext = undefined; + return; + } + const ready = readyState(connection); + const context = ready?.status?.collectionContext; + if ( + ready && + context !== undefined && + reportedContext !== context && + enqueue(connection, ready, { ...event, collectionContext: context }) + ) + reportedContext = context; + }; + // Only current visible state is read after discovery; never replay an + // action or a screen the user has already left. Bound retained observers. + if (state.views.size >= 16) return { refresh: () => {}, dispose: () => {} }; + state.views.add(refresh); + void Promise.resolve().then(refresh); + return { + refresh, + dispose: () => { + disposed = true; + state.views.delete(refresh); + }, + }; + }, readStatus, prime(connection: Connection) { if (stateFor(connection).status === null) prime(connection); diff --git a/apps/web/src/scient/analytics/viewCategories.test.ts b/apps/web/src/scient/analytics/viewCategories.test.ts new file mode 100644 index 000000000..9e7e4a53f --- /dev/null +++ b/apps/web/src/scient/analytics/viewCategories.test.ts @@ -0,0 +1,30 @@ +import { expect, it } from "vite-plus/test"; +import { panelCategory, settingsCategory } from "./viewCategories"; + +it("categorizes nested settings without transmitting identifiers", () => { + expect(settingsCategory("/settings/providers/PRIVATE")).toBe("providers"); + expect(settingsCategory("/settings/PRIVATE")).toBe("other"); + expect(settingsCategory("/PRIVATE")).toBeNull(); +}); +it("never uses panel identity, path or titles as categories", () => { + expect( + panelCategory({ + id: "file:PRIVATE", + kind: "file", + relativePath: "PRIVATE", + revealLine: null, + revealRequestId: 0, + }), + ).toBe("file-preview"); + expect(panelCategory({ id: "browser:PRIVATE", kind: "preview", resourceId: "PRIVATE" })).toBe( + "browser", + ); + expect( + panelCategory({ + id: "scient:compute:PRIVATE", + kind: "scient", + module: "compute", + cwd: "PRIVATE", + }), + ).toBe("compute"); +}); diff --git a/apps/web/src/scient/analytics/viewCategories.ts b/apps/web/src/scient/analytics/viewCategories.ts new file mode 100644 index 000000000..5e2eef314 --- /dev/null +++ b/apps/web/src/scient/analytics/viewCategories.ts @@ -0,0 +1,35 @@ +import type { RightPanelSurface } from "~/rightPanelStore"; + +export function panelCategory(surface: RightPanelSurface): string { + switch (surface.kind) { + case "preview": + return "browser"; + case "file": + return "file-preview"; + case "scient": + return surface.module === "file" ? "file-preview" : surface.module; + default: + return surface.kind; + } +} + +const SECTIONS = new Set([ + "general", + "appearance", + "projects", + "keybindings", + "providers", + "custom-models", + "voice", + "skills", + "integrations", + "scientific-computing", + "source-control", + "connections", + "archived", +]); +export function settingsCategory(pathname: string): string | null { + if (!pathname.startsWith("/settings/")) return null; + const section = pathname.split("/")[2] ?? ""; + return SECTIONS.has(section) ? section : "other"; +} diff --git a/docs/internals/product-analytics.md b/docs/internals/product-analytics.md index 1b9e65cda..5da283fe5 100644 --- a/docs/internals/product-analytics.md +++ b/docs/internals/product-analytics.md @@ -61,7 +61,7 @@ protections below. Final copy and layout require human review before activation. `contract.ts` normalizes raw call-site values. `wireContract.ts` is the strict persisted/wire validator; the website gateway consumes its generated copy. -Revision 2 has 45 registered names, while the envelope remains schema version 1. +Revision 3 adds product insight signals while the envelope remains schema version 1. Legacy events may omit `contractRevision`; new events carry the bounded revision. Unrecognized/custom model and build labels become safe categories, not raw text. @@ -71,8 +71,8 @@ registered event. Regenerate and compare both repositories from the desktop root ```sh node packages/scient-analytics/src/generateConformance.ts \ --wire=/absolute/website/workers/events/src/eventContract.ts \ - packages/scient-analytics/fixtures/contract-v2.json \ - /absolute/website/workers/events/fixtures/contract-v2.json + packages/scient-analytics/fixtures/contract-v3.json \ + /absolute/website/workers/events/fixtures/contract-v3.json # Repeat with --check to verify exact source/corpus parity without writing. ``` @@ -130,6 +130,58 @@ measured performance on every supported platform. ## Instrumentation and honest coverage +### Product insights (revision 3) + +`panel.viewed`, `settings.viewed`, `feature.viewed` and `usage.viewed` measure +visible category entries, not clicks, dwell time or successful work. Consecutive +identical categories are coalesced while mounted; reopening or returning from a +hidden document can create a new observation. A visible restored panel qualifies, +background tabs do not. Actor is deliberately not inferred. Settings paths map +to fixed sections; nested project/provider identifiers and search text never +leave the client. Global Settings/Usage observations belong to the primary +environment; panel observations belong to the panel's environment. + +The view observer reads current visibility after coalesced consent discovery; +leaving before discovery does not replay the abandoned view. At most 16 view +observers are retained per connection. These events carry the existing ephemeral +collection context and are rejected after consent changes/deletion. They use the +existing bounded transport, not a clickstream SDK, scan, poller or shutdown-only +summary. `usage.refresh.requested` is an explicit refresh request, not success; +`usage.availability` describes a settled non-Limits view's summary availability, +not completeness of all token reporting. Limits sources retain separate coverage. + +`provider.turn.usage` uses ProviderService's existing instance-aware completion +and model association. Its upstream `provider.turn.completed` input is mapped to +usage, never a second successful outcome. Counts are normalized main-agent +input/output, optional cache read/write and reasoning subsets, with +complete/partial/unavailable status. Each count is an integer from 0 to one +billion; invalid values are omitted rather than clamped. Unknown counts are not +zero. Cache and reasoning are already included in input/output and must not be +added again. Private or mixed model labels become `other`; absent labels remain +`unknown`. No transcript scan, raw usage object, cost estimate or subagent total +is exported. Product consent must cover the observed turn interval; epoch +changes suppress deferred totals. Missing/evicted starts emit unavailable usage +rather than attributing an unproven interval. Terminal failures/stops can consume +tokens but usage remains Product-class, separate from Essential failures. + +Existing provider discovery/readiness events mean observed ready, not a current +signed-in account inventory; existing sent-turn events measure attempted model +use and attachment-count buckets. Existing voice/import/compute/export outcomes +remain their authoritative measurements. This pass does not add generic error +capture, arbitrary preference values, precise attention tracking, or reconstruct +unreported history. Offline/retry durability starts at local outbox acceptance; +renderer disconnects, queue caps, expiry and abrupt termination can still lose +observations. Never promise lossless telemetry at the expense of product work. + +The gateway report `analytics:insights` and prepared PostHog insights use +revision-3 Product participants, complete reporting days, distinct installation +profiles and event-ID deduplication. Repeated use means at least two distinct +days, not runtime sessions. The observed population is not feature eligibility +or all users. D1 is UTC; qualify the PostHog project timezone before comparing. +Deploy the generated validator before releasing these producers. No deployment, +activation, dashboard installation or platform-wide performance proof follows +from the source implementation alone. + | Source owner | Observed meaning | Important limit | | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | | `AnalyticsService` and startup | Server session, startup outcome, heartbeat | A missing graceful shutdown does not prove a crash | @@ -348,7 +400,7 @@ adds a reviewed, privacy-bounded contract for them. ## Privacy invariants -Analytics must never include prompts, responses, file contents or paths, URLs, tokens, email +Analytics must never include prompts, responses, file contents or paths, URLs, credential tokens, email addresses, provider account identifiers, or user-assigned device names. Provider and model values are normalized into bounded families; unrecognized values collapse to safe categories. New inherited instrumentation is not automatically authorized by appearing in upstream code: it must either map to diff --git a/docs/user/telemetry.md b/docs/user/telemetry.md index 7292a46cb..a3711c902 100644 --- a/docs/user/telemetry.md +++ b/docs/user/telemetry.md @@ -8,8 +8,10 @@ Where analytics are deliberately enabled, Scient accepts only its registered eve properties, subject to the configured consent level. Prompts, responses, file contents, credentials, and raw provider events are not accepted as product analytics. -Product analytics are separate from the usage totals shown inside the app and from local resource -diagnostics. Seeing those totals does not mean they are being uploaded. +Product analytics can include reported token counts for work performed through +Scient, provider/model categories, and which features or Settings sections you +visit. They do not upload the Usage page's broader local transcript history or +private custom model names. Missing provider counts stay unknown. Use **Share usage and reliability** in Settings → General → Privacy and analytics to turn sharing off or on. Sharing covers feature usage, reliability and diff --git a/packages/contracts/src/scientAnalytics.ts b/packages/contracts/src/scientAnalytics.ts index a3bcfe110..01daeae47 100644 --- a/packages/contracts/src/scientAnalytics.ts +++ b/packages/contracts/src/scientAnalytics.ts @@ -33,6 +33,12 @@ export const ScientAnalyticsUiEventName = Schema.Literals([ "voice.transcription.failed", "voice.transcription.cancelled", "surface.opened", + "panel.viewed", + "settings.viewed", + "usage.viewed", + "usage.refresh.requested", + "usage.availability", + "feature.viewed", "setting.changed", "scient.operation.started", "scient.operation.completed", diff --git a/packages/scient-analytics/fixtures/contract-v3.json b/packages/scient-analytics/fixtures/contract-v3.json new file mode 100644 index 000000000..d7840007f --- /dev/null +++ b/packages/scient-analytics/fixtures/contract-v3.json @@ -0,0 +1,1455 @@ +{ + "schemaVersion": 1, + "contractRevision": "3", + "cases": [ + { + "case": "app.session.started:fallbacks", + "name": "app.session.started", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "appVersion": "0.6.8", + "buildChannel": "stable", + "platform": "other", + "architecture": "other", + "contractRevision": "3" + } + }, + { + "case": "app.session.started:representative", + "name": "app.session.started", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "appVersion": "0.6.8", + "buildChannel": "stable", + "platform": "macos", + "architecture": "arm64", + "contractRevision": "3" + } + }, + { + "case": "app.session.ended:fallbacks", + "name": "app.session.ended", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "durationBucket": "unknown", + "shutdownClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "app.session.ended:representative", + "name": "app.session.ended", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "durationBucket": "5-15s", + "shutdownClass": "graceful", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "app.health:fallbacks", + "name": "app.health", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "component": "unknown", + "operation": "unknown", + "outcome": "unknown", + "failureClass": "unknown", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "app.health:representative", + "name": "app.health", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "component": "server", + "operation": "startup", + "outcome": "completed", + "failureClass": "permission", + "durationBucket": "5-15s", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "app.diagnostics:fallbacks", + "name": "app.diagnostics", + "privacyLevel": "diagnostic", + "consentLevel": "diagnostic", + "properties": { + "queuedCountBucket": "unknown", + "droppedCountBucket": "unknown", + "retryCountBucket": "unknown", + "deliveryClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "app.diagnostics:representative", + "name": "app.diagnostics", + "privacyLevel": "diagnostic", + "consentLevel": "diagnostic", + "properties": { + "queuedCountBucket": "4-10", + "droppedCountBucket": "2-3", + "retryCountBucket": "1", + "deliveryClass": "network", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "server.boot.heartbeat:fallbacks", + "name": "server.boot.heartbeat", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "server.boot.heartbeat:representative", + "name": "server.boot.heartbeat", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.session.started:fallbacks", + "name": "provider.session.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "runtimeMode": "other", + "hasResumeCursor": false, + "hasCwd": false, + "hasModel": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.session.started:representative", + "name": "provider.session.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "runtimeMode": "full-access", + "hasResumeCursor": true, + "hasCwd": true, + "hasModel": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.session.recovered:fallbacks", + "name": "provider.session.recovered", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "strategy": "resume-thread", + "hasResumeCursor": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.session.recovered:representative", + "name": "provider.session.recovered", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "strategy": "adopt-existing", + "hasResumeCursor": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.session.stopped:fallbacks", + "name": "provider.session.stopped", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.session.stopped:representative", + "name": "provider.session.stopped", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.sessions.stopped_all:fallbacks", + "name": "provider.sessions.stopped_all", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "sessionCountBucket": "unknown", + "shutdownClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.sessions.stopped_all:representative", + "name": "provider.sessions.stopped_all", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "sessionCountBucket": "4-10", + "shutdownClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.runtime_mode.changed:fallbacks", + "name": "provider.runtime_mode.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "from": "other", + "to": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.runtime_mode.changed:representative", + "name": "provider.runtime_mode.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "from": "other", + "to": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.sent:fallbacks", + "name": "provider.turn.sent", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "modelFamily": "unknown", + "modelKey": "unknown", + "interactionMode": "unknown", + "runtimeMode": "other", + "attachmentCountBucket": "unknown", + "hasInput": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.sent:representative", + "name": "provider.turn.sent", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "modelFamily": "openai", + "modelKey": "gpt-5.6-sol", + "interactionMode": "plan", + "runtimeMode": "full-access", + "attachmentCountBucket": "1", + "hasInput": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.completed:fallbacks", + "name": "provider.turn.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "modelKey": "unknown", + "durationBucket": "unknown", + "usedTools": false, + "hasAttachment": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.completed:representative", + "name": "provider.turn.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "modelKey": "gpt-5.6-sol", + "durationBucket": "5-15s", + "usedTools": true, + "hasAttachment": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.failed:fallbacks", + "name": "provider.turn.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "other", + "modelKey": "unknown", + "failureClass": "unknown", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.failed:representative", + "name": "provider.turn.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "antigravity", + "modelKey": "gpt-5.6-sol", + "failureClass": "unknown", + "durationBucket": "5-15s", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.stopped:fallbacks", + "name": "provider.turn.stopped", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "modelKey": "unknown", + "durationBucket": "unknown", + "stopClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.stopped:representative", + "name": "provider.turn.stopped", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "modelKey": "gpt-5.6-sol", + "durationBucket": "5-15s", + "stopClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.interrupted:fallbacks", + "name": "provider.turn.interrupted", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "initiator": "user", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.interrupted:representative", + "name": "provider.turn.interrupted", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "initiator": "user", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.request.responded:fallbacks", + "name": "provider.request.responded", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "requestKind": "approval", + "decision": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.request.responded:representative", + "name": "provider.request.responded", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "requestKind": "approval", + "decision": "approved", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.conversation.rolled_back:fallbacks", + "name": "provider.conversation.rolled_back", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "turnCountBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.conversation.rolled_back:representative", + "name": "provider.conversation.rolled_back", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "turnCountBucket": "2-3", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.discovered:fallbacks", + "name": "provider.discovered", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "runtimeSource": "unknown", + "state": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.discovered:representative", + "name": "provider.discovered", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "runtimeSource": "scient_managed", + "state": "ready", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.readiness.changed:fallbacks", + "name": "provider.readiness.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "from": "unknown", + "to": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.readiness.changed:representative", + "name": "provider.readiness.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "from": "unknown", + "to": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.runtime.source.changed:fallbacks", + "name": "provider.runtime.source.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "from": "unknown", + "to": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.runtime.source.changed:representative", + "name": "provider.runtime.source.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "from": "unknown", + "to": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.started:fallbacks", + "name": "provider.lifecycle.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "action": "unknown", + "runtimeSource": "unknown", + "stage": "unknown", + "failureClass": "unknown", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.started:representative", + "name": "provider.lifecycle.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "action": "repair", + "runtimeSource": "scient_managed", + "stage": "downloading", + "failureClass": "permission", + "durationBucket": "5-15s", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.completed:fallbacks", + "name": "provider.lifecycle.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "action": "unknown", + "runtimeSource": "unknown", + "stage": "unknown", + "failureClass": "unknown", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.completed:representative", + "name": "provider.lifecycle.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "action": "repair", + "runtimeSource": "scient_managed", + "stage": "downloading", + "failureClass": "permission", + "durationBucket": "5-15s", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.failed:fallbacks", + "name": "provider.lifecycle.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "other", + "action": "unknown", + "runtimeSource": "unknown", + "stage": "unknown", + "failureClass": "unknown", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.failed:representative", + "name": "provider.lifecycle.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "antigravity", + "action": "repair", + "runtimeSource": "scient_managed", + "stage": "downloading", + "failureClass": "permission", + "durationBucket": "5-15s", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.cancelled:fallbacks", + "name": "provider.lifecycle.cancelled", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "action": "unknown", + "runtimeSource": "unknown", + "stage": "unknown", + "failureClass": "unknown", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.lifecycle.cancelled:representative", + "name": "provider.lifecycle.cancelled", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "action": "repair", + "runtimeSource": "scient_managed", + "stage": "downloading", + "failureClass": "permission", + "durationBucket": "5-15s", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.added:fallbacks", + "name": "project.added", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "method": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.added:representative", + "name": "project.added", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "method": "picker", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.add.failed:fallbacks", + "name": "project.add.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "stage": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.add.failed:representative", + "name": "project.add.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "stage": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.opened:fallbacks", + "name": "project.opened", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "projectState": "unknown", + "initializationState": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.opened:representative", + "name": "project.opened", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "projectState": "existing", + "initializationState": "initialized", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.initialization.completed:fallbacks", + "name": "project.initialization.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "outcome": "unknown", + "filesCreatedBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.initialization.completed:representative", + "name": "project.initialization.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "outcome": "unknown", + "filesCreatedBucket": "2-3", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.initialization.failed:fallbacks", + "name": "project.initialization.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "failureClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "project.initialization.failed:representative", + "name": "project.initialization.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "failureClass": "permission", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.created:fallbacks", + "name": "thread.created", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "creationSource": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.created:representative", + "name": "thread.created", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "creationSource": "new", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.fork.completed:fallbacks", + "name": "thread.fork.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "workspaceMode": "unknown", + "boundaryClass": "unknown", + "refork": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.fork.completed:representative", + "name": "thread.fork.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "workspaceMode": "local", + "boundaryClass": "checkpoint", + "refork": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.fork.failed:fallbacks", + "name": "thread.fork.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "workspaceMode": "unknown", + "failureClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.fork.failed:representative", + "name": "thread.fork.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "workspaceMode": "local", + "failureClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.revert.completed:fallbacks", + "name": "thread.revert.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "boundaryClass": "checkpoint", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.revert.completed:representative", + "name": "thread.revert.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "boundaryClass": "checkpoint", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.revert.failed:fallbacks", + "name": "thread.revert.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "failureClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "thread.revert.failed:representative", + "name": "thread.revert.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "failureClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.started:fallbacks", + "name": "voice.transcription.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "engineClass": "other", + "languageMode": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.started:representative", + "name": "voice.transcription.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "engineClass": "local-whisper", + "languageMode": "automatic", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.completed:fallbacks", + "name": "voice.transcription.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "engineClass": "other", + "durationBucket": "unknown", + "audioDurationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.completed:representative", + "name": "voice.transcription.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "engineClass": "local-whisper", + "durationBucket": "5-15s", + "audioDurationBucket": "5-15s", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.failed:fallbacks", + "name": "voice.transcription.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "engineClass": "other", + "failureClass": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.failed:representative", + "name": "voice.transcription.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "engineClass": "local-whisper", + "failureClass": "permission", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.cancelled:fallbacks", + "name": "voice.transcription.cancelled", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "stage": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "voice.transcription.cancelled:representative", + "name": "voice.transcription.cancelled", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "stage": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "surface.opened:fallbacks", + "name": "surface.opened", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "surface": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "surface.opened:representative", + "name": "surface.opened", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "surface": "preview", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "panel.viewed:fallbacks", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "panel.viewed:representative", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "browser", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "settings.viewed:fallbacks", + "name": "settings.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "section": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "settings.viewed:representative", + "name": "settings.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "section": "providers", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "usage.viewed:fallbacks", + "name": "usage.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "metric": "other", + "window": "other", + "breakdown": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "usage.viewed:representative", + "name": "usage.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "metric": "tokens", + "window": "7", + "breakdown": "model", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "usage.refresh.requested:fallbacks", + "name": "usage.refresh.requested", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "usage.refresh.requested:representative", + "name": "usage.refresh.requested", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "usage.availability:fallbacks", + "name": "usage.availability", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "state": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "usage.availability:representative", + "name": "usage.availability", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "state": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "feature.viewed:fallbacks", + "name": "feature.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "feature": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "feature.viewed:representative", + "name": "feature.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "feature": "search", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.usage:fallbacks", + "name": "provider.turn.usage", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "modelKey": "unknown", + "terminalStatus": "other", + "usageStatus": "unavailable", + "usageScope": "main_agent", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "provider.turn.usage:representative", + "name": "provider.turn.usage", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "modelKey": "gpt-5.6-sol", + "terminalStatus": "other", + "usageStatus": "complete", + "usageScope": "main_agent", + "hasSubagents": false, + "inputTokens": 1500, + "outputTokens": 100, + "cachedInputTokens": 500, + "reasoningTokens": 50, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "setting.changed:fallbacks", + "name": "setting.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "setting": "unknown", + "value": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "setting.changed:representative", + "name": "setting.changed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "setting": "direction", + "value": "rtl", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.started:fallbacks", + "name": "scient.operation.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "other", + "trigger": "other", + "durationBucket": "unknown", + "failureClass": "unknown", + "reviewRequired": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.started:representative", + "name": "scient.operation.started", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "latex-build", + "trigger": "agent", + "durationBucket": "5-15s", + "failureClass": "permission", + "reviewRequired": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.completed:fallbacks", + "name": "scient.operation.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "other", + "trigger": "other", + "durationBucket": "unknown", + "failureClass": "unknown", + "reviewRequired": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.completed:representative", + "name": "scient.operation.completed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "latex-build", + "trigger": "agent", + "durationBucket": "5-15s", + "failureClass": "permission", + "reviewRequired": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.failed:fallbacks", + "name": "scient.operation.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "operationKind": "other", + "trigger": "other", + "durationBucket": "unknown", + "failureClass": "unknown", + "reviewRequired": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.failed:representative", + "name": "scient.operation.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "operationKind": "latex-build", + "trigger": "agent", + "durationBucket": "5-15s", + "failureClass": "permission", + "reviewRequired": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.cancelled:fallbacks", + "name": "scient.operation.cancelled", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "other", + "trigger": "other", + "durationBucket": "unknown", + "failureClass": "unknown", + "reviewRequired": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.cancelled:representative", + "name": "scient.operation.cancelled", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "latex-build", + "trigger": "agent", + "durationBucket": "5-15s", + "failureClass": "permission", + "reviewRequired": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.skipped:fallbacks", + "name": "scient.operation.skipped", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "other", + "trigger": "other", + "durationBucket": "unknown", + "failureClass": "unknown", + "reviewRequired": false, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + }, + { + "case": "scient.operation.skipped:representative", + "name": "scient.operation.skipped", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "operationKind": "latex-build", + "trigger": "agent", + "durationBucket": "5-15s", + "failureClass": "permission", + "reviewRequired": true, + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "3" + } + } + ] +} diff --git a/packages/scient-analytics/src/conformance.test.ts b/packages/scient-analytics/src/conformance.test.ts index a1523c65c..0e8e66c42 100644 --- a/packages/scient-analytics/src/conformance.test.ts +++ b/packages/scient-analytics/src/conformance.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "@effect/vitest"; -import fixture from "../fixtures/contract-v2.json" with { type: "json" }; +import fixture from "../fixtures/contract-v3.json" with { type: "json" }; import { buildAnalyticsConformanceFixture } from "./conformance.ts"; import { ANALYTICS_EVENT_NAMES, consentAllows, normalizeInheritedEvent } from "./contract.ts"; import { eventContractViolation } from "./wireContract.ts"; diff --git a/packages/scient-analytics/src/conformance.ts b/packages/scient-analytics/src/conformance.ts index b0bb644aa..985bdf227 100644 --- a/packages/scient-analytics/src/conformance.ts +++ b/packages/scient-analytics/src/conformance.ts @@ -8,6 +8,18 @@ import { const context = { appVersion: "0.6.8", buildChannel: "stable" } as const; const representativeProperties = { provider: "antigravity", + usageStatus: "complete", + inputTokens: 1500, + outputTokens: 100, + cachedInputTokens: 500, + reasoningTokens: 50, + hasSubagents: false, + category: "browser", + section: "providers", + feature: "search", + metric: "tokens", + window: "7", + breakdown: "model", model: "gpt-5.6-sol", runtimeMode: "full-access", interactionMode: "plan", diff --git a/packages/scient-analytics/src/contract.test.ts b/packages/scient-analytics/src/contract.test.ts index ab4601090..2b96377d4 100644 --- a/packages/scient-analytics/src/contract.test.ts +++ b/packages/scient-analytics/src/contract.test.ts @@ -16,7 +16,17 @@ describe("Scient analytics contract", () => { { provider: "pi", terminalStatus }, context, ), - ).toBeNull(); + ).toMatchObject({ + name: "provider.turn.usage", + privacyLevel: "product", + properties: { + terminalStatus: + terminalStatus === "cancelled" || terminalStatus === "interrupted" + ? "stopped" + : terminalStatus, + usageStatus: "unavailable", + }, + }); } expect( normalizeInheritedEvent("provider.turn.completed", { provider: "pi" }, context)?.name, @@ -105,7 +115,7 @@ describe("Scient analytics contract", () => { expect(surface?.properties).toEqual({ surface: "settings", ...context, - contractRevision: "2", + contractRevision: "3", }); }); diff --git a/packages/scient-analytics/src/contract.ts b/packages/scient-analytics/src/contract.ts index 4fb79ea70..7f9b45a5c 100644 --- a/packages/scient-analytics/src/contract.ts +++ b/packages/scient-analytics/src/contract.ts @@ -1,6 +1,8 @@ export const ANALYTICS_SCHEMA_VERSION = 1 as const; export const ANALYTICS_SOURCE = "desktop" as const; -export const ANALYTICS_CONTRACT_REVISION = "2" as const; +import { EVENT_DEFINITIONS } from "./wireContract.ts"; + +export const ANALYTICS_CONTRACT_REVISION = "3" as const; export const ANALYTICS_EVENT_NAMES = [ "app.session.started", @@ -42,6 +44,13 @@ export const ANALYTICS_EVENT_NAMES = [ "voice.transcription.failed", "voice.transcription.cancelled", "surface.opened", + "panel.viewed", + "settings.viewed", + "usage.viewed", + "usage.refresh.requested", + "usage.availability", + "feature.viewed", + "provider.turn.usage", "setting.changed", "scient.operation.started", "scient.operation.completed", @@ -64,7 +73,7 @@ export interface AnalyticsEvent { readonly occurred_at: string; readonly privacy_level: EventPrivacyLevel; readonly consent_level: EventPrivacyLevel; - readonly properties: Readonly>; + readonly properties: Readonly>; } export interface AnalyticsBatch { @@ -82,7 +91,7 @@ export interface NormalizedEvent { readonly name: string; readonly privacyLevel: EventPrivacyLevel; readonly priority: AnalyticsPriority; - readonly properties: Readonly>; + readonly properties: Readonly>; } const PROVIDERS = new Set([ @@ -294,6 +303,10 @@ const KNOWN_MODEL_KEYS = new Set([ "gpt-5.6-sol", "gpt-5.6-terra", "grok-build", + "gemini-3.1-pro", + "gemini-3.7-flash", + "gemini-3.7-pro", + "gemini-3.8-flash", "openai/gpt-5", ]); @@ -392,7 +405,13 @@ function modelFamily(provider: string, model: unknown): string { export function modelKey(model: unknown): string { if (typeof model !== "string") return "unknown"; const normalized = model.trim().toLowerCase(); - const withoutPinnedVersion = normalized.replace(/-(?:20\d{6,8})$/u, ""); + if (KNOWN_MODEL_KEYS.has(normalized)) return normalized; + // Strip only known public namespaces; the result still must be allowlisted. + const publicSlug = normalized.replace(/^(?:openai|anthropic|google)\//u, ""); + const withoutPinnedVersion = publicSlug.replace(/-(?:20\d{6,8})$/u, ""); + const geminiVariant = withoutPinnedVersion.replace(/-(?:high|medium|low)$/u, ""); + if (geminiVariant.startsWith("gemini-") && KNOWN_MODEL_KEYS.has(geminiVariant)) + return geminiVariant; const canonical = MODEL_KEY_ALIASES[normalized] ?? MODEL_KEY_ALIASES[withoutPinnedVersion] ?? @@ -640,10 +659,95 @@ function normalizeEvent( hasInput: normalizedBoolean(property(input, "hasInput")), }, }; + case "usage.refresh.requested": { + return { name, privacyLevel: "product", priority: "core", properties: {} }; + } + case "panel.viewed": + case "settings.viewed": + case "usage.viewed": + case "usage.availability": + case "feature.viewed": { + const properties: Record = {}; + for (const [key, rule] of Object.entries(EVENT_DEFINITIONS[name].properties)) { + properties[key] = normalizedEnum(property(input, key), new Set(rule.values), "other"); + } + return { name, privacyLevel: "product", priority: "core", properties }; + } + case "provider.turn.usage": { + const properties: Record = { + provider, + modelKey: + property(input, "mixedModels") === true ? "other" : modelKey(property(input, "model")), + terminalStatus: + normalizedEnum( + property(input, "terminalStatus"), + new Set(["cancelled", "interrupted"]), + "other", + ) !== "other" + ? "stopped" + : normalizedEnum( + property(input, "terminalStatus"), + new Set(["completed", "failed", "stopped"]), + "other", + ), + usageStatus: normalizedEnum( + property(input, "usageStatus"), + new Set(["complete", "partial", "unavailable"]), + "unavailable", + ), + usageScope: "main_agent", + }; + if (typeof property(input, "hasSubagents") === "boolean") + properties.hasSubagents = property(input, "hasSubagents") as boolean; + for (const key of [ + "inputTokens", + "outputTokens", + "cachedInputTokens", + "cacheCreationTokens", + "reasoningTokens", + ]) { + const value = property(input, key); + if ( + typeof value === "number" && + Number.isSafeInteger(value) && + value >= 0 && + value <= 1_000_000_000 + ) + properties[key] = value; + } + for (const [subset, total] of [ + ["cachedInputTokens", "inputTokens"], + ["cacheCreationTokens", "inputTokens"], + ["reasoningTokens", "outputTokens"], + ] as const) { + if ( + typeof properties[subset] === "number" && + typeof properties[total] === "number" && + properties[subset] > properties[total] + ) + delete properties[subset]; + } + if ( + properties.usageStatus === "complete" && + (properties.inputTokens === undefined || properties.outputTokens === undefined) + ) + properties.usageStatus = "partial"; + const hasCounts = [ + "inputTokens", + "outputTokens", + "cachedInputTokens", + "cacheCreationTokens", + "reasoningTokens", + ].some((key) => properties[key] !== undefined); + if (!hasCounts) properties.usageStatus = "unavailable"; + else if (properties.usageStatus === "unavailable") properties.usageStatus = "partial"; + return { name, privacyLevel: "product", priority: "core", properties }; + } case "provider.turn.completed": - // Upstream uses this name for all terminal statuses. Scient's semantic - // observer owns completed/failed/stopped outcomes; do not count both paths. - if (property(input, "terminalStatus") !== undefined) return null; + // Keep outcomes owned by the semantic observer; reuse upstream's + // instance-aware terminal/model association only for product usage. + if (property(input, "terminalStatus") !== undefined) + return normalizeEvent("provider.turn.usage", input, context); return { name, privacyLevel: "product", diff --git a/packages/scient-analytics/src/generateConformance.ts b/packages/scient-analytics/src/generateConformance.ts index 18a40cf2e..7e8e4d73e 100644 --- a/packages/scient-analytics/src/generateConformance.ts +++ b/packages/scient-analytics/src/generateConformance.ts @@ -5,7 +5,7 @@ import * as NodeURL from "node:url"; import { buildAnalyticsConformanceFixture } from "./conformance.ts"; -const defaultPath = NodeURL.fileURLToPath(new URL("../fixtures/contract-v2.json", import.meta.url)); +const defaultPath = NodeURL.fileURLToPath(new URL("../fixtures/contract-v3.json", import.meta.url)); const args = process.argv.slice(2); const check = args.includes("--check"); const wireArgument = args.find((arg) => arg.startsWith("--wire=")); diff --git a/packages/scient-analytics/src/insights.test.ts b/packages/scient-analytics/src/insights.test.ts new file mode 100644 index 000000000..e7540df8c --- /dev/null +++ b/packages/scient-analytics/src/insights.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "@effect/vitest"; +import { normalizeInheritedEvent, consentAllows, modelKey } from "./contract.ts"; +import { eventContractViolation } from "./wireContract.ts"; + +const context = { appVersion: "0.6.10", buildChannel: "stable" } as const; +describe("product insight contract", () => { + it("recognizes bounded public namespaces and Antigravity variants without private labels", () => { + expect(modelKey("openai/gpt-5.4")).toBe("gpt-5.4"); + expect(modelKey("anthropic/claude-sonnet-4-6")).toBe("claude-sonnet-4-6"); + expect(modelKey("gemini-3.8-flash-high")).toBe("gemini-3.8-flash"); + expect(modelKey("google/gemini-3.7-flash-low")).toBe("gemini-3.7-flash"); + expect(modelKey("private/gpt-5.4")).toBe("other"); + expect(modelKey("openai/PRIVATE")).toBe("other"); + }); + it("preserves exact reported usage without adding overlapping subsets", () => { + const event = normalizeInheritedEvent( + "provider.turn.completed", + { + provider: "codex", + model: "gpt-5.6-sol", + terminalStatus: "completed", + usageStatus: "complete", + inputTokens: 500, + outputTokens: 100, + cachedInputTokens: 300, + reasoningTokens: 75, + hasSubagents: true, + path: "PRIVATE", + usage: { prompt: "PRIVATE" }, + }, + context, + )!; + expect(event.name).toBe("provider.turn.usage"); + expect(event.properties).toMatchObject({ + inputTokens: 500, + outputTokens: 100, + cachedInputTokens: 300, + reasoningTokens: 75, + modelKey: "gpt-5.6-sol", + usageScope: "main_agent", + usageStatus: "complete", + }); + expect( + eventContractViolation({ + name: event.name, + properties: event.properties, + privacyLevel: event.privacyLevel, + consentLevel: "product", + }), + ).toBeNull(); + expect(consentAllows("essential", event.privacyLevel)).toBe(false); + expect(JSON.stringify(event)).not.toContain("PRIVATE"); + }); + it("does not fabricate zero usage or attribute mixed/private model totals", () => { + for (const value of [-1, 0.5, Infinity, NaN, 1_000_000_001, "500"]) { + const event = normalizeInheritedEvent( + "provider.turn.usage", + { model: "PRIVATE", usageStatus: "complete", inputTokens: value, outputTokens: 20 }, + context, + )!; + expect(event.properties.inputTokens).toBeUndefined(); + expect(event.properties.usageStatus).toBe("partial"); + expect(event.properties.modelKey).toBe("other"); + } + expect( + normalizeInheritedEvent("provider.turn.usage", {}, context)?.properties.usageStatus, + ).toBe("unavailable"); + expect( + normalizeInheritedEvent( + "provider.turn.usage", + { model: "gpt-5.6-sol", mixedModels: true }, + context, + )?.properties.modelKey, + ).toBe("other"); + }); + it("bounds all UI categories and strips paths and arbitrary settings", () => { + for (const name of ["panel.viewed", "settings.viewed", "feature.viewed", "usage.viewed"]) { + const event = normalizeInheritedEvent( + name, + { + category: "PRIVATE", + section: "PRIVATE", + feature: "PRIVATE", + metric: "PRIVATE", + path: "PRIVATE", + value: "PRIVATE", + }, + context, + )!; + expect(JSON.stringify(event)).not.toContain("PRIVATE"); + expect( + eventContractViolation({ + name, + properties: event.properties, + privacyLevel: "product", + consentLevel: "product", + }), + ).toBeNull(); + } + }); +}); diff --git a/packages/scient-analytics/src/outbox.test.ts b/packages/scient-analytics/src/outbox.test.ts index 4774b64b9..59eb46157 100644 --- a/packages/scient-analytics/src/outbox.test.ts +++ b/packages/scient-analytics/src/outbox.test.ts @@ -43,6 +43,35 @@ afterEach(() => { }); describe("AnalyticsOutbox", () => { + it("preserves numeric insight events and retry identity across restart without duplicating", () => { + const path = fixturePath(); + const normalized = normalizeInheritedEvent( + "provider.turn.usage", + { + provider: "codex", + usageStatus: "complete", + inputTokens: 12345, + outputTokens: 678, + cachedInputTokens: 200, + }, + { appVersion: "0.6.10", buildChannel: "stable" }, + )!; + const usage = { ...event("provider.turn.usage"), properties: normalized.properties }; + const first = new AnalyticsOutbox(path); + expect(first.enqueue(usage)).toBe(true); + expect(first.enqueue(usage)).toBe(false); + first.markFailed([usage.id], "network", 0); + first.close(); + const reopened = new AnalyticsOutbox(path); + const pending = reopened.pending(50, Date.now()); + expect(pending).toHaveLength(1); + expect(pending[0]).toMatchObject({ + id: usage.id, + attemptCount: 1, + properties: { inputTokens: 12345, outputTokens: 678, cachedInputTokens: 200 }, + }); + reopened.close(); + }); it("writes a coalesced batch transaction and delivers critical events first", () => { const outbox = new AnalyticsOutbox(fixturePath()); const events = [event("surface.opened"), event("app.health"), event("project.opened")]; diff --git a/packages/scient-analytics/src/outbox.ts b/packages/scient-analytics/src/outbox.ts index 894b5e9cc..73c712cf3 100644 --- a/packages/scient-analytics/src/outbox.ts +++ b/packages/scient-analytics/src/outbox.ts @@ -220,7 +220,10 @@ export class AnalyticsOutbox { parsed === null || Array.isArray(parsed) || Object.values(parsed).some( - (value) => typeof value !== "boolean" && typeof value !== "string", + (value) => + typeof value !== "boolean" && + typeof value !== "string" && + !(typeof value === "number" && Number.isSafeInteger(value)), ) ) { throw new Error("invalid-properties"); @@ -247,7 +250,7 @@ export class AnalyticsOutbox { occurred_at: row.occurred_at, privacy_level: row.privacy_level, consent_level: row.consent_level, - properties: parsed as Readonly>, + properties: parsed as Readonly>, attemptCount: row.attempt_count, priority: analyticsPriority(row.priority), }); diff --git a/packages/scient-analytics/src/wireContract.ts b/packages/scient-analytics/src/wireContract.ts index 960a2190d..8cefee379 100644 --- a/packages/scient-analytics/src/wireContract.ts +++ b/packages/scient-analytics/src/wireContract.ts @@ -4,6 +4,7 @@ export const PRIVACY_LEVELS = ["essential", "product", "diagnostic", "contributi export type PrivacyLevel = (typeof PRIVACY_LEVELS)[number]; type PropertyRule = + | { readonly kind: "integer"; readonly max: number; readonly optional?: boolean } | { readonly kind: "boolean"; readonly optional?: boolean } | { readonly kind: "enum"; readonly values: ReadonlyArray; readonly optional?: boolean } | { readonly kind: "pattern"; readonly pattern: RegExp; readonly optional?: boolean }; @@ -68,6 +69,10 @@ const modelKey = { "gpt-5.6-sol", "gpt-5.6-terra", "grok-build", + "gemini-3.1-pro", + "gemini-3.7-flash", + "gemini-3.7-pro", + "gemini-3.8-flash", "openai/gpt-5", "other", "unknown", @@ -481,6 +486,90 @@ export const EVENT_DEFINITIONS = { }, }, }, + "panel.viewed": { + privacyLevel: "product", + properties: { + category: { + kind: "enum", + values: [ + "browser", + "terminal", + "files", + "file-preview", + "diff", + "pull-request", + "agents", + "sources", + "compute", + "source-pdf", + "generated-pdf", + "artifact", + "other", + ], + }, + }, + }, + "settings.viewed": { + privacyLevel: "product", + properties: { + section: { + kind: "enum", + values: [ + "general", + "appearance", + "projects", + "keybindings", + "providers", + "custom-models", + "voice", + "skills", + "integrations", + "scientific-computing", + "source-control", + "connections", + "archived", + "other", + ], + }, + }, + }, + "usage.viewed": { + privacyLevel: "product", + properties: { + metric: { kind: "enum", values: ["tokens", "cost", "limits", "other"] }, + window: { kind: "enum", values: ["1", "7", "30", "90", "other"] }, + breakdown: { kind: "enum", values: ["model", "time", "other"] }, + }, + }, + "usage.refresh.requested": { privacyLevel: "product", properties: {} }, + "feature.viewed": { + privacyLevel: "product", + properties: { + feature: { kind: "enum", values: ["search", "project-picker", "new-thread", "other"] }, + }, + }, + "usage.availability": { + privacyLevel: "product", + properties: { + state: { kind: "enum", values: ["available", "partial", "unavailable", "other"] }, + }, + }, + "provider.turn.usage": { + privacyLevel: "product", + properties: { + provider, + modelKey, + terminalStatus: { kind: "enum", values: ["completed", "failed", "stopped", "other"] }, + usageStatus: { kind: "enum", values: ["complete", "partial", "unavailable"] }, + usageScope: { kind: "enum", values: ["main_agent"] }, + hasSubagents: { kind: "boolean", optional: true }, + inputTokens: { kind: "integer", max: 1_000_000_000, optional: true }, + outputTokens: { kind: "integer", max: 1_000_000_000, optional: true }, + cachedInputTokens: { kind: "integer", max: 1_000_000_000, optional: true }, + cacheCreationTokens: { kind: "integer", max: 1_000_000_000, optional: true }, + reasoningTokens: { kind: "integer", max: 1_000_000_000, optional: true }, + }, + }, "setting.changed": { privacyLevel: "product", properties: { @@ -527,6 +616,14 @@ const PRIVACY_RANK: Readonly> = { }; function propertyViolation(key: string, value: unknown, rule: PropertyRule): string | null { + if (rule.kind === "integer") { + return typeof value === "number" && + Number.isSafeInteger(value) && + value >= 0 && + value <= rule.max + ? null + : `Invalid event property '${key}'`; + } if (rule.kind === "boolean") { return typeof value === "boolean" ? null : `Invalid event property '${key}'`; } @@ -560,7 +657,7 @@ export function eventContractViolation(input: { const rules: Readonly> = { appVersion, buildChannel, - contractRevision: { kind: "enum", values: ["1", "2"], optional: true }, + contractRevision: { kind: "enum", values: ["1", "2", "3"], optional: true }, ...definition.properties, }; for (const key of Object.keys(input.properties)) { @@ -577,5 +674,36 @@ export function eventContractViolation(input: { const violation = propertyViolation(key, value, rule); if (violation) return violation; } + if (input.name === "provider.turn.usage") { + const p = input.properties; + const counts = [ + "inputTokens", + "outputTokens", + "cachedInputTokens", + "cacheCreationTokens", + "reasoningTokens", + ]; + if ( + p.usageStatus === "complete" && + (p.inputTokens === undefined || p.outputTokens === undefined) + ) + return "Incomplete token totals"; + if (p.usageStatus === "unavailable" && counts.some((key) => p[key] !== undefined)) + return "Unavailable usage includes counts"; + for (const [subset, total] of [ + ["cachedInputTokens", "inputTokens"], + ["cacheCreationTokens", "inputTokens"], + ["reasoningTokens", "outputTokens"], + ] as const) { + const subsetValue = p[subset]; + const totalValue = p[total]; + if ( + typeof subsetValue === "number" && + typeof totalValue === "number" && + subsetValue > totalValue + ) + return "Token subset exceeds total"; + } + } return null; }