Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions apps/server/src/provider/Layers/ProviderService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,10 @@ interface RecordedAnalyticsEvent {

function makeRecordingAnalytics() {
const events: Array<RecordedAnalyticsEvent> = [];
let epoch = 1;
const layer = Layer.mock(AnalyticsService.AnalyticsService)({
status: Effect.succeed({ available: true, consent: "product" as const }),
collectionEpoch: Effect.sync(() => epoch),
record: (event, properties) =>
Effect.sync(() => {
events.push({ event, ...(properties ? { properties } : {}) });
Expand All @@ -360,6 +363,10 @@ function makeRecordingAnalytics() {
layer,
reset: () => {
events.length = 0;
epoch = 1;
},
changeEpoch: () => {
epoch += 1;
},
eventsByName: (event: string) => events.filter((entry) => entry.event === event),
};
Expand Down Expand Up @@ -4171,6 +4178,46 @@ turnAnalytics.layer("ProviderServiceLive turn analytics", (it) => {
}),
);

it.effect("does not export turn usage across a collection epoch change", () =>
Effect.gen(function* () {
recordedTurnAnalytics.reset();
const provider = yield* ProviderService.ProviderService;
const threadId = asThreadId("thread-turn-analytics-consent-change");
yield* provider.startSession(threadId, {
provider: CODEX_DRIVER,
providerInstanceId: codexInstanceId,
threadId,
runtimeMode: "approval-required",
});
const turn = yield* provider.sendTurn({ threadId, input: "test", attachments: [] });
recordedTurnAnalytics.changeEpoch();
const received = yield* Stream.take(provider.streamEvents, 1).pipe(
Stream.runDrain,
Effect.forkChild,
);
yield* Effect.yieldNow;
primaryAnalyticsCodex.emit({
type: "turn.aborted",
eventId: asEventId("evt-consent-change"),
provider: CODEX_DRIVER,
createdAt: "2026-01-01T00:00:00.000Z",
threadId,
turnId: turn.turnId,
payload: {
reason: "Interrupted",
tokenUsage: {
usageStatus: "partial",
usageScope: "main_agent",
inputTokens: 120,
outputTokens: 30,
},
},
});
yield* Fiber.join(received);
assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0);
}),
);

it.effect("records known token counts for an interrupted turn", () =>
Effect.gen(function* () {
recordedTurnAnalytics.reset();
Expand Down
58 changes: 54 additions & 4 deletions apps/server/src/provider/Layers/ProviderService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ export interface ProviderServiceLiveOptions {
}

interface TurnAnalyticsMetadata {
readonly collectionEpoch?: number;
readonly requestId: number;
readonly provider: ProviderDriverKind;
readonly startedAtMs: number;
Expand Down Expand Up @@ -387,8 +388,23 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
state.sessions.delete(input.sessionKey);
}

const terminal = { ...input.completion.terminalProperties };
if (!metadata) {
// A terminal without its start cannot establish a consent-covered token
// interval (including bounded-map eviction). Keep coverage, not totals.
for (const key of [
"inputTokens",
"outputTokens",
"cachedInputTokens",
"cacheCreationTokens",
"reasoningTokens",
])
delete terminal[key];
terminal.usageStatus = "unavailable";
}
return {
...input.completion.terminalProperties,
...terminal,
collectionEpoch: metadata ? metadata.collectionEpoch : terminal.collectionEpoch,
...(metadata?.model ? { model: metadata.model } : {}),
...(metadata?.effort ? { effort: metadata.effort } : {}),
...(metadata?.interactionMode ? { interactionMode: metadata.interactionMode } : {}),
Expand All @@ -403,9 +419,22 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
const recordCompletedTurnProperties = (
properties: ReadonlyArray<Readonly<Record<string, unknown>>>,
) =>
Effect.forEach(properties, (entry) => analytics.record("provider.turn.completed", entry), {
discard: true,
});
Effect.forEach(
properties,
(entry) =>
Effect.gen(function* () {
const { collectionEpoch, ...properties } = entry;
if (
collectionEpoch === undefined ||
collectionEpoch !== (yield* analytics.collectionEpoch)
)
return;
yield* analytics.record("provider.turn.completed", properties);
}),
{
discard: true,
},
);

const clearTurnAnalyticsSession = (providerInstanceId: ProviderInstanceId, threadId: ThreadId) =>
Effect.gen(function* () {
Expand Down Expand Up @@ -434,6 +463,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
readonly runtimeMode: string | undefined;
}) {
const startedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
const analyticsStatus = yield* analytics.status;
const collectionEpoch =
analyticsStatus.available &&
(analyticsStatus.consent === "product" || analyticsStatus.consent === "diagnostic")
? yield* analytics.collectionEpoch
: undefined;
turnAnalyticsRequestId += 1;
const requestId = turnAnalyticsRequestId;
const effort = turnEffort(input.modelSelection);
Expand All @@ -445,6 +480,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
deferredCompletionsByTurnId: new Map(),
};
const metadata: TurnAnalyticsMetadata = {
...(collectionEpoch === undefined ? {} : { collectionEpoch }),
provider: input.provider,
startedAtMs,
mixedModels: false,
Expand Down Expand Up @@ -569,6 +605,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
) {
if (!event.turnId) return;
const observedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
const analyticsStatus = yield* analytics.status;
const collectionEpoch =
analyticsStatus.available &&
(analyticsStatus.consent === "product" || analyticsStatus.consent === "diagnostic")
? yield* analytics.collectionEpoch
: undefined;
yield* Ref.update(turnAnalytics, (state) => {
const completionKey = turnAnalyticsCompletionKey(
source.instanceId,
Expand All @@ -590,6 +632,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
const current = session.activeByTurnId.get(String(event.turnId));
const metadata: TurnAnalyticsMetadata = {
...(current?.metadata ?? {
...(collectionEpoch === undefined ? {} : { collectionEpoch }),
requestId: ++turnAnalyticsRequestId,
provider: source.provider,
startedAtMs: observedAtMs,
Expand Down Expand Up @@ -641,6 +684,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
) {
if (!event.turnId) return;
const completedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
const analyticsStatus = yield* analytics.status;
const collectionEpoch =
analyticsStatus.available &&
(analyticsStatus.consent === "product" || analyticsStatus.consent === "diagnostic")
? yield* analytics.collectionEpoch
: undefined;
const tokenUsage = event.payload.tokenUsage;
const completion: DeferredTurnAnalyticsCompletion = {
completionKey: turnAnalyticsCompletionKey(
Expand All @@ -650,6 +699,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
),
completedAtMs,
terminalProperties: {
collectionEpoch,
provider: source.provider,
terminalStatus:
event.type === "turn.completed"
Expand Down
14 changes: 14 additions & 0 deletions apps/server/src/telemetry/AnalyticsUiAdapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,20 @@ export function makeAnalyticsUiAdapter(analytics: AnalyticsService["Service"]) {
);

const record = Effect.fn("AnalyticsUiAdapter.record")(function* (event: ScientAnalyticsUiEvent) {
if (
event.name === "panel.viewed" ||
event.name === "settings.viewed" ||
event.name === "usage.viewed" ||
event.name === "feature.viewed" ||
event.name === "usage.availability"
) {
const current = yield* status;
if (
event.collectionContext === undefined ||
event.collectionContext !== current.collectionContext
)
return { accepted: false } as const;
}
if (event.name.startsWith("scient.operation.")) {
const current = yield* status;
if (
Expand Down
1 change: 1 addition & 0 deletions apps/web/src/components/ChatView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9191,6 +9191,7 @@ function ChatViewContent(props: ChatViewProps) {
>
<RightPanelTabs
mode="sheet"
open={rightPanelOpen}
// Same effective inset as the closed-state titlebar controls
// (pr-3 in the tab bar plus this pixel equals the absolute
// right inset plus mr-px), so the cluster does not creep when
Expand Down
16 changes: 16 additions & 0 deletions apps/web/src/components/CommandPalette.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"use client";

import { scopeProjectRef, scopeThreadRef } from "@t3tools/client-runtime/environment";
import { useScientAnalyticsView } from "~/scient/analytics/client";
import {
canCreateProjectInEnvironment,
getCloneDestinationBrowsePath,
Expand Down Expand Up @@ -428,6 +429,21 @@ export function CommandPalette({ children }: { children: ReactNode }) {
openIntent: null,
});
const setOpen = useCallback((open: boolean) => dispatch({ _tag: "SetOpen", open }), []);
useScientAnalyticsView(
state.open
? {
name: "feature.viewed",
properties: {
feature:
state.openIntent?.kind === "add-project"
? "project-picker"
: state.openIntent?.kind === "new-thread-in"
? "new-thread"
: "search",
},
}
: null,
);
const toggleMode = useCallback(
(mode: SearchOverlayMode) => dispatch({ _tag: "ToggleMode", mode }),
[],
Expand Down
14 changes: 14 additions & 0 deletions apps/web/src/components/RightPanelTabs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ import { useEnvironmentQuery } from "~/state/query";
import { COLLAPSED_SIDEBAR_TITLEBAR_INSET_CLASS } from "~/workspaceTitlebar";
import { ScientRightPanelSurfaceIcon } from "~/scient/rightPanel/ScientRightPanelSurfaceIcon";
import { scientRightPanelSurfaceTitle } from "~/scient/rightPanel/surfaces";
import { useScientAnalyticsView } from "~/scient/analytics/client";
import { panelCategory } from "~/scient/analytics/viewCategories";

import { PreviewPanelShell, type PreviewPanelMode } from "./preview/PreviewPanelShell";
import { FaviconImage } from "./preview/PreviewFaviconIcon";
Expand Down Expand Up @@ -803,6 +805,18 @@ function PullRequestSurfaceIcon({
}

export function RightPanelTabs(props: RightPanelTabsProps) {
const analyticsSurface =
props.open === false
? undefined
: props.surfaces.find((surface) => surface.id === props.activeSurfaceId);
useScientAnalyticsView(
analyticsSurface
? { name: "panel.viewed", properties: { category: panelCategory(analyticsSurface) } }
: null,
analyticsSurface?.kind === "pull-request" && analyticsSurface.environmentId
? (analyticsSurface.environmentId as EnvironmentId)
: props.environmentId,
);
const ownsDesktopTitleBar = isElectron && props.mode === "inline";
const browserProfiles = useBrowserDefaults().profiles;
const { resolvedTheme } = useTheme();
Expand Down
28 changes: 28 additions & 0 deletions apps/web/src/components/usage/UsagePage.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { RefreshIcon } from "~/components/ui/refresh-icon";
import { useRecordScientAnalytics, useScientAnalyticsView } from "~/scient/analytics/client";
import { useAtomValue } from "@effect/atom-react";
import {
USAGE_CONTRACT_VERSION,
Expand Down Expand Up @@ -108,12 +109,38 @@ export function UsagePage() {
const [selectedEnvironmentIds, setSelectedEnvironmentIds] =
useState<ReadonlySet<EnvironmentId> | null>(null);
const { days: windowDays, window } = windowSelection;
const recordAnalytics = useRecordScientAnalytics();
useScientAnalyticsView({
name: "usage.viewed",
properties: {
metric,
window: String(windowDays),
breakdown: showingLimits ? "other" : breakdown,
},
});
const isPast24Hours = windowDays === 1;
const { merged, environments, selectedEnvironments, isPending, isPartial, refresh } = useUsage(
window,
selectedEnvironmentIds,
);
const presentations = useAtomValue(environmentPresentations.presentationsAtom);
useScientAnalyticsView(
showingLimits ||
isPending ||
isRefreshing ||
selectedEnvironments.some((entry) => entry.isPending)
? null
: {
name: "usage.availability",
properties: {
state: !selectedEnvironments.some((entry) => entry.summary !== null)
? "unavailable"
: selectedEnvironments.some((entry) => entry.error !== null || entry.summary === null)
? "partial"
: "available",
},
},
);
const refreshProviders = useAtomCommand(serverEnvironment.refreshProviders, {
reportFailure: false,
});
Expand Down Expand Up @@ -164,6 +191,7 @@ export function UsagePage() {
};
const refreshWindow = () => {
if (refreshingRef.current) return;
recordAnalytics({ name: "usage.refresh.requested", properties: {} });

if (showingLimits) {
refreshingRef.current = true;
Expand Down
2 changes: 2 additions & 0 deletions apps/web/src/routes/__root.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { SshPasswordPromptDialog } from "../components/desktop/SshPasswordPrompt
import { DesktopAppActivationCoordinator } from "../components/desktop/DesktopAppActivationCoordinator";
import { ProviderUpdateLaunchNotification } from "../components/ProviderUpdateLaunchNotification";
import { AnalyticsSharingNotice } from "../scient/analytics/AnalyticsSharingNotice";
import { SettingsAnalyticsObserver } from "../scient/analytics/SettingsAnalyticsObserver";
import { SlowRpcRequestToastCoordinator } from "../components/SlowRpcRequestToastCoordinator";
import { ThemeEditorHost } from "../components/settings/ThemeEditorHost";
import { useCopyToClipboard } from "../hooks/useCopyToClipboard";
Expand Down Expand Up @@ -180,6 +181,7 @@ function RootRouteView() {
{primaryEnvironmentAuthenticated ? <PlanAgentSelectionHeal /> : null}
{primaryEnvironmentAuthenticated ? <ProviderUpdateLaunchNotification /> : null}
{primaryEnvironmentAuthenticated ? <AnalyticsSharingNotice /> : null}
{primaryEnvironmentAuthenticated ? <SettingsAnalyticsObserver /> : null}
{appShell}
{/* Above the router: a theme draft is judged by walking the app, so the
editor has to survive navigation away from settings. */}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,10 @@ describe("analytics sharing controls", () => {
await render();
await details();
const dialog = document.querySelector('[role="dialog"]');
expect(dialog?.textContent).toContain("Share which features you use");
expect(dialog?.textContent).toContain(
"Share which features and provider/model categories you use",
);
expect(dialog?.textContent).toContain("reported token counts");
expect(dialog?.querySelector("dl")).toBeNull();
expect(dialog?.textContent).toContain(
"Analytics never includes prompts, responses, file contents",
Expand Down
6 changes: 3 additions & 3 deletions apps/web/src/scient/analytics/AnalyticsSharingInfo.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ export function AnalyticsSharingInfo({ consent }: { consent: ScientAnalyticsCons
<PopoverTitle className="text-sm">What’s shared?</PopoverTitle>
<div className="mt-3 space-y-3 text-xs leading-relaxed text-muted-foreground">
<p>
Share which features you use, whether operations succeed or fail, basic performance
information, and counters that help check analytics delivery. Turn sharing off in
Settings to stop sending analytics.
Share which features and provider/model categories you use, reported token counts,
whether operations succeed or fail, basic performance information, and counters that
help check analytics delivery. Turn sharing off in Settings to stop sending analytics.
</p>
{consent === "essential" || consent === "product" ? (
<p>
Expand Down
11 changes: 11 additions & 0 deletions apps/web/src/scient/analytics/SettingsAnalyticsObserver.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { useLocation } from "@tanstack/react-router";
import { useScientAnalyticsView } from "./client";
import { settingsCategory } from "./viewCategories";

export function SettingsAnalyticsObserver() {
const section = settingsCategory(useLocation().pathname);
useScientAnalyticsView(
section === null ? null : { name: "settings.viewed", properties: { section } },
);
return null;
}
Loading
Loading