Skip to content

fix(analytics): count actual provider installations #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yaacovcorcos merged 1 commit into from
Sep 9, 2026
Merged

fix(analytics): count actual provider installations #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,10 @@ registered event name, its exact allowlisted property set, the event's declared
privacy level, and sufficient explicit consent. Unknown events, extra
properties, raw text, and mismatched consent or privacy classifications are
rejected before storage. The versioned registry and its focused tests live in
`workers/events/src/eventContract.ts`. Revision 3 is generated from
`workers/events/src/eventContract.ts`. Revision 4 is generated from
`scient-desktop/packages/scient-analytics/src/wireContract.ts`, with a shared
conformance fixture for every registered event, plus revision-2 compatibility tests. Do not edit that copy
conformance fixture for every registered event, plus revision-2 and revision-3
compatibility tests. Do not edit that copy
independently; the desktop analytics document owns regeneration instructions.
New events add an optional bounded `contractRevision`; legacy revision-1
payloads remain supported. Deploy this validator before releasing new producers.
Expand Down
19 changes: 13 additions & 6 deletions scripts/product-insights-hogql.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Same Product population and complete-day window as product-insights.mjs.
const population = `properties.source = 'desktop' AND properties.consent_level IN ('product', 'diagnostic')
AND properties.contractRevision = '3' AND timestamp >= toStartOfDay(now()) - INTERVAL 30 DAY AND timestamp < toStartOfDay(now())`;
AND properties.contractRevision IN ('3', '4') AND timestamp >= toStartOfDay(now()) - INTERVAL 30 DAY AND timestamp < toStartOfDay(now())`;
const insight = (name, description, query) => ({
name,
description,
Expand Down Expand Up @@ -33,11 +33,18 @@ AND event = 'provider.turn.usage' GROUP BY id)
GROUP BY provider ORDER BY reported_turns DESC`,
),
insight(
"Providers observed ready",
"Installations with a provider observed ready during the window. Not signed-in people, current connection inventory, or proof the provider was used.",
`SELECT properties.provider AS provider, uniqExact(distinct_id) AS installations
FROM events WHERE ${population} AND ((event = 'provider.discovered' AND properties.state = 'ready')
OR (event = 'provider.readiness.changed' AND properties.to = 'ready')) GROUP BY provider ORDER BY installations DESC`,
"Latest observed provider installations",
"Latest explicit installed state per pseudonymous installation and provider. Bundled-but-missing providers and historical readiness are not counted as installed.",
`SELECT provider, countIf(installed = true) AS installations
FROM (
SELECT distinct_id, properties.provider AS provider,
argMax(if(event = 'provider.installation.observed', properties.installed, properties.toInstalled),
tuple(timestamp, properties.event_id)) AS installed
FROM events WHERE ${population} AND properties.contractRevision = '4'
AND event IN ('provider.installation.observed', 'provider.installation.changed')
GROUP BY distinct_id, provider
)
GROUP BY provider HAVING installations > 0 ORDER BY installations DESC`,
),
];

Expand Down
33 changes: 25 additions & 8 deletions scripts/product-insights.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ import { fileURLToPath } from "node:url";
// Full UTC days avoid comparing today's partial activity with complete days.
export const productInsightsQuery = `
WITH eligible AS (
SELECT event_name, distinct_id, date(occurred_at) AS day, properties_json AS p
SELECT event_id, event_name, distinct_id, occurred_at, date(occurred_at) AS day,
properties_json AS p
FROM analytics_events
WHERE source = 'desktop' AND consent_level IN ('product', 'diagnostic')
AND julianday(occurred_at) >= julianday(date('now', '-30 days'))
AND julianday(occurred_at) < julianday(date('now'))
AND json_extract(properties_json, '$.contractRevision') = '3'
AND json_extract(properties_json, '$.contractRevision') IN ('3', '4')
), views AS (
SELECT event_name, distinct_id, day,
CASE event_name
Expand All @@ -34,6 +35,24 @@ WITH eligible AS (
json_extract(p, '$.inputTokens') AS input_tokens,
json_extract(p, '$.outputTokens') AS output_tokens
FROM eligible WHERE event_name = 'provider.turn.usage'
), provider_installation_events AS (
SELECT event_id, distinct_id, occurred_at, json_extract(p, '$.provider') AS provider,
CASE event_name
WHEN 'provider.installation.observed' THEN json_extract(p, '$.installed')
ELSE json_extract(p, '$.toInstalled') END AS installed
FROM eligible
WHERE json_extract(p, '$.contractRevision') = '4'
AND event_name IN ('provider.installation.observed', 'provider.installation.changed')
), latest_provider_installations AS (
SELECT distinct_id, provider, installed
FROM (
SELECT distinct_id, provider, installed,
row_number() OVER (
PARTITION BY distinct_id, provider ORDER BY occurred_at DESC, event_id DESC
) AS recency
FROM provider_installation_events
)
WHERE recency = 1
)
SELECT 'feature_observations' AS metric, event_name || ':' || category AS category,
count(*) AS installations, sum(observations) AS observations,
Expand All @@ -52,18 +71,16 @@ UNION ALL
SELECT 'reported_output_tokens', provider, count(DISTINCT distinct_id), count(output_tokens), NULL, sum(output_tokens)
FROM tokens GROUP BY provider
UNION ALL
SELECT 'provider_observed_ready', json_extract(p, '$.provider'), count(DISTINCT distinct_id), count(*), NULL, NULL
FROM eligible
WHERE event_name = 'provider.discovered' AND json_extract(p, '$.state') = 'ready'
OR event_name = 'provider.readiness.changed' AND json_extract(p, '$.to') = 'ready'
GROUP BY json_extract(p, '$.provider')
SELECT 'provider_latest_observed_installed', provider, count(DISTINCT distinct_id), count(*), NULL, NULL
FROM latest_provider_installations WHERE installed = 1
GROUP BY provider
UNION ALL
SELECT 'provider_terminal_outcomes', json_extract(p, '$.provider') || ':' || event_name,
count(DISTINCT distinct_id), count(*), NULL, NULL
FROM eligible WHERE event_name IN ('provider.turn.completed', 'provider.turn.failed', 'provider.turn.stopped')
GROUP BY json_extract(p, '$.provider'), event_name
UNION ALL
SELECT 'observed_product_population', 'revision-3', count(DISTINCT distinct_id), count(*), NULL, NULL
SELECT 'observed_product_population', 'revision-3-or-4', count(DISTINCT distinct_id), count(*), NULL, NULL
FROM eligible
ORDER BY metric, observations DESC, category
`;
Expand Down
30 changes: 26 additions & 4 deletions scripts/product-insights.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { expect, it } from "vitest";
import { testDatabase } from "../workers/events/src/sqlite.testSupport.ts";
import { productInsightsQuery } from "./product-insights.mjs";

it("separates adoption, repeat days, readiness, unknown tokens and failure populations", () => {
it("separates adoption, latest installation state, unknown tokens and failure populations", () => {
const store = testDatabase();
try {
const insert = store.sqlite.prepare(
Expand All @@ -28,8 +28,28 @@ it("separates adoption, repeat days, readiness, unknown tokens and failure popul
add("panel.viewed", { category: "browser" }, { consent: "essential" });
add("panel.viewed", { category: "browser" }, { revision: "2" });
add("panel.viewed", { category: "browser" }, { day: "2026-09-06" });
add("provider.discovered", { provider: "codex", state: "ready" });
add("provider.discovered", { provider: "pi", state: "unavailable" });
add(
"provider.installation.observed",
{ provider: "codex", installed: true },
{ revision: "4", day: "2026-09-05T09:00:00Z" },
);
add(
"provider.installation.changed",
{ provider: "codex", fromInstalled: true, toInstalled: false },
{ revision: "4", day: "2026-09-05T10:00:00Z" },
);
add(
"provider.installation.observed",
{ provider: "pi", installed: false },
{ revision: "4", installation: "PRIVATE-2", day: "2026-09-05T09:00:00Z" },
);
add(
"provider.installation.changed",
{ provider: "pi", fromInstalled: false, toInstalled: true },
{ revision: "4", installation: "PRIVATE-2", day: "2026-09-05T10:00:00Z" },
);
// Legacy readiness is not treated as an explicit current installation.
add("provider.discovered", { provider: "grok", state: "ready" });
add("provider.turn.usage", {
provider: "codex",
modelKey: "gpt-5.6-sol",
Expand Down Expand Up @@ -59,7 +79,9 @@ it("separates adoption, repeat days, readiness, unknown tokens and failure popul
expect(
rows.find((r) => r.metric === "reported_input_tokens" && r.category === "pi"),
).toMatchObject({ observations: 0, value: null });
expect(rows.filter((r) => r.metric === "provider_observed_ready")).toHaveLength(1);
expect(rows.filter((r) => r.metric === "provider_latest_observed_installed")).toEqual([
expect.objectContaining({ category: "pi", installations: 1, observations: 1 }),
]);
expect(rows.find((r) => r.metric === "provider_terminal_outcomes")).toMatchObject({
observations: 1,
});
Expand Down
Loading