From 124e956a3b9404c57ab0e3ff9654359270f6b7aa Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:01:29 +0000 Subject: [PATCH 1/7] fix: self-heal corrupt analytics rows that wallpaper the dashboard A rare native-layer corruption can materialize a phantom events row whose varchar columns concatenate every non-null value from an entire insert batch (observed: a 17KB model string spanning ~670 events). That single row renders as one giant legend entry in Spend over time, covering the whole Analytics dashboard. Add deleteCorruptAnalyticsRows: a length-threshold sweep over events and delegation_rollups, run at worker init and after each write cycle (ingest, syncCheck incremental/full_rebuild, rebuildAll). Donor rows are written correctly, so deleting the phantom loses no real data. Validated against the affected production DB: exactly 1 row deleted, 58,478 kept. --- .../services/analytics/analyticsWorker.ts | 17 ++++++++ src/node/services/analytics/etl.test.ts | 39 +++++++++++++++++++ src/node/services/analytics/etl.ts | 31 +++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/src/node/services/analytics/analyticsWorker.ts b/src/node/services/analytics/analyticsWorker.ts index b1d6ab53bf3..89d621439ff 100644 --- a/src/node/services/analytics/analyticsWorker.ts +++ b/src/node/services/analytics/analyticsWorker.ts @@ -6,6 +6,7 @@ import { decideSyncPlan, type SyncAction } from "./backfillDecision"; import { shouldCheckpointAfterSync } from "./checkpointDecision"; import { clearWorkspaceAnalyticsState, + deleteCorruptAnalyticsRows, getCurrentPricingFingerprint, ingestWorkspace, readStoredPricingFingerprint, @@ -147,6 +148,18 @@ async function handleInit(data: InitData): Promise { for (const migrationSql of DELEGATION_ROLLUPS_COLUMN_MIGRATIONS_SQL) { await activeConn.run(migrationSql); } + + await sweepCorruptRows("init"); +} + +/** Delete corruption-class rows and log when anything was actually removed. */ +async function sweepCorruptRows(context: string): Promise { + const deleted = await deleteCorruptAnalyticsRows(getConn()); + if (deleted > 0) { + process.stderr.write( + `[analytics-worker] Deleted ${deleted} corrupt analytics row(s) (${context})\n` + ); + } } async function handleIngest(data: IngestData): Promise { @@ -154,6 +167,7 @@ async function handleIngest(data: IngestData): Promise { assert(data.sessionDir.trim().length > 0, "ingest requires sessionDir"); await ingestWorkspace(getConn(), data.workspaceId, data.sessionDir, data.meta ?? {}); + await sweepCorruptRows("ingest"); } async function handleRebuildAll(data: RebuildAllData): Promise<{ workspacesIngested: number }> { @@ -169,6 +183,7 @@ async function handleRebuildAll(data: RebuildAllData): Promise<{ workspacesInges // A completed rebuild priced everything with the current tables; refresh the // fingerprint so the next sync check does not schedule a redundant rebuild. await storePricingFingerprint(getConn()); + await sweepCorruptRows("rebuildAll"); return result; } @@ -390,6 +405,7 @@ async function handleSyncCheck(data: SyncCheckData): Promise { if (pricingFingerprintChanged) { await storePricingFingerprint(getConn()); } + await sweepCorruptRows("syncCheck full_rebuild"); await checkpointIfNeeded(plan.action, workspacesIngested, 0); const elapsedMs = Math.round(performance.now() - syncStartMs); @@ -448,6 +464,7 @@ async function handleSyncCheck(data: SyncCheckData): Promise { } } + await sweepCorruptRows("syncCheck incremental"); await checkpointIfNeeded(plan.action, workspacesIngested, workspacesPurged); const elapsedMs = Math.round(performance.now() - syncStartMs); diff --git a/src/node/services/analytics/etl.test.ts b/src/node/services/analytics/etl.test.ts index 74b71631482..24360c35d84 100644 --- a/src/node/services/analytics/etl.test.ts +++ b/src/node/services/analytics/etl.test.ts @@ -9,6 +9,7 @@ import { appendEvents, CHAT_FILE_NAME, clearWorkspaceAnalyticsState, + deleteCorruptAnalyticsRows, getCurrentPricingFingerprint, ingestWorkspace, parseWorkspaceFromDisk, @@ -1634,3 +1635,41 @@ describe("pricing fingerprint", () => { expect(await readStoredPricingFingerprint(conn)).toBe(getCurrentPricingFingerprint()); }); }); + +describe("deleteCorruptAnalyticsRows", () => { + test("deletes rows with impossible string lengths while keeping healthy rows", async () => { + const conn = await createTestConn(); + + await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ + "ws-healthy", + "anthropic:claude-haiku-4-5", + 1.0, + ]); + // Phantom corruption row: varchar columns hold cross-row concatenations. + await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ + "x".repeat(500), + "anthropic:claude-haiku-4-5".repeat(100), + 0.05, + ]); + await conn.run( + `INSERT INTO delegation_rollups (parent_workspace_id, child_workspace_id, model) + VALUES (?, ?, ?)`, + ["parent-healthy", "child-healthy", "openai:gpt-5.6-sol"] + ); + await conn.run( + `INSERT INTO delegation_rollups (parent_workspace_id, child_workspace_id, model) + VALUES (?, ?, ?)`, + ["p".repeat(500), "child-corrupt", "openai:gpt-5.6-sol"] + ); + + expect(await deleteCorruptAnalyticsRows(conn)).toBe(2); + + const eventRows = await queryRows(conn, "SELECT workspace_id FROM events"); + expect(eventRows).toEqual([{ workspace_id: "ws-healthy" }]); + const rollupRows = await queryRows(conn, "SELECT parent_workspace_id FROM delegation_rollups"); + expect(rollupRows).toEqual([{ parent_workspace_id: "parent-healthy" }]); + + // Idempotent: nothing left to delete. + expect(await deleteCorruptAnalyticsRows(conn)).toBe(0); + }); +}); diff --git a/src/node/services/analytics/etl.ts b/src/node/services/analytics/etl.ts index ad366ecda97..b6f7a788e41 100644 --- a/src/node/services/analytics/etl.ts +++ b/src/node/services/analytics/etl.ts @@ -865,6 +865,37 @@ export async function clearWorkspaceAnalyticsState( } } +/** + * Self-healing sweep for a rare native-layer corruption class: a phantom row + * can materialize whose every VARCHAR column is the concatenation of that + * column's non-null values across an entire batch of inserted rows (observed + * once in the wild: a 17KB "model" string spanning ~670 events, which then + * wallpapered the Analytics dashboard as one giant legend entry). The donor + * rows are written correctly, so deleting rows with impossible string lengths + * loses no real data. Thresholds are generous: legitimate values are all far + * shorter (workspace IDs are short hex; model IDs are < 100 chars). + */ +export async function deleteCorruptAnalyticsRows(conn: DuckDBConnection): Promise { + const eventsResult = await conn.run(` + DELETE FROM events + WHERE LENGTH(workspace_id) > 64 + OR LENGTH(parent_workspace_id) > 64 + OR LENGTH(model) > 256 + OR LENGTH(agent_id) > 64 + OR LENGTH(thinking_level) > 64 + `); + + const rollupsResult = await conn.run(` + DELETE FROM delegation_rollups + WHERE LENGTH(parent_workspace_id) > 64 + OR LENGTH(child_workspace_id) > 64 + OR LENGTH(model) > 256 + OR LENGTH(agent_type) > 64 + `); + + return eventsResult.rowsChanged + rollupsResult.rowsChanged; +} + function serializeHeadSignatureValue(value: string | number | null): string { if (value === null) { return "null"; From 726ac12a6d00c51789086928967c82243dac8b52 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:16:55 +0000 Subject: [PATCH 2/7] chore: upgrade @duckdb/node-api 1.4.4-r.1 -> 1.5.5-r.4 Validated against a copy of a production analytics.db: aggregate stats are identical across versions (only sub-ulp FP summation noise that also varies run-to-run within one version), 1.4.4 can reopen and write a DB that 1.5.5 wrote and checkpointed (downgrade-safe), and uncheckpointed WALs replay cleanly in both upgrade and downgrade directions. --- bun.lock | 22 ++++++++++++++-------- package.json | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/bun.lock b/bun.lock index 1e6e403f9cc..a1751a4c2d4 100644 --- a/bun.lock +++ b/bun.lock @@ -17,7 +17,7 @@ "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", - "@duckdb/node-api": "^1.4.4-r.1", + "@duckdb/node-api": "^1.5.5-r.4", "@homebridge/ciao": "^1.3.4", "@jitl/quickjs-wasmfile-release-asyncify": "^0.31.0", "@lydell/node-pty": "1.1.0", @@ -561,19 +561,25 @@ "@dnd-kit/utilities": ["@dnd-kit/utilities@3.2.2", "", { "dependencies": { "tslib": "^2.0.0" }, "peerDependencies": { "react": ">=16.8.0" } }, "sha512-+MKAJEOfaBe5SmV6t34p80MMKhjvUz0vRrvVJbPT0WElzaOJ/1xs+D+KDv+tD/NE5ujfrChEcshd4fLn0wpiqg=="], - "@duckdb/node-api": ["@duckdb/node-api@1.4.4-r.1", "", { "dependencies": { "@duckdb/node-bindings": "1.4.4-r.1" } }, "sha512-oqaH9DXTJNwyLkd2FgJwmSnWVqjB5irbESeTeNVMBnM03iRaNY545BhfBDumu1TnOV2koIdG1mNsmjgq/ZTIkA=="], + "@duckdb/node-api": ["@duckdb/node-api@1.5.5-r.4", "", { "dependencies": { "@duckdb/node-bindings": "1.5.5-r.4" } }, "sha512-8v0CZNo7aM6GQCNUHERGTrZWIfss8xKzZPF3ACNhPdMMrt7UjkNI5nRQ9FTFO7Yx8ghxYxVpdotmBBMQ5vXUOA=="], - "@duckdb/node-bindings": ["@duckdb/node-bindings@1.4.4-r.1", "", { "optionalDependencies": { "@duckdb/node-bindings-darwin-arm64": "1.4.4-r.1", "@duckdb/node-bindings-darwin-x64": "1.4.4-r.1", "@duckdb/node-bindings-linux-arm64": "1.4.4-r.1", "@duckdb/node-bindings-linux-x64": "1.4.4-r.1", "@duckdb/node-bindings-win32-x64": "1.4.4-r.1" } }, "sha512-NFm0AMrK3kiVLQhgnGUEjX5c8Elm93dYePZ9BUCvvd0AVVTKEBeRhBp9afziuzP3Sl5+7XQ1TyaBLsZJKKBDBQ=="], + "@duckdb/node-bindings": ["@duckdb/node-bindings@1.5.5-r.4", "", { "dependencies": { "detect-libc": "^2.1.2" }, "optionalDependencies": { "@duckdb/node-bindings-darwin-arm64": "1.5.5-r.4", "@duckdb/node-bindings-darwin-x64": "1.5.5-r.4", "@duckdb/node-bindings-linux-arm64": "1.5.5-r.4", "@duckdb/node-bindings-linux-arm64-musl": "1.5.5-r.4", "@duckdb/node-bindings-linux-x64": "1.5.5-r.4", "@duckdb/node-bindings-linux-x64-musl": "1.5.5-r.4", "@duckdb/node-bindings-win32-arm64": "1.5.5-r.4", "@duckdb/node-bindings-win32-x64": "1.5.5-r.4" } }, "sha512-n+4hEfjp4vny3BuWn5p1Gh5CzHaPRxoI8TzTytxL1GMlIKKrXcg/o6sSAjrsROUXGAM4WQCiWPLKnPsjJJSggg=="], - "@duckdb/node-bindings-darwin-arm64": ["@duckdb/node-bindings-darwin-arm64@1.4.4-r.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-/NtbkCgCAOJDxw41XvSGV/mxQAlsx+2xUvhIVUj6fxoOfTG4jTttRhuphwE3EXNoWzJOjZxCZ5LwhC/qb6ZwLg=="], + "@duckdb/node-bindings-darwin-arm64": ["@duckdb/node-bindings-darwin-arm64@1.5.5-r.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-4OdO3pkoJzAZDnZ2iyehi67XL4+WqHPCGYWbyAsYyhGJS+WscGrAnFhMhHudMM2XF8pIEM2tuOKMmwWnTNN2wQ=="], - "@duckdb/node-bindings-darwin-x64": ["@duckdb/node-bindings-darwin-x64@1.4.4-r.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-lzFRDrZwc1EoV513vmKufasiAQ2WlhEb0O6guRBarbvOKKVhRb8tQ5H7LPVTrIewjTI3XDgHrnK+vfh9L+xQcA=="], + "@duckdb/node-bindings-darwin-x64": ["@duckdb/node-bindings-darwin-x64@1.5.5-r.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-WHg3E+TupdujG31LGujMMcmtPIdW6rqRHeihlKgJR8EMetHycoYkwYxRud0niitJvS+uV0du/6pdemzs3HG9GQ=="], - "@duckdb/node-bindings-linux-arm64": ["@duckdb/node-bindings-linux-arm64@1.4.4-r.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-wq92/EcTiOTRW1RSDOwjeLyMMXWwNVNwU21TQdfu3sgS86+Ih3raaK68leDgY5cWgf72We3J2W7HYz8GwxcMYw=="], + "@duckdb/node-bindings-linux-arm64": ["@duckdb/node-bindings-linux-arm64@1.5.5-r.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-VIeHMpYAKpGiWZ4QYsOhODAHDEcXcn089aKty0MFsMEKaEfRJWixKwnwXy/ba2/wwFmnPiSFhKNqygj2BrQbqw=="], - "@duckdb/node-bindings-linux-x64": ["@duckdb/node-bindings-linux-x64@1.4.4-r.1", "", { "os": "linux", "cpu": "x64" }, "sha512-fjYNc+t4/T7mhzZ57oJoIQaWvbYVvxhidcNNansQFiWnd6/JMLCULd4qnt8XI3Tt2BrZsraH690KSBIS3QPt0w=="], + "@duckdb/node-bindings-linux-arm64-musl": ["@duckdb/node-bindings-linux-arm64-musl@1.5.5-r.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-Tswnf+/XWpOcFJNUUu1fkFIX/su4tMcnNz0ZV0Nru3/CkJKIMwBh+VL4SM9YV4zPK90GC4Lm8gzafjc1qDmfyQ=="], - "@duckdb/node-bindings-win32-x64": ["@duckdb/node-bindings-win32-x64@1.4.4-r.1", "", { "os": "win32", "cpu": "x64" }, "sha512-+J+MUYGvYWfX0balWToDIy3CBYg7hHI0KQUQ39+SniinXlMF8+puRW6ebyQ+AXrcrKkwuj4wzJuEBD0AdhHGtw=="], + "@duckdb/node-bindings-linux-x64": ["@duckdb/node-bindings-linux-x64@1.5.5-r.4", "", { "os": "linux", "cpu": "x64" }, "sha512-EY+CL/4h8MQZd9MxTBq+98m3U9osmvHBwhE9b5fMWQGt2I6p1j8fvX0SXiwy9KBfQIbjVUOf5XvGdXncI54KSg=="], + + "@duckdb/node-bindings-linux-x64-musl": ["@duckdb/node-bindings-linux-x64-musl@1.5.5-r.4", "", { "os": "linux", "cpu": "x64" }, "sha512-h0ixrgGHtHh+C/Fu1eAL9hX4iCf8yuyJN6Y24Gh8axXXP8UuSh0rQRAQUQTYarQ8pm2qSG/67ZYjyYYydD+J/w=="], + + "@duckdb/node-bindings-win32-arm64": ["@duckdb/node-bindings-win32-arm64@1.5.5-r.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-uorAnySIMwWkRDuUhM1yTDxWIislnX8mndhFIw6r4VKhVW61HEOOMX5oAgkSQFII4RNWpC5PCcUAEhSATgNvVQ=="], + + "@duckdb/node-bindings-win32-x64": ["@duckdb/node-bindings-win32-x64@1.5.5-r.4", "", { "os": "win32", "cpu": "x64" }, "sha512-X9XGcWQ10P3mvUIaMXXk2bi94Cow7b/ziTPMKxJ0U8U3wQPxsLzEUP7D7C/KrBWINl5am2k+5SkDVyA/THUgPg=="], "@electron/asar": ["@electron/asar@3.4.1", "", { "dependencies": { "commander": "^5.0.0", "glob": "^7.1.6", "minimatch": "^3.0.4" }, "bin": { "asar": "bin/asar.js" } }, "sha512-i4/rNPRS84t0vSRa2HorerGRXWyF4vThfHesw0dmcWHp+cspK743UanA0suA5Q5y8kzY2y6YKrvbIUn69BCAiA=="], diff --git a/package.json b/package.json index 8cae898fa81..41a8f2f391e 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", - "@duckdb/node-api": "^1.4.4-r.1", + "@duckdb/node-api": "^1.5.5-r.4", "@homebridge/ciao": "^1.3.4", "@jitl/quickjs-wasmfile-release-asyncify": "^0.31.0", "@lydell/node-pty": "1.1.0", From ad7c767b8e5476bd3ee27c0767421035a1fa6530 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:33:22 +0000 Subject: [PATCH 3/7] fix: address Codex review round 1 - Raise sweep thresholds above legal maxima: migrated legacy workspace IDs (${projectBasename}-${workspaceBasename}) can reach 511 chars, so cap workspace/agent identifiers at 1024 and model strings at 512; document the derivation. Add a retention test for a max-length legacy ID. - Make sweepCorruptRows best-effort: catch and log failures so a sweep error cannot reject worker init (caching a permanent workerError) or fail an otherwise-successful ingest. --- .../services/analytics/analyticsWorker.ts | 19 ++++++++--- src/node/services/analytics/etl.test.ts | 19 ++++++++--- src/node/services/analytics/etl.ts | 32 ++++++++++++------- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/node/services/analytics/analyticsWorker.ts b/src/node/services/analytics/analyticsWorker.ts index 89d621439ff..d76b0117836 100644 --- a/src/node/services/analytics/analyticsWorker.ts +++ b/src/node/services/analytics/analyticsWorker.ts @@ -152,12 +152,23 @@ async function handleInit(data: InitData): Promise { await sweepCorruptRows("init"); } -/** Delete corruption-class rows and log when anything was actually removed. */ +/** + * Delete corruption-class rows and log when anything was actually removed. + * Best-effort: a failed sweep must never reject init (which would cache a + * worker error and disable analytics until restart) or fail an + * otherwise-successful ingest, so errors are logged and swallowed. + */ async function sweepCorruptRows(context: string): Promise { - const deleted = await deleteCorruptAnalyticsRows(getConn()); - if (deleted > 0) { + try { + const deleted = await deleteCorruptAnalyticsRows(getConn()); + if (deleted > 0) { + process.stderr.write( + `[analytics-worker] Deleted ${deleted} corrupt analytics row(s) (${context})\n` + ); + } + } catch (error) { process.stderr.write( - `[analytics-worker] Deleted ${deleted} corrupt analytics row(s) (${context})\n` + `[analytics-worker] Corrupt-row sweep failed (${context}): ${getErrorMessage(error)}\n` ); } } diff --git a/src/node/services/analytics/etl.test.ts b/src/node/services/analytics/etl.test.ts index 24360c35d84..9078c9584ea 100644 --- a/src/node/services/analytics/etl.test.ts +++ b/src/node/services/analytics/etl.test.ts @@ -1645,9 +1645,17 @@ describe("deleteCorruptAnalyticsRows", () => { "anthropic:claude-haiku-4-5", 1.0, ]); + // Migrated legacy IDs are `${projectBasename}-${workspaceBasename}` with + // no length limit (up to 2x NAME_MAX + 1 = 511 chars) and must survive. + const legacyId = `${"p".repeat(255)}-${"w".repeat(255)}`; + await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ + legacyId, + "anthropic:claude-haiku-4-5", + 2.0, + ]); // Phantom corruption row: varchar columns hold cross-row concatenations. await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ - "x".repeat(500), + "x".repeat(2000), "anthropic:claude-haiku-4-5".repeat(100), 0.05, ]); @@ -1659,13 +1667,16 @@ describe("deleteCorruptAnalyticsRows", () => { await conn.run( `INSERT INTO delegation_rollups (parent_workspace_id, child_workspace_id, model) VALUES (?, ?, ?)`, - ["p".repeat(500), "child-corrupt", "openai:gpt-5.6-sol"] + ["p".repeat(2000), "child-corrupt", "openai:gpt-5.6-sol"] ); expect(await deleteCorruptAnalyticsRows(conn)).toBe(2); - const eventRows = await queryRows(conn, "SELECT workspace_id FROM events"); - expect(eventRows).toEqual([{ workspace_id: "ws-healthy" }]); + const eventRows = await queryRows( + conn, + "SELECT workspace_id FROM events ORDER BY LENGTH(workspace_id)" + ); + expect(eventRows).toEqual([{ workspace_id: "ws-healthy" }, { workspace_id: legacyId }]); const rollupRows = await queryRows(conn, "SELECT parent_workspace_id FROM delegation_rollups"); expect(rollupRows).toEqual([{ parent_workspace_id: "parent-healthy" }]); diff --git a/src/node/services/analytics/etl.ts b/src/node/services/analytics/etl.ts index b6f7a788e41..715129b863d 100644 --- a/src/node/services/analytics/etl.ts +++ b/src/node/services/analytics/etl.ts @@ -872,25 +872,35 @@ export async function clearWorkspaceAnalyticsState( * once in the wild: a 17KB "model" string spanning ~670 events, which then * wallpapered the Analytics dashboard as one giant legend entry). The donor * rows are written correctly, so deleting rows with impossible string lengths - * loses no real data. Thresholds are generous: legitimate values are all far - * shorter (workspace IDs are short hex; model IDs are < 100 chars). + * loses no real data. + * + * Thresholds sit above the longest legitimately constructible value for each + * column so no legal row can match: + * - Workspace IDs: new IDs are short hex, but migrated legacy IDs are + * `${projectBasename}-${workspaceBasename}` (config.generateLegacyId) with + * no explicit limit; each basename is bounded by the filesystem's NAME_MAX + * (255 bytes), so 511 is the legal ceiling. Cap at 1024. + * - Agent IDs/types also derive from file basenames: same 1024 cap. + * - Model strings are `provider:modelId` config values (longest observed in a + * large production DB: 41 chars); 512 is far beyond any real model ID. + * The observed phantom row exceeded every one of these by an order of + * magnitude (6,720-char workspace_id, 17,229-char model). */ export async function deleteCorruptAnalyticsRows(conn: DuckDBConnection): Promise { const eventsResult = await conn.run(` DELETE FROM events - WHERE LENGTH(workspace_id) > 64 - OR LENGTH(parent_workspace_id) > 64 - OR LENGTH(model) > 256 - OR LENGTH(agent_id) > 64 - OR LENGTH(thinking_level) > 64 + WHERE LENGTH(workspace_id) > 1024 + OR LENGTH(parent_workspace_id) > 1024 + OR LENGTH(model) > 512 + OR LENGTH(agent_id) > 1024 `); const rollupsResult = await conn.run(` DELETE FROM delegation_rollups - WHERE LENGTH(parent_workspace_id) > 64 - OR LENGTH(child_workspace_id) > 64 - OR LENGTH(model) > 256 - OR LENGTH(agent_type) > 64 + WHERE LENGTH(parent_workspace_id) > 1024 + OR LENGTH(child_workspace_id) > 1024 + OR LENGTH(model) > 512 + OR LENGTH(agent_type) > 1024 `); return eventsResult.rowsChanged + rollupsResult.rowsChanged; From 1121a53ef27d66cfd5334ceae7775140d8f4ec88 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:40:57 +0000 Subject: [PATCH 4/7] fix: anchor corruption detection on bounded identifier columns only Codex round 2: custom-provider model IDs have no schema max length (ProviderModelEntrySchema), so a model-length cap could delete real spend. Since the corruption concatenates every varchar column at once and workspace identifiers appear on every row, identifier evidence alone detects the phantom row; drop the model clause from both tables and pin retention of a 2KB custom model ID. --- src/node/services/analytics/etl.test.ts | 14 +++++++++++++- src/node/services/analytics/etl.ts | 15 +++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/node/services/analytics/etl.test.ts b/src/node/services/analytics/etl.test.ts index 9078c9584ea..9cfa2cffcca 100644 --- a/src/node/services/analytics/etl.test.ts +++ b/src/node/services/analytics/etl.test.ts @@ -1653,6 +1653,14 @@ describe("deleteCorruptAnalyticsRows", () => { "anthropic:claude-haiku-4-5", 2.0, ]); + // Custom-provider model IDs have no schema max length; an extremely long + // model on an otherwise-healthy row must never be deletion evidence. + const longModel = `custom:${"m".repeat(2000)}`; + await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ + "ws-long-model", + longModel, + 3.0, + ]); // Phantom corruption row: varchar columns hold cross-row concatenations. await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ "x".repeat(2000), @@ -1676,7 +1684,11 @@ describe("deleteCorruptAnalyticsRows", () => { conn, "SELECT workspace_id FROM events ORDER BY LENGTH(workspace_id)" ); - expect(eventRows).toEqual([{ workspace_id: "ws-healthy" }, { workspace_id: legacyId }]); + expect(eventRows).toEqual([ + { workspace_id: "ws-healthy" }, + { workspace_id: "ws-long-model" }, + { workspace_id: legacyId }, + ]); const rollupRows = await queryRows(conn, "SELECT parent_workspace_id FROM delegation_rollups"); expect(rollupRows).toEqual([{ parent_workspace_id: "parent-healthy" }]); diff --git a/src/node/services/analytics/etl.ts b/src/node/services/analytics/etl.ts index 715129b863d..35382e6ef89 100644 --- a/src/node/services/analytics/etl.ts +++ b/src/node/services/analytics/etl.ts @@ -874,24 +874,24 @@ export async function clearWorkspaceAnalyticsState( * rows are written correctly, so deleting rows with impossible string lengths * loses no real data. * - * Thresholds sit above the longest legitimately constructible value for each - * column so no legal row can match: + * Detection anchors ONLY on identifier columns with a provable legal maximum, + * because the corruption concatenates every VARCHAR column at once and + * workspace identifiers are present on every row: * - Workspace IDs: new IDs are short hex, but migrated legacy IDs are * `${projectBasename}-${workspaceBasename}` (config.generateLegacyId) with * no explicit limit; each basename is bounded by the filesystem's NAME_MAX * (255 bytes), so 511 is the legal ceiling. Cap at 1024. * - Agent IDs/types also derive from file basenames: same 1024 cap. - * - Model strings are `provider:modelId` config values (longest observed in a - * large production DB: 41 chars); 512 is far beyond any real model ID. - * The observed phantom row exceeded every one of these by an order of - * magnitude (6,720-char workspace_id, 17,229-char model). + * Unbounded columns (model: custom-provider model IDs have no schema max, see + * ProviderModelEntrySchema; paths; workspace names) must NOT be deletion + * evidence on their own, or a legitimately long value would wipe real spend. + * The observed phantom row's workspace_id was 6,720 chars. */ export async function deleteCorruptAnalyticsRows(conn: DuckDBConnection): Promise { const eventsResult = await conn.run(` DELETE FROM events WHERE LENGTH(workspace_id) > 1024 OR LENGTH(parent_workspace_id) > 1024 - OR LENGTH(model) > 512 OR LENGTH(agent_id) > 1024 `); @@ -899,7 +899,6 @@ export async function deleteCorruptAnalyticsRows(conn: DuckDBConnection): Promis DELETE FROM delegation_rollups WHERE LENGTH(parent_workspace_id) > 1024 OR LENGTH(child_workspace_id) > 1024 - OR LENGTH(model) > 512 OR LENGTH(agent_type) > 1024 `); From 59c03c6a29366972f373e3616d66d51e0cdf620d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:50:06 +0000 Subject: [PATCH 5/7] fix: add watermark-membership evidence to catch small-batch corruption Codex round 3: a corrupted batch small enough to keep concatenated identifiers under the length caps would evade the sweep while its 3KB model string still breaks the dashboard. Add structural evidence that is batch-size independent: a concatenation of two or more workspace IDs can never equal a real workspace ID, and every legitimate row's workspace has an ingest_watermarks entry by the time sweeps run (crash-window orphans are safe to delete because the missing watermark forces a full re-ingest of that workspace on the next syncCheck). Rollups join on parent only; children may be legitimately removed. Verified on the production DB copy: the join evidence alone identifies exactly the one phantom row among 58,479 events and 14,671 watermarks. --- src/node/services/analytics/etl.test.ts | 89 +++++++++++++++---------- src/node/services/analytics/etl.ts | 38 +++++++---- 2 files changed, 80 insertions(+), 47 deletions(-) diff --git a/src/node/services/analytics/etl.test.ts b/src/node/services/analytics/etl.test.ts index 9cfa2cffcca..e7955d1f6ed 100644 --- a/src/node/services/analytics/etl.test.ts +++ b/src/node/services/analytics/etl.test.ts @@ -1637,48 +1637,61 @@ describe("pricing fingerprint", () => { }); describe("deleteCorruptAnalyticsRows", () => { - test("deletes rows with impossible string lengths while keeping healthy rows", async () => { + async function seedWatermark(conn: DuckDBConnection, workspaceId: string): Promise { + await conn.run( + "INSERT INTO ingest_watermarks (workspace_id, last_sequence, last_modified) VALUES (?, ?, ?)", + [workspaceId, 1, 1] + ); + } + + test("deletes corrupt rows while keeping healthy rows", async () => { const conn = await createTestConn(); - await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ - "ws-healthy", - "anthropic:claude-haiku-4-5", - 1.0, - ]); // Migrated legacy IDs are `${projectBasename}-${workspaceBasename}` with // no length limit (up to 2x NAME_MAX + 1 = 511 chars) and must survive. const legacyId = `${"p".repeat(255)}-${"w".repeat(255)}`; - await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ - legacyId, - "anthropic:claude-haiku-4-5", - 2.0, - ]); // Custom-provider model IDs have no schema max length; an extremely long // model on an otherwise-healthy row must never be deletion evidence. const longModel = `custom:${"m".repeat(2000)}`; - await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ - "ws-long-model", - longModel, - 3.0, - ]); - // Phantom corruption row: varchar columns hold cross-row concatenations. - await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ - "x".repeat(2000), - "anthropic:claude-haiku-4-5".repeat(100), - 0.05, - ]); - await conn.run( - `INSERT INTO delegation_rollups (parent_workspace_id, child_workspace_id, model) - VALUES (?, ?, ?)`, - ["parent-healthy", "child-healthy", "openai:gpt-5.6-sol"] - ); - await conn.run( - `INSERT INTO delegation_rollups (parent_workspace_id, child_workspace_id, model) - VALUES (?, ?, ?)`, - ["p".repeat(2000), "child-corrupt", "openai:gpt-5.6-sol"] - ); - expect(await deleteCorruptAnalyticsRows(conn)).toBe(2); + for (const workspaceId of ["ws-healthy", legacyId, "ws-long-model", "parent-healthy"]) { + await seedWatermark(conn, workspaceId); + } + + for (const [workspaceId, model, cost] of [ + ["ws-healthy", "anthropic:claude-haiku-4-5", 1.0], + [legacyId, "anthropic:claude-haiku-4-5", 2.0], + ["ws-long-model", longModel, 3.0], + // Large-batch phantom: concatenated identifiers exceed the length caps. + ["x".repeat(2000), "anthropic:claude-haiku-4-5".repeat(100), 0.05], + // Small-batch phantom: two concatenated 10-char workspace IDs stay far + // under the length caps but can never match a real watermark. + ["aaaaabbbbbcccccddddd", "openai:gpt-5.6-solopenai:gpt-5.6-sol", 0.05], + ] as const) { + await conn.run("INSERT INTO events (workspace_id, model, total_cost_usd) VALUES (?, ?, ?)", [ + workspaceId, + model, + cost, + ]); + } + + for (const [parent, child] of [ + ["parent-healthy", "child-healthy"], + // A rollup may outlive its removed child workspace; only the parent + // must be a known workspace. + ["parent-healthy", "child-removed"], + ["p".repeat(2000), "child-corrupt"], + // Small-batch phantom parent: unknown to watermarks. + ["par-aaaaapar-bbbbb", "child-x"], + ] as const) { + await conn.run( + `INSERT INTO delegation_rollups (parent_workspace_id, child_workspace_id, model) + VALUES (?, ?, ?)`, + [parent, child, "openai:gpt-5.6-sol"] + ); + } + + expect(await deleteCorruptAnalyticsRows(conn)).toBe(4); const eventRows = await queryRows( conn, @@ -1689,8 +1702,14 @@ describe("deleteCorruptAnalyticsRows", () => { { workspace_id: "ws-long-model" }, { workspace_id: legacyId }, ]); - const rollupRows = await queryRows(conn, "SELECT parent_workspace_id FROM delegation_rollups"); - expect(rollupRows).toEqual([{ parent_workspace_id: "parent-healthy" }]); + const rollupRows = await queryRows( + conn, + "SELECT child_workspace_id FROM delegation_rollups ORDER BY child_workspace_id" + ); + expect(rollupRows).toEqual([ + { child_workspace_id: "child-healthy" }, + { child_workspace_id: "child-removed" }, + ]); // Idempotent: nothing left to delete. expect(await deleteCorruptAnalyticsRows(conn)).toBe(0); diff --git a/src/node/services/analytics/etl.ts b/src/node/services/analytics/etl.ts index 35382e6ef89..8a69408af73 100644 --- a/src/node/services/analytics/etl.ts +++ b/src/node/services/analytics/etl.ts @@ -874,18 +874,25 @@ export async function clearWorkspaceAnalyticsState( * rows are written correctly, so deleting rows with impossible string lengths * loses no real data. * - * Detection anchors ONLY on identifier columns with a provable legal maximum, - * because the corruption concatenates every VARCHAR column at once and - * workspace identifiers are present on every row: - * - Workspace IDs: new IDs are short hex, but migrated legacy IDs are - * `${projectBasename}-${workspaceBasename}` (config.generateLegacyId) with - * no explicit limit; each basename is bounded by the filesystem's NAME_MAX - * (255 bytes), so 511 is the legal ceiling. Cap at 1024. - * - Agent IDs/types also derive from file basenames: same 1024 cap. - * Unbounded columns (model: custom-provider model IDs have no schema max, see - * ProviderModelEntrySchema; paths; workspace names) must NOT be deletion - * evidence on their own, or a legitimately long value would wipe real spend. - * The observed phantom row's workspace_id was 6,720 chars. + * Two evidence classes, both structural (unbounded columns like model, + * paths, and workspace names are never deletion evidence on their own, since + * custom-provider model IDs etc. have no schema max length): + * + * 1. Identifier length beyond the legal construction maximum. New workspace + * IDs are short hex; migrated legacy IDs are + * `${projectBasename}-${workspaceBasename}` (config.generateLegacyId), + * each basename bounded by the filesystem's NAME_MAX (255 bytes), so 511 + * is the ceiling. Agent IDs/types also derive from basenames. Cap at 1024. + * + * 2. Workspace identity unknown to ingest_watermarks. A concatenation of two + * or more workspace IDs can never equal a real workspace ID, no matter how + * small the corrupted batch, while every legitimate row's workspace gets a + * watermark by the end of the ingest/rebuild pass that wrote it (sweeps + * run after those passes complete). If a crash lands between the event + * write and the watermark write, deleting the orphans is still safe: the + * missing watermark makes the next syncCheck re-ingest that workspace from + * disk in full. delegation_rollups joins on parent_workspace_id only; + * child_workspace_id may legitimately reference a removed child workspace. */ export async function deleteCorruptAnalyticsRows(conn: DuckDBConnection): Promise { const eventsResult = await conn.run(` @@ -893,6 +900,9 @@ export async function deleteCorruptAnalyticsRows(conn: DuckDBConnection): Promis WHERE LENGTH(workspace_id) > 1024 OR LENGTH(parent_workspace_id) > 1024 OR LENGTH(agent_id) > 1024 + OR NOT EXISTS ( + SELECT 1 FROM ingest_watermarks w WHERE w.workspace_id = events.workspace_id + ) `); const rollupsResult = await conn.run(` @@ -900,6 +910,10 @@ export async function deleteCorruptAnalyticsRows(conn: DuckDBConnection): Promis WHERE LENGTH(parent_workspace_id) > 1024 OR LENGTH(child_workspace_id) > 1024 OR LENGTH(agent_type) > 1024 + OR NOT EXISTS ( + SELECT 1 FROM ingest_watermarks w + WHERE w.workspace_id = delegation_rollups.parent_workspace_id + ) `); return eventsResult.rowsChanged + rollupsResult.rowsChanged; From 6ca49488f95bacaa25f84028ff93368c1d1ceecc Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:09:20 +0000 Subject: [PATCH 6/7] fmt: run prettier on README (pre-existing breakage on main tip) --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ac1d6ba8ea0..af87dced4d2 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,12 @@ [![Discord](https://img.shields.io/discord/1446553342699507907?logo=discord&label=Discord)](https://cdr.co/mux-discord) [![X (formerly Twitter)](https://img.shields.io/badge/Follow-%40codermux-black?logo=x)](https://x.com/codermux) - > [!IMPORTANT] > This project was renamed from Mux to Shux after Mux.com raised a trademark concern. “Mux” is a common technical abbreviation of “multiplexer” and we do not expect confusion between the projects, but chose to rename ours rather than spend more time on the dispute. “Shux” captures our reaction to the process and bears no other significance. -> -Shux is a desktop & browser application for parallel agentic development. It enables developers to plan and execute tasks with multiple AI agents on local or remote compute. +> +> Shux is a desktop & browser application for parallel agentic development. It enables developers to plan and execute tasks with multiple AI agents on local or remote compute.

Shux product demo

From bdefee4e4fe2f8b7a385cb0be959f545f068757d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:45:23 +0000 Subject: [PATCH 7/7] chore: refresh flake offline-cache hash for the duckdb bump --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 242be6b69bb..73f445d47cd 100644 --- a/flake.nix +++ b/flake.nix @@ -84,7 +84,7 @@ outputHashMode = "recursive"; # Marker used by scripts/update_flake_hash.sh to update this hash in place. - outputHash = "sha256-Ci2q4ZCIymKhf4rinh6VKdzaGVBCRBsMUgjQOXTqotM="; # shux-offline-cache-hash + outputHash = "sha256-ri3Q1gY4ifnjT9FMg3wNqoIT5OJbfQRKAWw5Zt8DH9k="; # shux-offline-cache-hash }; configurePhase = ''