From dbf97f30be9d97cb14f4bb49918573b1d892c018 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 17 Aug 2026 10:49:06 -0400 Subject: [PATCH 1/5] host: Drain indexing after restoring a cached realm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A restored index does not stop the realm indexing. Each test's adapter stamps its fixtures with the current second while the restored rows carry the first test's, and `discover-invalidations` compares those for equality, so `realm.start()` invalidates the lot and re-indexes. The work is cheap — the snapshot also restored the transpile and prerender caches — but it is asynchronous, and a test that reads index-derived state races it. That is how `ai-assistant`'s code-mode test lost the cursor position it clicks a definition to reach: the lookup ran while indexing was still in flight. So settle the realms before returning, the way setupLocalIndexing's afterEach already does between tests. Deliberately not fixed by making the mtimes match. That comparison is what rebuilds the index when a module's fixtures diverge from its snapshot under an unchanged cache key — the only protection this helper has against silently serving the wrong fixtures. Content-derived mtimes would keep the protection in principle, but fixture content includes live card instances that are serialized later and carry a per-construction id, so they cannot be hashed at construction. ai-assistant moves back to the shared cache here, so the run either proves the drain fixes that race or says the diagnosis was wrong. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/acceptance/ai-assistant-test.gts | 265 +++++++++--------- packages/host/tests/helpers/index.gts | 26 +- 2 files changed, 158 insertions(+), 133 deletions(-) diff --git a/packages/host/tests/acceptance/ai-assistant-test.gts b/packages/host/tests/acceptance/ai-assistant-test.gts index f09666d8eaa..6f6316e1ae8 100644 --- a/packages/host/tests/acceptance/ai-assistant-test.gts +++ b/packages/host/tests/acceptance/ai-assistant-test.gts @@ -37,6 +37,8 @@ import type MonacoService from '@cardstack/host/services/monaco-service'; import { AiAssistantMessageDrafts } from '@cardstack/host/utils/local-storage-keys'; import { + withCachedRealmSetup, + setupRealmCacheTeardown, setupLocalIndexing, addSkillToAiAssistant, setupOnSave, @@ -142,13 +144,8 @@ function modelNameFor(llmId: string): string { module('Acceptance | AI Assistant tests', function (hooks) { setupApplicationTest(hooks); - // Every test in this module builds the same realm fixtures in the beforeEach - // below, so the realm is indexed once and that index restored for each - // subsequent test. What a test writes afterwards stays with that test — the - // snapshot is restored, not carried forward. - setupLocalIndexing(hooks, { - reuseIndexAcrossTests: 'aiAssistant', - }); + setupLocalIndexing(hooks); + setupRealmCacheTeardown(hooks); setupOnSave(hooks); let mockMatrixUtils = setupMockMatrix(hooks, { @@ -431,149 +428,153 @@ module('Acceptance | AI Assistant tests', function (hooks) { modelConfigurations: [deepseekModel, geminiFlashModel, openAiGpt5Model], }); - await setupAcceptanceTestRealm({ - mockMatrixUtils, - contents: { - ...SYSTEM_CARD_FIXTURE_CONTENTS, - 'person.gts': { Person }, - 'pet.gts': { Pet }, - 'broken-card.gts': ` - import { CardDef, field, contains } from '@cardstack/base/card-api'; - import StringField from '@cardstack/base/string'; - import { BrokenField } from './does-not-exist'; - export class BrokenCard extends CardDef { - static displayName = 'Broken Card'; - @field name = contains(StringField); - @field broken = contains(BrokenField); - } - `, - 'BrokenCard/errored.json': { - data: { - attributes: { - name: 'Errored Instance', - }, - meta: { - adoptsFrom: { - module: '../broken-card', - name: 'BrokenCard', + // Every test builds these fixtures identically, so the indexed result is + // cached for the module and restored rather than rebuilt. + await withCachedRealmSetup(async () => { + await setupAcceptanceTestRealm({ + mockMatrixUtils, + contents: { + ...SYSTEM_CARD_FIXTURE_CONTENTS, + 'person.gts': { Person }, + 'pet.gts': { Pet }, + 'broken-card.gts': ` + import { CardDef, field, contains } from '@cardstack/base/card-api'; + import StringField from '@cardstack/base/string'; + import { BrokenField } from './does-not-exist'; + export class BrokenCard extends CardDef { + static displayName = 'Broken Card'; + @field name = contains(StringField); + @field broken = contains(BrokenField); + } + `, + 'BrokenCard/errored.json': { + data: { + attributes: { + name: 'Errored Instance', + }, + meta: { + adoptsFrom: { + module: '../broken-card', + name: 'BrokenCard', + }, }, }, }, - }, - 'country.gts': countryDefinition, - 'Country/indonesia.json': { - data: { - attributes: { - name: 'Indonesia', - }, - meta: { - adoptsFrom: { - module: `${testRealmURL}country`, - name: 'Country', + 'country.gts': countryDefinition, + 'Country/indonesia.json': { + data: { + attributes: { + name: 'Indonesia', + }, + meta: { + adoptsFrom: { + module: `${testRealmURL}country`, + name: 'Country', + }, }, }, }, - }, - 'Pet/ringo.json': new Pet({ name: 'Ringo' }), - 'Person/hassan.json': new Person({ - firstName: 'Hassan', - lastName: 'Abdel-Rahman', - pet: mangoPet, - friends: [mangoPet], - }), - 'Pet/mango.json': mangoPet, - 'Pet/vangogh.json': new Pet({ name: 'Van Gogh' }), - 'Person/fadhlan.json': new Person({ - firstName: 'Fadhlan', - pet: mangoPet, - friends: [mangoPet], - }), - 'plant.gts': ` - import { CardDef, field, contains, StringField } from '@cardstack/base/card-api'; - export class Plant extends CardDef { - static displayName = "Plant"; - @field commonName = contains(StringField); - } - `, - 'Plant/highbush-blueberry.json': { - data: { - attributes: { - commonName: 'Highbush Blueberry', - }, - meta: { - adoptsFrom: { - module: `../plant`, - name: 'Plant', + 'Pet/ringo.json': new Pet({ name: 'Ringo' }), + 'Person/hassan.json': new Person({ + firstName: 'Hassan', + lastName: 'Abdel-Rahman', + pet: mangoPet, + friends: [mangoPet], + }), + 'Pet/mango.json': mangoPet, + 'Pet/vangogh.json': new Pet({ name: 'Van Gogh' }), + 'Person/fadhlan.json': new Person({ + firstName: 'Fadhlan', + pet: mangoPet, + friends: [mangoPet], + }), + 'plant.gts': ` + import { CardDef, field, contains, StringField } from '@cardstack/base/card-api'; + export class Plant extends CardDef { + static displayName = "Plant"; + @field commonName = contains(StringField); + } + `, + 'Plant/highbush-blueberry.json': { + data: { + attributes: { + commonName: 'Highbush Blueberry', + }, + meta: { + adoptsFrom: { + module: `../plant`, + name: 'Plant', + }, }, }, }, - }, - 'Spec/plant-spec.json': { - data: { - type: 'card', - attributes: { - ref: { - name: 'Plant', - module: `${testRealmURL}plant`, + 'Spec/plant-spec.json': { + data: { + type: 'card', + attributes: { + ref: { + name: 'Plant', + module: `${testRealmURL}plant`, + }, + specType: 'card', + cardTitle: 'Plant spec', }, - specType: 'card', - cardTitle: 'Plant spec', - }, - meta: { - adoptsFrom: { - module: '@cardstack/base/spec', - name: 'Spec', + meta: { + adoptsFrom: { + module: '@cardstack/base/spec', + name: 'Spec', + }, }, }, }, - }, - 'Skill/example.json': { - data: { - attributes: { - cardTitle: 'Example Skill', - cardDescription: 'This skill card is for testing purposes', - instructions: 'This is an example skill card', - commands: [], - }, - meta: { - adoptsFrom: skillCardRef, + 'Skill/example.json': { + data: { + attributes: { + cardTitle: 'Example Skill', + cardDescription: 'This skill card is for testing purposes', + instructions: 'This is an example skill card', + commands: [], + }, + meta: { + adoptsFrom: skillCardRef, + }, }, }, - }, - 'Skill/example2.json': { - data: { - attributes: { - cardTitle: 'Example 2 Skill', - cardDescription: 'This skill card is also for testing purposes', - instructions: 'This is a second example skill card', - commands: [], - }, - meta: { - adoptsFrom: skillCardRef, + 'Skill/example2.json': { + data: { + attributes: { + cardTitle: 'Example 2 Skill', + cardDescription: 'This skill card is also for testing purposes', + instructions: 'This is a second example skill card', + commands: [], + }, + meta: { + adoptsFrom: skillCardRef, + }, }, }, + 'ModelConfiguration/gpt-4o-mini.json': openAiGpt4oMiniModel, + 'ModelConfiguration/gpt-5.json': openAiGpt5Model, + 'ModelConfiguration/gpt-5-extended.json': openAiGpt5ExtendedModel, + 'ModelConfiguration/claude-sonnet-4.6.json': + anthropicClaudeSonnet46Model, + 'ModelConfiguration/claude-sonnet-4.5.json': + anthropicClaudeSonnet45Model, + 'ModelConfiguration/claude-sonnet-3.7.json': + anthropicClaudeSonnet37Model, + 'SystemCard/default.json': defaultSystemCard, + 'ModelConfiguration/deepseek-chat-v3-0324.json': deepseekModel, + 'ModelConfiguration/gemini-2.5-flash.json': geminiFlashModel, + 'SystemCard/productivity.json': alternateSystemCard, + 'index.json': new CardsGrid(), + 'realm.json': realmConfigCardJSON({ + name: 'Test Workspace B', + backgroundURL: + 'https://i.postimg.cc/VNvHH93M/pawel-czerwinski-Ly-ZLa-A5jti-Y-unsplash.jpg', + iconURL: 'https://i.postimg.cc/L8yXRvws/icon.png', + }), }, - 'ModelConfiguration/gpt-4o-mini.json': openAiGpt4oMiniModel, - 'ModelConfiguration/gpt-5.json': openAiGpt5Model, - 'ModelConfiguration/gpt-5-extended.json': openAiGpt5ExtendedModel, - 'ModelConfiguration/claude-sonnet-4.6.json': - anthropicClaudeSonnet46Model, - 'ModelConfiguration/claude-sonnet-4.5.json': - anthropicClaudeSonnet45Model, - 'ModelConfiguration/claude-sonnet-3.7.json': - anthropicClaudeSonnet37Model, - 'SystemCard/default.json': defaultSystemCard, - 'ModelConfiguration/deepseek-chat-v3-0324.json': deepseekModel, - 'ModelConfiguration/gemini-2.5-flash.json': geminiFlashModel, - 'SystemCard/productivity.json': alternateSystemCard, - 'index.json': new CardsGrid(), - 'realm.json': realmConfigCardJSON({ - name: 'Test Workspace B', - backgroundURL: - 'https://i.postimg.cc/VNvHH93M/pawel-czerwinski-Ly-ZLa-A5jti-Y-unsplash.jpg', - iconURL: 'https://i.postimg.cc/L8yXRvws/icon.png', - }), - }, + }); }); getService('matrix-service').fetchMatrixHostedFile = async (_url) => { diff --git a/packages/host/tests/helpers/index.gts b/packages/host/tests/helpers/index.gts index ed8d8135a6a..5a757f1800b 100644 --- a/packages/host/tests/helpers/index.gts +++ b/packages/host/tests/helpers/index.gts @@ -247,13 +247,37 @@ export async function withCachedRealmSetup( let dbAdapter = await getDbAdapter(); if (dbAdapter.hasSnapshot(snapshotName)) { await dbAdapter.importSnapshot(snapshotName); - return setup(); + let result = await setup(); + // A restored index does not stop the realm indexing. Every fixture carries + // the mtime the *current* adapter stamped it with, while the restored rows + // carry the first test's, and `discover-invalidations` compares those for + // equality — so `realm.start()` invalidates the lot and re-indexes. The work + // is cheap, because the snapshot also restored the transpile and prerender + // caches, but it is asynchronous: without draining it here, a test that + // reads index-derived state races indexing still in flight. That is not + // hypothetical — it is how `ai-assistant`'s code-mode test lost the cursor + // position it clicks a definition to reach. + // + // Deliberately not solved by making the mtimes match. The mtime comparison + // is what rebuilds the index when a module's fixtures diverge from the + // snapshot under an unchanged cache key, which is this helper's only + // protection against silently serving the wrong fixtures. + await drainRealmIndexing(); + return result; } let result = await setup(); await dbAdapter.exportSnapshot(snapshotName); return result; } +// Settle any indexing the realms have in flight. Mirrors what +// `setupLocalIndexing`'s afterEach does between tests. +async function drainRealmIndexing(): Promise { + for (let { realm } of getTestRealmRegistry().values()) { + await realm.incrementalIndexing(); + } +} + function getCurrentModuleCacheKey(): string { let config = QUnit.config as QUnit['config'] & { currentModule?: { name?: string }; From fc5a09a2cc9871a5bc1c8b999edab9916a688cba Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 17 Aug 2026 17:53:00 -0400 Subject: [PATCH 2/5] host: Report the selectionRange a failed context assertion saw `code mode context sent with message` failed this assertion on CI with no way to see what it got: the TAP reporter renders a failed deepEqual's operands as `[object Object]`, and a shard that fails uploads no test-report artifact to read them from. Put the value in the message. Co-Authored-By: Claude Opus 5 --- packages/host/tests/acceptance/ai-assistant-test.gts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/host/tests/acceptance/ai-assistant-test.gts b/packages/host/tests/acceptance/ai-assistant-test.gts index 6f6316e1ae8..6a803f23db7 100644 --- a/packages/host/tests/acceptance/ai-assistant-test.gts +++ b/packages/host/tests/acceptance/ai-assistant-test.gts @@ -1981,7 +1981,12 @@ module('Acceptance | AI Assistant tests', function (hooks) { endLine: 1, endColumn: 1, }, - 'Context sent with message contains correct selectionRange', + // The value is in the message because the TAP reporter renders a failed + // deepEqual's operands as `[object Object]`, and a shard that fails + // uploads no test-report artifact to read them from. + `Context sent with message contains correct selectionRange (got ${JSON.stringify( + contextSent.codeMode!.selectionRange, + )})`, ); assert.strictEqual( contextSent.codeMode!.moduleInspectorPanel, From 27e9c715ab6414170252a629205d849910a78d01 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 17 Aug 2026 19:52:57 -0400 Subject: [PATCH 3/5] host: Report the selectionRange for the other two context assertions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The failing one turned out to be a later stage of the same test — it asserts three times, against a cursor position and then a selection in plant.gts, and only the first was instrumented. Co-Authored-By: Claude Opus 5 --- packages/host/tests/acceptance/ai-assistant-test.gts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/host/tests/acceptance/ai-assistant-test.gts b/packages/host/tests/acceptance/ai-assistant-test.gts index 6a803f23db7..aa7162f9292 100644 --- a/packages/host/tests/acceptance/ai-assistant-test.gts +++ b/packages/host/tests/acceptance/ai-assistant-test.gts @@ -2074,7 +2074,9 @@ module('Acceptance | AI Assistant tests', function (hooks) { endLine: 3, endColumn: 45, }, - 'Context sent with message contains correct selectionRange', + `Context sent with message contains correct selectionRange (got ${JSON.stringify( + contextSent.codeMode!.selectionRange, + )})`, ); assert.strictEqual( contextSent.codeMode!.moduleInspectorPanel, @@ -2231,7 +2233,9 @@ module('Acceptance | AI Assistant tests', function (hooks) { endLine: 6, endColumn: 1, }, - 'Context sent with message contains correct selectionRange', + `Context sent with message contains correct selectionRange (got ${JSON.stringify( + contextSent.codeMode!.selectionRange, + )})`, ); assert.strictEqual( contextSent.codeMode!.moduleInspectorPanel, From a784f43b12db1641c6a04731a9eb9ee54708f041 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 18 Aug 2026 09:42:48 -0400 Subject: [PATCH 4/5] host: Keep the module-source fixtures' original indentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrapping the realm setup in `withCachedRealmSetup` indented its body, and that carried into the template literals holding plant.gts and broken-card.gts — template-literal content is data, so every line of those modules gained two leading spaces. `code mode context sent with message` asserts the cursor position that clicking a definition produces, which is derived from the module source, so it moved with the fixture: the instrumented message reported column 47 against an expected 45. Restore both fixtures to the bytes main has. The expectations were right; the fixtures had silently changed. Co-Authored-By: Claude Opus 5 --- .../tests/acceptance/ai-assistant-test.gts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/host/tests/acceptance/ai-assistant-test.gts b/packages/host/tests/acceptance/ai-assistant-test.gts index aa7162f9292..1179b1e037a 100644 --- a/packages/host/tests/acceptance/ai-assistant-test.gts +++ b/packages/host/tests/acceptance/ai-assistant-test.gts @@ -438,15 +438,15 @@ module('Acceptance | AI Assistant tests', function (hooks) { 'person.gts': { Person }, 'pet.gts': { Pet }, 'broken-card.gts': ` - import { CardDef, field, contains } from '@cardstack/base/card-api'; - import StringField from '@cardstack/base/string'; - import { BrokenField } from './does-not-exist'; - export class BrokenCard extends CardDef { - static displayName = 'Broken Card'; - @field name = contains(StringField); - @field broken = contains(BrokenField); - } - `, + import { CardDef, field, contains } from '@cardstack/base/card-api'; + import StringField from '@cardstack/base/string'; + import { BrokenField } from './does-not-exist'; + export class BrokenCard extends CardDef { + static displayName = 'Broken Card'; + @field name = contains(StringField); + @field broken = contains(BrokenField); + } + `, 'BrokenCard/errored.json': { data: { attributes: { @@ -489,12 +489,12 @@ module('Acceptance | AI Assistant tests', function (hooks) { friends: [mangoPet], }), 'plant.gts': ` - import { CardDef, field, contains, StringField } from '@cardstack/base/card-api'; - export class Plant extends CardDef { - static displayName = "Plant"; - @field commonName = contains(StringField); - } - `, + import { CardDef, field, contains, StringField } from '@cardstack/base/card-api'; + export class Plant extends CardDef { + static displayName = "Plant"; + @field commonName = contains(StringField); + } + `, 'Plant/highbush-blueberry.json': { data: { attributes: { From 0e0899f3163d98af1340a3739c27977881bf051b Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 18 Aug 2026 10:16:24 -0400 Subject: [PATCH 5/5] host: Drop the selectionRange diagnostics They named the value that identified the shifted fixture; the assertions go back to their plain messages now that the cause is fixed. Co-Authored-By: Claude Opus 5 --- .../host/tests/acceptance/ai-assistant-test.gts | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/host/tests/acceptance/ai-assistant-test.gts b/packages/host/tests/acceptance/ai-assistant-test.gts index 1179b1e037a..05950463621 100644 --- a/packages/host/tests/acceptance/ai-assistant-test.gts +++ b/packages/host/tests/acceptance/ai-assistant-test.gts @@ -1981,12 +1981,7 @@ module('Acceptance | AI Assistant tests', function (hooks) { endLine: 1, endColumn: 1, }, - // The value is in the message because the TAP reporter renders a failed - // deepEqual's operands as `[object Object]`, and a shard that fails - // uploads no test-report artifact to read them from. - `Context sent with message contains correct selectionRange (got ${JSON.stringify( - contextSent.codeMode!.selectionRange, - )})`, + 'Context sent with message contains correct selectionRange', ); assert.strictEqual( contextSent.codeMode!.moduleInspectorPanel, @@ -2074,9 +2069,7 @@ module('Acceptance | AI Assistant tests', function (hooks) { endLine: 3, endColumn: 45, }, - `Context sent with message contains correct selectionRange (got ${JSON.stringify( - contextSent.codeMode!.selectionRange, - )})`, + 'Context sent with message contains correct selectionRange', ); assert.strictEqual( contextSent.codeMode!.moduleInspectorPanel, @@ -2233,9 +2226,7 @@ module('Acceptance | AI Assistant tests', function (hooks) { endLine: 6, endColumn: 1, }, - `Context sent with message contains correct selectionRange (got ${JSON.stringify( - contextSent.codeMode!.selectionRange, - )})`, + 'Context sent with message contains correct selectionRange', ); assert.strictEqual( contextSent.codeMode!.moduleInspectorPanel,