diff --git a/packages/cli/src/__tests__/pi-transcript.test.ts b/packages/cli/src/__tests__/pi-transcript.test.ts index b57e43775c..5295a71713 100644 --- a/packages/cli/src/__tests__/pi-transcript.test.ts +++ b/packages/cli/src/__tests__/pi-transcript.test.ts @@ -3742,6 +3742,84 @@ describe('transcript entry render memoization', () => { assert.match(rendered, /\(2s · 2 lines\)/); }); + test('provider retry activity strip counts down in the client clock domain', (t) => { + // #3393: a subscription quota window can hand the runtime an hours-long + // Retry-After. The strip stamps the client-local receipt time when the + // event lands and ticks down from it, so the display never mixes the + // (possibly remote) Runtime Host clock with the client clock. + const start = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date'], now: start }); + const state = createMakaPiTranscriptState(); + applyMakaSessionEventToTranscript( + state, + event({ + type: 'provider_retry', + phase: 'scheduled', + attempt: 2, + maxAttempts: 10, + delayMs: 16_083_000, + reason: 'rate_limit', + }), + ); + // Receipt is stamped on the client clock at application time. + assert.equal(state.providerRetry?.receivedAtMs, start); + + const strip = () => + stripAnsi(renderMakaPiActivityStrip({ ...meta(), providerRetry: state.providerRetry }, 120)); + + // Hours-long waits render as a humanized duration, not a raw second count. + assert.match(strip(), /Retrying in 4h 28m 3s \(2\/10\)/); + + // Elapsed time ticks the countdown down; zero-value units are omitted. + t.mock.timers.setTime(start + 63_000); + assert.match(strip(), /Retrying in 4h 27m \(2\/10\)/); + + // An elapsed wait floors at 1s until `started` replaces the banner. + t.mock.timers.setTime(start + 17_000_000); + assert.match(strip(), /Retrying in 1s \(2\/10\)/); + }); + + test('provider retry strip counts down from the host-authoritative remainingMs', (t) => { + // A host re-projection mid-wait (reconnect) sends the recomputed + // remainingMs duration; the strip counts THAT down from receipt instead + // of restarting at the full delay. + const start = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date'], now: start }); + const state = createMakaPiTranscriptState(); + applyMakaSessionEventToTranscript( + state, + event({ + type: 'provider_retry', + phase: 'scheduled', + attempt: 2, + maxAttempts: 10, + delayMs: 16_083_000, + remainingMs: 61_000, + reason: 'rate_limit', + }), + ); + assert.match( + stripAnsi(renderMakaPiActivityStrip({ ...meta(), providerRetry: state.providerRetry }, 120)), + /Retrying in 1m 1s \(2\/10\)/, + ); + + // The started phase carries no countdown at all. + applyMakaSessionEventToTranscript( + state, + event({ + type: 'provider_retry', + phase: 'started', + attempt: 2, + maxAttempts: 10, + reason: 'rate_limit', + }), + ); + assert.match( + stripAnsi(renderMakaPiActivityStrip({ ...meta(), providerRetry: state.providerRetry }, 120)), + /^Retrying \(2\/10\)$/, + ); + }); + test('re-renders equal-length ShellRun output only when revision advances', () => { const state = createMakaPiTranscriptState(); applyMakaSessionEventToTranscript( diff --git a/packages/cli/src/pi-transcript.ts b/packages/cli/src/pi-transcript.ts index 35de5bbd80..8d89e35f2e 100644 --- a/packages/cli/src/pi-transcript.ts +++ b/packages/cli/src/pi-transcript.ts @@ -20,6 +20,7 @@ import { Markdown, visibleWidth } from '@earendil-works/pi-tui'; import type { ProviderRetryEvent, + ProviderRetryScheduledEvent, SandboxBoundaryRequestEvent, UserQuestionRequestEvent, SessionEvent, @@ -33,6 +34,7 @@ import { type SystemNoteMessage, } from '@maka/core/session'; import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types'; +import { providerRetryDisplaySeconds } from '@maka/core/provider-retry-countdown'; import type { ThinkingLevel } from '@maka/core/model-thinking'; import type { UiLocale } from '@maka/core/ui-locale'; import { isActiveShellRunStatus } from '@maka/core/shell-run'; @@ -114,11 +116,23 @@ export interface MakaPiTranscriptState { */ pendingFallback: Array<{ text: string; enqueue: 'steer' | 'queue' }>; /** Current non-durable provider retry progress for the activity strip. */ - providerRetry?: ProviderRetryEvent; + providerRetry?: ProviderRetryCountdown; } export type MakaPiPendingInteraction = SandboxBoundaryRequestEvent | UserQuestionRequestEvent; +/** + * A provider retry event plus the CLIENT-local time it was applied. Counting + * down from `receivedAtMs` keeps the whole countdown in one clock domain — + * the event's own `ts` is stamped on the (possibly remote) Runtime Host + * clock, so subtracting it from a client clock would skew the display by the + * clock offset between the two machines. + */ +export interface ProviderRetryCountdown { + event: ProviderRetryEvent; + receivedAtMs: number; +} + export interface MakaPiRenderGeometry { /** * First rendered transcript-line index per entry, from the latest render. @@ -196,7 +210,7 @@ export interface MakaPiTranscriptMetadata { modelContextWindow?: number; /** Elapsed milliseconds of the running agent turn, for the activity strip. */ turnElapsedMs?: number; - providerRetry?: ProviderRetryEvent; + providerRetry?: ProviderRetryCountdown; /** Resolved locale for primary TUI guidance. Defaults to English for direct embeddings. */ uiLocale?: UiLocale; /** @@ -720,7 +734,7 @@ export function applyMakaSessionEventToTranscript( break; case 'provider_retry': - state.providerRetry = event; + state.providerRetry = { event, receivedAtMs: Date.now() }; break; case 'token_usage': { @@ -1409,10 +1423,10 @@ export function renderMakaPiActivityStrip( ): string { const safeWidth = Math.max(1, width); if (metadata.providerRetry) { - const retry = metadata.providerRetry; + const { event: retry, receivedAtMs } = metadata.providerRetry; const text = retry.phase === 'scheduled' - ? `Retrying in ${formatRetryDuration(retry.delayMs)} (${retry.attempt}/${retry.maxAttempts})` + ? `Retrying in ${formatRetryCountdown(retry, receivedAtMs)} (${retry.attempt}/${retry.maxAttempts})` : `Retrying (${retry.attempt}/${retry.maxAttempts})`; return fitLine(ansi.dim(text), safeWidth); } @@ -1420,18 +1434,18 @@ export function renderMakaPiActivityStrip( return fitLine(ansi.dim(`Working… ${formatElapsedDuration(metadata.turnElapsedMs)}`), safeWidth); } -function formatRetryDuration(delayMs: number): string { - let s = Math.max(1, Math.ceil(delayMs / 1_000)); - const d = Math.floor(s / 86_400); - const h = Math.floor((s % 86_400) / 3_600); - const m = Math.floor((s % 3_600) / 60); - const sec = s % 60; - const parts: string[] = []; - if (d > 0) parts.push(`${d}d`); - if (h > 0) parts.push(`${h}h`); - if (m > 0) parts.push(`${m}m`); - if (sec > 0 || parts.length === 0) parts.push(`${sec}s`); - return parts.join(' '); +/** + * Remaining wait for a scheduled provider retry, ticked against the client's + * own receipt time so the strip counts down on the 1s heartbeat instead of + * pinning the original delay for the whole sleep. The computation itself is + * shared with the desktop banner in `@maka/core/provider-retry-countdown`. + * Long provider-mandated waits (a subscription quota window can be hours) + * render as `4h 28m 3s` via the shared duration formatter rather than a raw + * five-digit second count. + */ +function formatRetryCountdown(retry: ProviderRetryScheduledEvent, receivedAtMs: number): string { + const seconds = providerRetryDisplaySeconds(retry, Date.now() - receivedAtMs); + return formatElapsedDuration(seconds * 1_000); } function formatElapsedDuration(elapsedMs: number): string { diff --git a/packages/core/package.json b/packages/core/package.json index 88bb6d7655..f0086d4c2a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -21,6 +21,7 @@ "./execution-log-coverage": "./dist/execution-log-coverage.js", "./execution-inspect": "./dist/execution-inspect.js", "./events": "./dist/events.js", + "./provider-retry-countdown": "./dist/provider-retry-countdown.js", "./interaction": "./dist/interaction.js", "./session": "./dist/session.js", "./session-revisions": "./dist/session-revisions.js", diff --git a/packages/core/src/__tests__/provider-retry-countdown.test.ts b/packages/core/src/__tests__/provider-retry-countdown.test.ts new file mode 100644 index 0000000000..6e8076145b --- /dev/null +++ b/packages/core/src/__tests__/provider-retry-countdown.test.ts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + providerRetryDisplaySeconds, + providerRetryRemainingMs, +} from '../provider-retry-countdown.js'; + +test('providerRetryRemainingMs counts down from the granted length to a zero floor', () => { + // Host-authoritative remainingMs wins over the full delay (reconnect path). + assert.equal( + providerRetryRemainingMs({ delayMs: 3_600_000, remainingMs: 300_000 }, 60_000), + 240_000, + ); + // Older emitters lack remainingMs; the full delay is the fallback. + assert.equal(providerRetryRemainingMs({ delayMs: 10_000 }, 4_000), 6_000); + // One agreed floor across surfaces: an expired countdown reads zero. + assert.equal(providerRetryRemainingMs({ delayMs: 10_000 }, 60_000), 0); + // Clock jitter between emission and receipt never inflates the wait. + assert.equal(providerRetryRemainingMs({ delayMs: 10_000 }, -500), 10_000); +}); + +test('providerRetryDisplaySeconds floors the humanized countdown at 1s', () => { + assert.equal(providerRetryDisplaySeconds({ delayMs: 10_000 }, 60_000), 1); + assert.equal(providerRetryDisplaySeconds({ delayMs: 300_000 }, 0), 300); +}); diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index f73952184f..a536c8567f 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -1153,6 +1153,16 @@ export interface ProviderRetryScheduledEvent extends BaseEvent { attempt: number; maxAttempts: number; delayMs: number; + /** + * Authoritative remaining wait at emission, as a DURATION — unlike `ts`, + * it carries no clock domain, so a client on another machine (remote + * Runtime Host) can count it down from its own receipt time without being + * skewed against the host clock. Runtime sets it to `delayMs` at + * scheduling; a host re-projection mid-wait recomputes it from the stored + * schedule time. Absent from older emitters; clients fall back to + * `delayMs`. + */ + remainingMs?: number; reason: ProviderRetryReason; } diff --git a/packages/core/src/provider-retry-countdown.ts b/packages/core/src/provider-retry-countdown.ts new file mode 100644 index 0000000000..d5906999e8 --- /dev/null +++ b/packages/core/src/provider-retry-countdown.ts @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { ProviderRetryScheduledEvent } from './events.js'; + +/** + * The one remaining-wait computation behind every provider retry countdown + * surface (TUI activity strip, desktop banner). Extracted in #3393 after the + * two client copies drifted apart at the expiry floor. + * + * `elapsedSinceReceiptMs` is measured on the CLIENT's own clock from the + * moment the event entered the client projection, keeping the whole + * computation in one clock domain; the granted length comes from the + * skew-free `remainingMs` duration when the emitter provided one (older + * emitters fall back to the full `delayMs`). Floors at zero: an expired + * countdown is `0ms` here; callers that render a humanized countdown floor + * the *displayed* seconds at `1s` via `providerRetryDisplaySeconds` so both + * surfaces agree until the `started` event replaces the banner. + */ +export function providerRetryRemainingMs( + retry: Pick, + elapsedSinceReceiptMs: number, +): number { + const grantedMs = retry.remainingMs ?? retry.delayMs; + return Math.max(0, grantedMs - Math.max(0, elapsedSinceReceiptMs)); +} + +/** + * Humanized countdown value shared by every surface (#3393 P3). Floors at + * `1s` so an expired scheduled wait still reads `1s` on both the TUI strip + * and the desktop banner until the `started` event replaces it. + */ +export function providerRetryDisplaySeconds( + retry: Pick, + elapsedSinceReceiptMs: number, +): number { + return Math.max(1, Math.ceil(providerRetryRemainingMs(retry, elapsedSinceReceiptMs) / 1_000)); +} diff --git a/packages/runtime-host/src/__tests__/protocol.test.ts b/packages/runtime-host/src/__tests__/protocol.test.ts index c4f2fad83b..0b3a2618d6 100644 --- a/packages/runtime-host/src/__tests__/protocol.test.ts +++ b/packages/runtime-host/src/__tests__/protocol.test.ts @@ -317,6 +317,24 @@ describe('Runtime Host bootstrap protocol', () => { }, }; assert.deepEqual(decodeSessionContinuitySnapshot(retrying), retrying); + // Snapshots written after #3393 carry the host-clock schedule time so a + // re-projection can recompute the remaining wait; the field is optional + // for older snapshots. + const retryingWithTs = { + ...continuitySnapshot('epoch-1'), + rootTurn: { + ...continuitySnapshot('epoch-1').rootTurn, + providerRetry: { + phase: 'scheduled' as const, + attempt: 8, + maxAttempts: 10, + delayMs: 40_000, + ts: 1_700_000_000_000, + reason: 'rate_limit' as const, + }, + }, + }; + assert.deepEqual(decodeSessionContinuitySnapshot(retryingWithTs), retryingWithTs); assert.throws( () => decodeSessionContinuitySnapshot({ diff --git a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts index 4ca2ec4d36..6ea2ec7cb8 100644 --- a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts @@ -1727,6 +1727,9 @@ test('keeps the current provider retry on the live Turn until the next content e attempt: 8, maxAttempts: 10, delayMs: 40_000, + // The host-clock schedule time is kept so a re-projection mid-wait can + // recompute the authoritative remaining duration (#3393). + ts: 1, reason: 'rate_limit' as const, }; coordinator.attachConnection('connection-remount', new RecordingSink()); diff --git a/packages/runtime-host/src/__tests__/session-projector.test.ts b/packages/runtime-host/src/__tests__/session-projector.test.ts index b0a5ac001b..8ee4dc9454 100644 --- a/packages/runtime-host/src/__tests__/session-projector.test.ts +++ b/packages/runtime-host/src/__tests__/session-projector.test.ts @@ -269,6 +269,38 @@ test('reseeds the latest provider retry when the active Turn still carries one', assert.equal(seeded[0] && 'phase' in seeded[0] ? seeded[0].phase : undefined, 'scheduled'); }); +test('reseeds a scheduled retry with remainingMs recomputed from the stored schedule time', () => { + // #3393: a reconnect mid-wait must not restart the countdown. The snapshot + // keeps the host-clock schedule time; the projector re-derives the skew-free + // remaining duration at projection time. + const projector = new RuntimeHostSessionProjector( + snapshot({ + rootTurn: { + sessionId: 'session-1', + turnId: 'turn-1', + runId: 'run-1', + status: 'running', + providerRetry: { + phase: 'scheduled' as const, + attempt: 8, + maxAttempts: 10, + delayMs: 40_000, + ts: 5, // scheduled 5ms before the projector clock's `now` + reason: 'rate_limit' as const, + }, + }, + }), + createRuntimeHostSessionProjectionSeed([], snapshot()), + () => 10, + ); + + const seeded = projector.seedActive(true); + const retry = seeded[0]; + assert.ok(retry && retry.type === 'provider_retry' && retry.phase === 'scheduled'); + assert.equal(retry.delayMs, 40_000); + assert.equal(retry.remainingMs, 39_995); +}); + test('emits a live provider retry when the snapshot overlay appears, then drops it after content', () => { const projector = new RuntimeHostSessionProjector( snapshot(), diff --git a/packages/runtime-host/src/adapter/session-projector.ts b/packages/runtime-host/src/adapter/session-projector.ts index 08b14a9604..6b44328b94 100644 --- a/packages/runtime-host/src/adapter/session-projector.ts +++ b/packages/runtime-host/src/adapter/session-projector.ts @@ -754,15 +754,40 @@ function providerRetryEvent( root: LiveTurnSnapshot, ts: number, ): Extract { - if (!root.providerRetry) { + const retry = root.providerRetry; + if (!retry) { throw new Error('Non-terminal Turn snapshot has no provider retry'); } + if (retry.phase !== 'scheduled') { + return { + type: 'provider_retry', + id: `host-seed:${root.runId}:provider_retry`, + turnId: root.turnId, + ts, + phase: 'started', + attempt: retry.attempt, + maxAttempts: retry.maxAttempts, + reason: retry.reason, + }; + } + // remainingMs is the skew-free countdown authority for clients on another + // machine: a duration, recomputed from the host-clock schedule time stored + // in the snapshot, so a mid-wait re-projection (reconnect) does not restart + // the countdown. Snapshots from older runtimes lack `ts` and degrade to the + // full delay. + const remainingMs = + retry.ts === undefined ? retry.delayMs : Math.max(0, retry.delayMs - (ts - retry.ts)); return { type: 'provider_retry', id: `host-seed:${root.runId}:provider_retry`, turnId: root.turnId, ts, - ...root.providerRetry, + phase: 'scheduled', + attempt: retry.attempt, + maxAttempts: retry.maxAttempts, + delayMs: retry.delayMs, + remainingMs, + reason: retry.reason, }; } diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 30f01b2cb7..b1e67e2606 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -92,7 +92,11 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 50 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 51 as const; +// 51: Scheduled Turn provider-retry frames may carry an optional host-clock +// `ts`, letting a mid-wait re-projection recompute the authoritative +// remaining duration. Older peers decode the frame with an exact key list +// and reject the added field, so mixed peers must fail the handshake. // 50: WorkHub can append durable coordination summaries and admit tool-free // answers through its reserved Coordination Session authority. // 49: WorkHub resolves one durable Coordination Session per Runtime Host. diff --git a/packages/runtime-host/src/protocol/turn.ts b/packages/runtime-host/src/protocol/turn.ts index 100cb6c230..b4cc97ada9 100644 --- a/packages/runtime-host/src/protocol/turn.ts +++ b/packages/runtime-host/src/protocol/turn.ts @@ -167,6 +167,12 @@ export type TurnProviderRetry = attempt: number; maxAttempts: number; delayMs: number; + /** + * Host-clock time the wait was scheduled at, kept so a re-projection + * mid-wait can recompute the authoritative remaining duration. Absent + * from snapshots written by older runtimes. + */ + ts?: number; reason: ProviderRetryReason; } | { @@ -732,18 +738,18 @@ export function decodeTurnProviderRetry(value: unknown): TurnProviderRetry { if (attempt > maxAttempts) throw invalidProtocolFrame('Invalid Turn provider retry'); const reason = requireProviderRetryReason(record.reason); if (phase === 'scheduled') { - assertExactKeys(record, 'scheduled Turn provider retry', [ - 'phase', - 'attempt', - 'maxAttempts', - 'delayMs', - 'reason', - ]); + const requiredKeys = ['phase', 'attempt', 'maxAttempts', 'delayMs', 'reason'] as const; + assertExactKeys( + record, + 'scheduled Turn provider retry', + record.ts === undefined ? requiredKeys : [...requiredKeys, 'ts'], + ); return { phase, attempt, maxAttempts, delayMs: requireCount(record.delayMs, 'delayMs'), + ...(record.ts !== undefined ? { ts: requireCount(record.ts, 'ts') } : {}), reason, }; } diff --git a/packages/runtime-host/src/server/session-continuity-coordinator.ts b/packages/runtime-host/src/server/session-continuity-coordinator.ts index d5cc3104a5..9d21b88a36 100644 --- a/packages/runtime-host/src/server/session-continuity-coordinator.ts +++ b/packages/runtime-host/src/server/session-continuity-coordinator.ts @@ -1862,6 +1862,7 @@ function withProviderRetry( attempt: event.attempt, maxAttempts: event.maxAttempts, delayMs: event.delayMs, + ts: event.ts, reason: event.reason, } : { diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 6f1a31eb36..25ad9d692f 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -2611,6 +2611,7 @@ export class AiSdkBackend implements AgentBackend { attempt: nextAttempt, maxAttempts, delayMs, + remainingMs: delayMs, reason, } satisfies ProviderRetryEvent); await this.providerRetrySleep(delayMs, turnAbortController.signal); diff --git a/packages/ui/src/__tests__/live-turn-projection.test.ts b/packages/ui/src/__tests__/live-turn-projection.test.ts index 66fd53ca77..63d788c85f 100644 --- a/packages/ui/src/__tests__/live-turn-projection.test.ts +++ b/packages/ui/src/__tests__/live-turn-projection.test.ts @@ -183,7 +183,7 @@ describe('applyLiveTurnEvent', () => { delayMs: 4_000, reason: 'rate_limit', }); - assert.deepEqual(scheduled?.providerRetry, { + assert.deepEqual(scheduled?.providerRetry?.event, { type: 'provider_retry', id: 'retry-1', turnId: 'turn-1', @@ -194,6 +194,9 @@ describe('applyLiveTurnEvent', () => { delayMs: 4_000, reason: 'rate_limit', }); + // Receipt is stamped on the client clock so the countdown ticks in one + // clock domain, immune to skew against a remote Runtime Host. + assert.equal(typeof scheduled?.providerRetry?.receivedAtMs, 'number'); const started = applyLiveTurnEvent(scheduled, { type: 'provider_retry', @@ -205,16 +208,7 @@ describe('applyLiveTurnEvent', () => { maxAttempts: 10, reason: 'rate_limit', }); - assert.deepEqual(started?.providerRetry, { - type: 'provider_retry', - id: 'retry-2', - turnId: 'turn-1', - ts: 101, - phase: 'started', - attempt: 2, - maxAttempts: 10, - reason: 'rate_limit', - }); + assert.equal(started?.providerRetry?.event.phase, 'started'); const streamed = applyLiveTurnEvent(started, { type: 'text_delta', @@ -251,7 +245,7 @@ describe('applyLiveTurnEvent', () => { reason: 'provider_capacity', }); - assert.equal(started?.providerRetry?.reason, 'provider_capacity'); + assert.equal(started?.providerRetry?.event.reason, 'provider_capacity'); }); @@ -844,14 +838,17 @@ describe('reconcileTerminalLiveTurn', () => { turnId: 'turn-1', phase: 'streamed', providerRetry: { - type: 'provider_retry', - id: 'retry-1', - turnId: 'turn-1', - ts: 2, - phase: 'started', - attempt: 2, - maxAttempts: 3, - reason: 'network', + event: { + type: 'provider_retry', + id: 'retry-1', + turnId: 'turn-1', + ts: 2, + phase: 'started', + attempt: 2, + maxAttempts: 3, + reason: 'network', + }, + receivedAtMs: 2, }, steps: [{ stepId: 'assistant-1', diff --git a/packages/ui/src/__tests__/provider-retry-countdown.test.tsx b/packages/ui/src/__tests__/provider-retry-countdown.test.tsx new file mode 100644 index 0000000000..40b665e4dc --- /dev/null +++ b/packages/ui/src/__tests__/provider-retry-countdown.test.tsx @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { afterEach, test } from 'node:test'; +import { act } from 'react'; +import { createRoot } from 'react-dom/client'; +import { parseHTML } from 'linkedom'; +import type { ProviderRetryScheduledEvent } from '@maka/core/events'; +import type { LiveProviderRetry } from '../live-turn-projection.js'; +import { ModelProviderRetryIndicator } from '../chat-turn.js'; +import { LocaleProvider } from '../locale-context.js'; + +const originalGlobals = { + document: globalThis.document, + matchMedia: globalThis.matchMedia, + requestAnimationFrame: globalThis.requestAnimationFrame, + cancelAnimationFrame: globalThis.cancelAnimationFrame, + window: globalThis.window, +}; +const originalActEnvironment = (globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; +}).IS_REACT_ACT_ENVIRONMENT; + +const mountedRoots: ReturnType[] = []; + +afterEach(async () => { + // Unmount before restoring globals: React's cleanup reads `document`. + for (const root of mountedRoots.splice(0)) await act(() => root.unmount()); + Object.assign(globalThis, { + ...originalGlobals, + IS_REACT_ACT_ENVIRONMENT: originalActEnvironment, + }); +}); + +function domRoot() { + const { document, window } = parseHTML('
'); + Object.assign(globalThis, { + document, + window, + matchMedia: () => ({ matches: false, addEventListener() {}, removeEventListener() {} }), + requestAnimationFrame: () => 1, + cancelAnimationFrame() {}, + IS_REACT_ACT_ENVIRONMENT: true, + }); + // linkedom's window.setInterval resolves to globalThis.setInterval at call + // time, so `t.mock.timers` (which patches the global) drives the banner's + // one-second interval too — no adapter needed here. + const container = document.querySelector('#root'); + assert.ok(container); + const root = createRoot(container); + mountedRoots.push(root); + return { container, root }; +} + +function scheduledRetry( + overrides: Partial = {}, +): ProviderRetryScheduledEvent { + return { + type: 'provider_retry', + id: 'retry-1', + turnId: 'turn-1', + ts: 1, + phase: 'scheduled', + attempt: 2, + maxAttempts: 10, + delayMs: 10_000, + reason: 'rate_limit', + ...overrides, + }; +} + +async function renderRetry(root: ReturnType, retry: LiveProviderRetry) { + await act(() => + root.render( + + + , + ), + ); +} + +/** + * #3393: a subscription quota window can hand the runtime an hours-long + * Retry-After. The banner counts down against the CLIENT-local receipt time — + * a single clock domain, immune to skew between the client and a possibly + * remote Runtime Host clock. + */ +test('provider retry banner subtracts the time already waited since receipt', async (t) => { + const now = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date'], now }); + const { container, root } = domRoot(); + + await renderRetry(root, { event: scheduledRetry(), receivedAtMs: now }); + assert.match(container.textContent ?? '', /Retrying in 10s \(2\/10\)/); + + // Four seconds into the wait the same event renders the remaining six. + await renderRetry(root, { event: scheduledRetry(), receivedAtMs: now - 4_000 }); + assert.match(container.textContent ?? '', /Retrying in 6s \(2\/10\)/); +}); + +test('provider retry banner never shows a negative countdown', async (t) => { + const now = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date'], now }); + const { container, root } = domRoot(); + + await renderRetry(root, { event: scheduledRetry(), receivedAtMs: now - 60_000 }); + // Floors at 1s (formatRetryDelay uses Math.max(1, …)) until the `started` + // event replaces it. + assert.match(container.textContent ?? '', /Retrying in 1s \(2\/10\)/); +}); + +test('reduced motion keeps a correct static value at mount without ticking', async (t) => { + const now = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date', 'setInterval'], now }); + const { container, root } = domRoot(); + // The reduced-motion preference freezes the per-second tick, but the + // initial measurement still lands: four seconds into the wait the banner + // reads 6s from the start instead of pinning the full delay. + document.documentElement.dataset.makaReducedMotion = 'true'; + + await renderRetry(root, { event: scheduledRetry(), receivedAtMs: now - 4_000 }); + assert.match(container.textContent ?? '', /Retrying in 6s \(2\/10\)/); + + await act(() => t.mock.timers.tick(2_000)); + assert.match(container.textContent ?? '', /Retrying in 6s \(2\/10\)/); +}); + +test('provider retry banner counts down from remainingMs when the host provides it', async (t) => { + const now = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date'], now }); + const { container, root } = domRoot(); + + // A mid-wait host re-projection (reconnect) recomputes the remaining + // duration; the banner counts THAT down instead of restarting at delayMs. + await renderRetry(root, { + event: scheduledRetry({ delayMs: 3_600_000, remainingMs: 300_000 }), + receivedAtMs: now, + }); + assert.match(container.textContent ?? '', /Retrying in 5m \(2\/10\)/); +}); + +test('a mounted provider retry banner actually ticks once per second', async (t) => { + const now = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date', 'setInterval'], now }); + const { container, root } = domRoot(); + + await renderRetry(root, { event: scheduledRetry(), receivedAtMs: now }); + assert.match(container.textContent ?? '', /Retrying in 10s \(2\/10\)/); + + await act(() => t.mock.timers.tick(1_000)); + assert.match(container.textContent ?? '', /Retrying in 9s \(2\/10\)/); + + await act(() => t.mock.timers.tick(2_000)); + assert.match(container.textContent ?? '', /Retrying in 7s \(2\/10\)/); +}); + +test('the ticking countdown stays hidden from the live region, which keeps a stable label', async (t) => { + const now = 1_700_000_000_000; + t.mock.timers.enable({ apis: ['Date', 'setInterval'], now }); + const { container, root } = domRoot(); + + await renderRetry(root, { event: scheduledRetry(), receivedAtMs: now }); + const banner = container.querySelector('.maka-turn-provider-retry'); + assert.ok(banner); + assert.equal(banner.getAttribute('role'), 'status'); + // The stable accessible name carries reason + attempt — no countdown. + assert.equal(banner.getAttribute('aria-label'), 'Model rate limit reached · Waiting to retry (2/10)'); + // The visible countdown lives inside an aria-hidden subtree (the banner's + // status icon is aria-hidden too, so find the node carrying the text). + const tickingText = () => + [...banner.querySelectorAll('[aria-hidden="true"]')] + .map((node) => node.textContent ?? '') + .find((text) => /Retrying in/.test(text)); + assert.match(tickingText() ?? '', /Retrying in 10s/); + + // One second later the visual text ticks, the accessible name does not. + await act(() => t.mock.timers.tick(1_000)); + assert.equal(banner.getAttribute('aria-label'), 'Model rate limit reached · Waiting to retry (2/10)'); + assert.match(tickingText() ?? '', /Retrying in 9s/); +}); diff --git a/packages/ui/src/chat-turn.tsx b/packages/ui/src/chat-turn.tsx index e0e6f27383..ed44c869e2 100644 --- a/packages/ui/src/chat-turn.tsx +++ b/packages/ui/src/chat-turn.tsx @@ -54,9 +54,10 @@ import { SKILL_INVOCATION_TOKEN_SOURCE } from '@maka/core/skill-invocation-token import { type AttachmentRef, type InlineReference, - type ProviderRetryEvent, type QuoteRef, } from '@maka/core/events'; +import { type LiveProviderRetry } from './live-turn-projection.js'; +import { providerRetryDisplaySeconds } from '@maka/core/provider-retry-countdown'; import { finalAssistantReplyText, type TurnTimelineItem, @@ -420,7 +421,7 @@ export const TurnView = memo(function TurnView(props: { * looks abandoned and the user most needs to see it is still working. */ runningStatus?: boolean; - providerRetry?: ProviderRetryEvent; + providerRetry?: LiveProviderRetry; initialLiveContent?: ReadonlyMap; }; /** @@ -1062,24 +1063,67 @@ export function TurnRunningStatus(props: { ); } -export function ModelProviderRetryIndicator(props: { retry: ProviderRetryEvent }) { +export function ModelProviderRetryIndicator(props: { retry: LiveProviderRetry }) { const copy = getConversationCopy(useUiLocale()).messages; - const title = - props.retry.phase === 'scheduled' + const { event: retry, receivedAtMs } = props.retry; + const rootRef = useRef(null); + // Undefined until an effect measures it, so SSR and first paint render the + // granted delay untouched; the effect then counts down against the + // CLIENT-local receipt time (a single clock domain — the event's `ts` + // belongs to the possibly remote Runtime Host clock), taking its length + // from the skew-free `remainingMs` duration when the emitter provided one. + const [nowMs, setNowMs] = useState(undefined); + useEffect(() => { + if (retry.phase !== 'scheduled') return; + // The initial measurement sits OUTSIDE the motion gate on purpose: under + // a genuine reduced-motion preference the banner must still show the + // correct remaining wait at mount — gating it would pin the full delay + // for the whole wait, the exact #3393 symptom. Only the per-second tick + // respects the preference (and the frozen-fixture contract). + setNowMs(Date.now()); + if (!isTimeDrivenMotionEnabled(rootRef.current)) return; + const tick = window.setInterval(() => setNowMs(Date.now()), ELAPSED_TICK_MS); + return () => window.clearInterval(tick); + }, [retry.phase, retry.id, receivedAtMs]); + const displaySeconds = + retry.phase !== 'scheduled' + ? 0 + : // nowMs undefined (SSR / first paint) reads as zero elapsed. + providerRetryDisplaySeconds(retry, (nowMs ?? receivedAtMs) - receivedAtMs); + const titleText = + retry.phase === 'scheduled' ? copy.providerRetryScheduled( - Math.max(1, Math.ceil(props.retry.delayMs / 1_000)), - props.retry.attempt, - props.retry.maxAttempts, + displaySeconds, + retry.attempt, + retry.maxAttempts, ) - : copy.providerRetryStarted(props.retry.attempt, props.retry.maxAttempts); + : copy.providerRetryStarted(retry.attempt, retry.maxAttempts); + // The banner is a role="status" live region: a title that changes every + // second would be announced every second — for hours during a quota wait. + // The ticking text is aria-hidden; the region exposes a stable label that + // follows the running-turn indicator's pattern (the row's accessible name + // is the whole status, the moving text is decoration). + const scheduledA11y = retry.phase === 'scheduled'; return (