Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions packages/cli/src/__tests__/pi-transcript.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3742,6 +3742,84 @@ describe('transcript entry render memoization', () => {
assert.match(rendered, /\(2s · 2 lines\)/);
});

test('provider retry activity strip counts down in the client clock domain', (t) => {
// #3393: a subscription quota window can hand the runtime an hours-long
// Retry-After. The strip stamps the client-local receipt time when the
// event lands and ticks down from it, so the display never mixes the
// (possibly remote) Runtime Host clock with the client clock.
const start = 1_700_000_000_000;
t.mock.timers.enable({ apis: ['Date'], now: start });
const state = createMakaPiTranscriptState();
applyMakaSessionEventToTranscript(
state,
event({
type: 'provider_retry',
phase: 'scheduled',
attempt: 2,
maxAttempts: 10,
delayMs: 16_083_000,
reason: 'rate_limit',
}),
);
// Receipt is stamped on the client clock at application time.
assert.equal(state.providerRetry?.receivedAtMs, start);

const strip = () =>
stripAnsi(renderMakaPiActivityStrip({ ...meta(), providerRetry: state.providerRetry }, 120));

// Hours-long waits render as a humanized duration, not a raw second count.
assert.match(strip(), /Retrying in 4h 28m 3s \(2\/10\)/);

// Elapsed time ticks the countdown down; zero-value units are omitted.
t.mock.timers.setTime(start + 63_000);
assert.match(strip(), /Retrying in 4h 27m \(2\/10\)/);

// An elapsed wait floors at 1s until `started` replaces the banner.
t.mock.timers.setTime(start + 17_000_000);
assert.match(strip(), /Retrying in 1s \(2\/10\)/);
});

test('provider retry strip counts down from the host-authoritative remainingMs', (t) => {
// A host re-projection mid-wait (reconnect) sends the recomputed
// remainingMs duration; the strip counts THAT down from receipt instead
// of restarting at the full delay.
const start = 1_700_000_000_000;
t.mock.timers.enable({ apis: ['Date'], now: start });
const state = createMakaPiTranscriptState();
applyMakaSessionEventToTranscript(
state,
event({
type: 'provider_retry',
phase: 'scheduled',
attempt: 2,
maxAttempts: 10,
delayMs: 16_083_000,
remainingMs: 61_000,
reason: 'rate_limit',
}),
);
assert.match(
stripAnsi(renderMakaPiActivityStrip({ ...meta(), providerRetry: state.providerRetry }, 120)),
/Retrying in 1m 1s \(2\/10\)/,
);

// The started phase carries no countdown at all.
applyMakaSessionEventToTranscript(
state,
event({
type: 'provider_retry',
phase: 'started',
attempt: 2,
maxAttempts: 10,
reason: 'rate_limit',
}),
);
assert.match(
stripAnsi(renderMakaPiActivityStrip({ ...meta(), providerRetry: state.providerRetry }, 120)),
/^Retrying \(2\/10\)$/,
);
});

test('re-renders equal-length ShellRun output only when revision advances', () => {
const state = createMakaPiTranscriptState();
applyMakaSessionEventToTranscript(
Expand Down
48 changes: 31 additions & 17 deletions packages/cli/src/pi-transcript.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import { Markdown, visibleWidth } from '@earendil-works/pi-tui';
import type {
ProviderRetryEvent,
ProviderRetryScheduledEvent,
SandboxBoundaryRequestEvent,
UserQuestionRequestEvent,
SessionEvent,
Expand All @@ -33,6 +34,7 @@ import {
type SystemNoteMessage,
} from '@maka/core/session';
import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types';
import { providerRetryDisplaySeconds } from '@maka/core/provider-retry-countdown';
import type { ThinkingLevel } from '@maka/core/model-thinking';
import type { UiLocale } from '@maka/core/ui-locale';
import { isActiveShellRunStatus } from '@maka/core/shell-run';
Expand Down Expand Up @@ -114,11 +116,23 @@ export interface MakaPiTranscriptState {
*/
pendingFallback: Array<{ text: string; enqueue: 'steer' | 'queue' }>;
/** Current non-durable provider retry progress for the activity strip. */
providerRetry?: ProviderRetryEvent;
providerRetry?: ProviderRetryCountdown;
}

export type MakaPiPendingInteraction = SandboxBoundaryRequestEvent | UserQuestionRequestEvent;

/**
* A provider retry event plus the CLIENT-local time it was applied. Counting
* down from `receivedAtMs` keeps the whole countdown in one clock domain —
* the event's own `ts` is stamped on the (possibly remote) Runtime Host
* clock, so subtracting it from a client clock would skew the display by the
* clock offset between the two machines.
*/
export interface ProviderRetryCountdown {
event: ProviderRetryEvent;
receivedAtMs: number;
}

export interface MakaPiRenderGeometry {
/**
* First rendered transcript-line index per entry, from the latest render.
Expand Down Expand Up @@ -196,7 +210,7 @@ export interface MakaPiTranscriptMetadata {
modelContextWindow?: number;
/** Elapsed milliseconds of the running agent turn, for the activity strip. */
turnElapsedMs?: number;
providerRetry?: ProviderRetryEvent;
providerRetry?: ProviderRetryCountdown;
/** Resolved locale for primary TUI guidance. Defaults to English for direct embeddings. */
uiLocale?: UiLocale;
/**
Expand Down Expand Up @@ -720,7 +734,7 @@ export function applyMakaSessionEventToTranscript(
break;

case 'provider_retry':
state.providerRetry = event;
state.providerRetry = { event, receivedAtMs: Date.now() };
break;

case 'token_usage': {
Expand Down Expand Up @@ -1409,29 +1423,29 @@ export function renderMakaPiActivityStrip(
): string {
const safeWidth = Math.max(1, width);
if (metadata.providerRetry) {
const retry = metadata.providerRetry;
const { event: retry, receivedAtMs } = metadata.providerRetry;
const text =
retry.phase === 'scheduled'
? `Retrying in ${formatRetryDuration(retry.delayMs)} (${retry.attempt}/${retry.maxAttempts})`
? `Retrying in ${formatRetryCountdown(retry, receivedAtMs)} (${retry.attempt}/${retry.maxAttempts})`
: `Retrying (${retry.attempt}/${retry.maxAttempts})`;
return fitLine(ansi.dim(text), safeWidth);
}
if (metadata.turnElapsedMs === undefined) return '';
return fitLine(ansi.dim(`Working… ${formatElapsedDuration(metadata.turnElapsedMs)}`), safeWidth);
}

function formatRetryDuration(delayMs: number): string {
let s = Math.max(1, Math.ceil(delayMs / 1_000));
const d = Math.floor(s / 86_400);
const h = Math.floor((s % 86_400) / 3_600);
const m = Math.floor((s % 3_600) / 60);
const sec = s % 60;
const parts: string[] = [];
if (d > 0) parts.push(`${d}d`);
if (h > 0) parts.push(`${h}h`);
if (m > 0) parts.push(`${m}m`);
if (sec > 0 || parts.length === 0) parts.push(`${sec}s`);
return parts.join(' ');
/**
* Remaining wait for a scheduled provider retry, ticked against the client's
* own receipt time so the strip counts down on the 1s heartbeat instead of
* pinning the original delay for the whole sleep. The computation itself is
* shared with the desktop banner in `@maka/core/provider-retry-countdown`.
* Long provider-mandated waits (a subscription quota window can be hours)
* render as `4h 28m 3s` via the shared duration formatter rather than a raw
* five-digit second count.
*/
function formatRetryCountdown(retry: ProviderRetryScheduledEvent, receivedAtMs: number): string {
const seconds = providerRetryDisplaySeconds(retry, Date.now() - receivedAtMs);
return formatElapsedDuration(seconds * 1_000);
}

function formatElapsedDuration(elapsedMs: number): string {
Expand Down
1 change: 1 addition & 0 deletions packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"./execution-log-coverage": "./dist/execution-log-coverage.js",
"./execution-inspect": "./dist/execution-inspect.js",
"./events": "./dist/events.js",
"./provider-retry-countdown": "./dist/provider-retry-countdown.js",
"./interaction": "./dist/interaction.js",
"./session": "./dist/session.js",
"./session-revisions": "./dist/session-revisions.js",
Expand Down
44 changes: 44 additions & 0 deletions packages/core/src/__tests__/provider-retry-countdown.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import assert from 'node:assert/strict';
import { test } from 'node:test';
import {
providerRetryDisplaySeconds,
providerRetryRemainingMs,
} from '../provider-retry-countdown.js';

test('providerRetryRemainingMs counts down from the granted length to a zero floor', () => {
// Host-authoritative remainingMs wins over the full delay (reconnect path).
assert.equal(
providerRetryRemainingMs({ delayMs: 3_600_000, remainingMs: 300_000 }, 60_000),
240_000,
);
// Older emitters lack remainingMs; the full delay is the fallback.
assert.equal(providerRetryRemainingMs({ delayMs: 10_000 }, 4_000), 6_000);
// One agreed floor across surfaces: an expired countdown reads zero.
assert.equal(providerRetryRemainingMs({ delayMs: 10_000 }, 60_000), 0);
// Clock jitter between emission and receipt never inflates the wait.
assert.equal(providerRetryRemainingMs({ delayMs: 10_000 }, -500), 10_000);
});

test('providerRetryDisplaySeconds floors the humanized countdown at 1s', () => {
assert.equal(providerRetryDisplaySeconds({ delayMs: 10_000 }, 60_000), 1);
assert.equal(providerRetryDisplaySeconds({ delayMs: 300_000 }, 0), 300);
});
10 changes: 10 additions & 0 deletions packages/core/src/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1140,6 +1140,16 @@ export interface ProviderRetryScheduledEvent extends BaseEvent {
attempt: number;
maxAttempts: number;
delayMs: number;
/**
* Authoritative remaining wait at emission, as a DURATION — unlike `ts`,
* it carries no clock domain, so a client on another machine (remote
* Runtime Host) can count it down from its own receipt time without being
* skewed against the host clock. Runtime sets it to `delayMs` at
* scheduling; a host re-projection mid-wait recomputes it from the stored
* schedule time. Absent from older emitters; clients fall back to
* `delayMs`.
*/
remainingMs?: number;
reason: ProviderRetryReason;
}

Expand Down
54 changes: 54 additions & 0 deletions packages/core/src/provider-retry-countdown.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import type { ProviderRetryScheduledEvent } from './events.js';

/**
* The one remaining-wait computation behind every provider retry countdown
* surface (TUI activity strip, desktop banner). Extracted in #3393 after the
* two client copies drifted apart at the expiry floor.
*
* `elapsedSinceReceiptMs` is measured on the CLIENT's own clock from the
* moment the event entered the client projection, keeping the whole
* computation in one clock domain; the granted length comes from the
* skew-free `remainingMs` duration when the emitter provided one (older
* emitters fall back to the full `delayMs`). Floors at zero: an expired
* countdown is `0ms` here; callers that render a humanized countdown floor
* the *displayed* seconds at `1s` via `providerRetryDisplaySeconds` so both
* surfaces agree until the `started` event replaces the banner.
*/
export function providerRetryRemainingMs(
retry: Pick<ProviderRetryScheduledEvent, 'delayMs' | 'remainingMs'>,
elapsedSinceReceiptMs: number,
): number {
const grantedMs = retry.remainingMs ?? retry.delayMs;
return Math.max(0, grantedMs - Math.max(0, elapsedSinceReceiptMs));
}

/**
* Humanized countdown value shared by every surface (#3393 P3). Floors at
* `1s` so an expired scheduled wait still reads `1s` on both the TUI strip
* and the desktop banner until the `started` event replaces it.
*/
export function providerRetryDisplaySeconds(
retry: Pick<ProviderRetryScheduledEvent, 'delayMs' | 'remainingMs'>,
elapsedSinceReceiptMs: number,
): number {
return Math.max(1, Math.ceil(providerRetryRemainingMs(retry, elapsedSinceReceiptMs) / 1_000));
}
18 changes: 18 additions & 0 deletions packages/runtime-host/src/__tests__/protocol.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,24 @@ describe('Runtime Host bootstrap protocol', () => {
},
};
assert.deepEqual(decodeSessionContinuitySnapshot(retrying), retrying);
// Snapshots written after #3393 carry the host-clock schedule time so a
// re-projection can recompute the remaining wait; the field is optional
// for older snapshots.
const retryingWithTs = {
...continuitySnapshot('epoch-1'),
rootTurn: {
...continuitySnapshot('epoch-1').rootTurn,
providerRetry: {
phase: 'scheduled' as const,
attempt: 8,
maxAttempts: 10,
delayMs: 40_000,
ts: 1_700_000_000_000,
reason: 'rate_limit' as const,
},
},
};
assert.deepEqual(decodeSessionContinuitySnapshot(retryingWithTs), retryingWithTs);
assert.throws(
() =>
decodeSessionContinuitySnapshot({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1727,6 +1727,9 @@ test('keeps the current provider retry on the live Turn until the next content e
attempt: 8,
maxAttempts: 10,
delayMs: 40_000,
// The host-clock schedule time is kept so a re-projection mid-wait can
// recompute the authoritative remaining duration (#3393).
ts: 1,
reason: 'rate_limit' as const,
};
coordinator.attachConnection('connection-remount', new RecordingSink());
Expand Down
32 changes: 32 additions & 0 deletions packages/runtime-host/src/__tests__/session-projector.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,38 @@ test('reseeds the latest provider retry when the active Turn still carries one',
assert.equal(seeded[0] && 'phase' in seeded[0] ? seeded[0].phase : undefined, 'scheduled');
});

test('reseeds a scheduled retry with remainingMs recomputed from the stored schedule time', () => {
// #3393: a reconnect mid-wait must not restart the countdown. The snapshot
// keeps the host-clock schedule time; the projector re-derives the skew-free
// remaining duration at projection time.
const projector = new RuntimeHostSessionProjector(
snapshot({
rootTurn: {
sessionId: 'session-1',
turnId: 'turn-1',
runId: 'run-1',
status: 'running',
providerRetry: {
phase: 'scheduled' as const,
attempt: 8,
maxAttempts: 10,
delayMs: 40_000,
ts: 5, // scheduled 5ms before the projector clock's `now`
reason: 'rate_limit' as const,
},
},
}),
createRuntimeHostSessionProjectionSeed([], snapshot()),
() => 10,
);

const seeded = projector.seedActive(true);
const retry = seeded[0];
assert.ok(retry && retry.type === 'provider_retry' && retry.phase === 'scheduled');
assert.equal(retry.delayMs, 40_000);
assert.equal(retry.remainingMs, 39_995);
});

test('emits a live provider retry when the snapshot overlay appears, then drops it after content', () => {
const projector = new RuntimeHostSessionProjector(
snapshot(),
Expand Down
Loading