From 25aa6c0bdfbc775ee3cfb3a7f78fbcfb90fa7895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 13:25:45 +0000 Subject: [PATCH 1/4] fix(gc): right-size idle arena capacity --- .../perry-runtime/src/gc/arena_right_size.rs | 316 ++++++++++++++++++ crates/perry-runtime/src/gc/copying.rs | 2 +- crates/perry-runtime/src/gc/cycle.rs | 2 +- crates/perry-runtime/src/gc/idle_reclaim.rs | 87 ++++- crates/perry-runtime/src/gc/mod.rs | 5 + crates/perry-runtime/src/gc/policy.rs | 8 +- .../src/gc/tests/arena_right_size.rs | 170 ++++++++++ .../src/gc/tests/idle_reclaim.rs | 78 ++++- crates/perry-runtime/src/gc/tests/mod.rs | 1 + docs/src/internals/garbage-collector.md | 29 +- 10 files changed, 673 insertions(+), 25 deletions(-) create mode 100644 crates/perry-runtime/src/gc/arena_right_size.rs create mode 100644 crates/perry-runtime/src/gc/tests/arena_right_size.rs diff --git a/crates/perry-runtime/src/gc/arena_right_size.rs b/crates/perry-runtime/src/gc/arena_right_size.rs new file mode 100644 index 0000000000..adcacd15c5 --- /dev/null +++ b/crates/perry-runtime/src/gc/arena_right_size.rs @@ -0,0 +1,316 @@ +//! Arena capacity right-sizing for idle heaps. +//! +//! Reclaiming dead objects and returning arena capacity are deliberately two +//! different operations. General-arena blocks are released only after two +//! full collection observations: the first resets a proven-empty block and +//! the second proves that the mutator did not reuse it. The idle reducer used +//! to subtract its own fulls from its activity clock, so a burst followed by +//! complete silence received one full and then parked forever. Empty blocks +//! stopped at `dead_cycles == 1`, even when live bytes occupied a small +//! fraction of reserved arena capacity. +//! +//! This module turns sustained post-collection slack into a bounded debt that +//! [`super::idle_reclaim`] may service without new mutator activity: +//! +//! * capacity must be above [`ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES`]; +//! * live bytes must be at or below [`ARENA_RIGHT_SIZE_TRIGGER_PCT`] percent +//! for [`ARENA_RIGHT_SIZE_LOW_COLLECTIONS`] consecutive collections; +//! * the episode asks for enough idle fulls to reach +//! [`ARENA_RIGHT_SIZE_FULL_OBSERVATIONS`] full observations, counting fulls +//! already present in that low-utilization streak; +//! * it stops early in the target band and is bounded even when fragmentation +//! prevents a block release. +//! +//! The start and stop bands are intentionally different. After an episode +//! stops below the re-arm watermark, another cannot begin until utilization +//! rises to [`ARENA_RIGHT_SIZE_REARM_PCT`] percent or reserved capacity grows +//! materially. That hysteresis is the burst-then-idle-then-burst protection: +//! stable low occupancy cannot buy a full every timer interval, while a real +//! new peak can earn another right-size episode. + +use super::*; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Small heaps do not buy extra whole-heap work merely to save a few blocks. +pub const ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES: usize = 32 * 1024 * 1024; + +/// Open a right-size episode at or below this live/capacity percentage. +pub const ARENA_RIGHT_SIZE_TRIGGER_PCT: usize = 50; + +/// Stop an episode once live bytes reach this share of reserved capacity. +pub const ARENA_RIGHT_SIZE_TARGET_PCT: usize = 60; + +/// A completed episode below this utilization stays disarmed until the heap +/// grows materially. This gap from the trigger is the utilization hysteresis. +pub const ARENA_RIGHT_SIZE_REARM_PCT: usize = 70; + +/// Number of consecutive low-utilization collection results required. +pub const ARENA_RIGHT_SIZE_LOW_COLLECTIONS: u32 = 2; + +/// Full observations needed for the arena's two-cycle empty-block release. +pub const ARENA_RIGHT_SIZE_FULL_OBSERVATIONS: u8 = 2; + +/// Capacity growth that re-arms a disarmed episode is at least this many +/// bytes, as well as at least [`ARENA_RIGHT_SIZE_REARM_GROWTH_PCT`] percent of +/// the capacity at which it disarmed. +pub const ARENA_RIGHT_SIZE_REARM_GROWTH_MIN_BYTES: usize = 8 * 1024 * 1024; + +/// See [`ARENA_RIGHT_SIZE_REARM_GROWTH_MIN_BYTES`]. +pub const ARENA_RIGHT_SIZE_REARM_GROWTH_PCT: usize = 25; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(super) struct ArenaUsage { + pub(super) live_bytes: usize, + pub(super) capacity_bytes: usize, +} + +#[derive(Default)] +struct ArenaRightSizeState { + /// Consecutive post-collection samples in the trigger band. + low_collections: u32, + /// Full samples among `low_collections`, capped at the required count. + low_full_observations: u8, + /// Full collections still owed by the active episode. + fulls_remaining: u8, + /// A bounded episode finished without returning to the re-arm band. + disarmed: bool, + /// Capacity at disarm, used to recognize a later, material new peak. + disarmed_capacity_bytes: usize, + /// Capacity at the beginning of an active episode. + episode_start_capacity_bytes: usize, + /// Last post-collection sample, for diagnostics. + last_usage: ArenaUsage, +} + +thread_local! { + static STATE: RefCell = + RefCell::new(ArenaRightSizeState::default()); + #[cfg(test)] + static TEST_USAGE: Cell> = const { Cell::new(None) }; +} + +static EPISODES: AtomicU64 = AtomicU64::new(0); +static STARTS: AtomicU64 = AtomicU64::new(0); +static RELEASED_CAPACITY_BYTES: AtomicU64 = AtomicU64::new(0); + +/// Sustained-low-utilization episodes opened in this process. +pub fn arena_right_size_episodes() -> u64 { + EPISODES.load(Ordering::Relaxed) +} + +/// Idle fulls started specifically to service arena right-size debt. +pub fn arena_right_size_starts() -> u64 { + STARTS.load(Ordering::Relaxed) +} + +/// Reserved arena bytes removed over completed right-size episodes. +pub fn arena_right_size_released_capacity_bytes() -> u64 { + RELEASED_CAPACITY_BYTES.load(Ordering::Relaxed) +} + +#[inline] +fn utilization_at_most(usage: ArenaUsage, pct: usize) -> bool { + (usage.live_bytes as u128) * 100 <= (usage.capacity_bytes as u128) * (pct as u128) +} + +#[inline] +fn utilization_at_least(usage: ArenaUsage, pct: usize) -> bool { + (usage.live_bytes as u128) * 100 >= (usage.capacity_bytes as u128) * (pct as u128) +} + +fn in_trigger_band(usage: ArenaUsage) -> bool { + usage.capacity_bytes > ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES + && utilization_at_most(usage, ARENA_RIGHT_SIZE_TRIGGER_PCT) +} + +fn at_target(usage: ArenaUsage) -> bool { + usage.capacity_bytes <= ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES + || utilization_at_least(usage, ARENA_RIGHT_SIZE_TARGET_PCT) +} + +fn materially_regrew(usage: ArenaUsage, disarmed_capacity_bytes: usize) -> bool { + let relative = disarmed_capacity_bytes.saturating_mul(ARENA_RIGHT_SIZE_REARM_GROWTH_PCT) / 100; + let required = relative.max(ARENA_RIGHT_SIZE_REARM_GROWTH_MIN_BYTES); + usage.capacity_bytes.saturating_sub(disarmed_capacity_bytes) >= required +} + +fn rearm_reached(usage: ArenaUsage, disarmed_capacity_bytes: usize) -> bool { + utilization_at_least(usage, ARENA_RIGHT_SIZE_REARM_PCT) + || materially_regrew(usage, disarmed_capacity_bytes) +} + +fn current_usage(live_bytes: usize) -> ArenaUsage { + #[cfg(test)] + if let Some(usage) = TEST_USAGE.with(Cell::get) { + return usage; + } + ArenaUsage { + live_bytes, + capacity_bytes: crate::arena::arena_total_bytes(), + } +} + +fn reset_low_streak(st: &mut ArenaRightSizeState) { + st.low_collections = 0; + st.low_full_observations = 0; +} + +fn finish_episode(st: &mut ArenaRightSizeState, usage: ArenaUsage) { + let released = st + .episode_start_capacity_bytes + .saturating_sub(usage.capacity_bytes); + RELEASED_CAPACITY_BYTES.fetch_add(released as u64, Ordering::Relaxed); + st.fulls_remaining = 0; + st.episode_start_capacity_bytes = 0; + reset_low_streak(st); + st.disarmed = !utilization_at_least(usage, ARENA_RIGHT_SIZE_REARM_PCT); + st.disarmed_capacity_bytes = if st.disarmed { usage.capacity_bytes } else { 0 }; +} + +/// Observe the exact post-collection live census and current reserved block +/// capacity. `full` is true only when this collection supplied a whole-heap +/// sweep observation; copied and non-moving minors are still utilization +/// samples but cannot satisfy the two full observations by themselves. +pub(super) fn note_collection_finished(live_bytes: usize, full: bool) { + let usage = current_usage(live_bytes); + STATE.with(|state| { + let mut st = state.borrow_mut(); + st.last_usage = usage; + + if st.disarmed { + if !rearm_reached(usage, st.disarmed_capacity_bytes) { + return; + } + st.disarmed = false; + st.disarmed_capacity_bytes = 0; + reset_low_streak(&mut st); + } + + if st.fulls_remaining != 0 { + if at_target(usage) { + finish_episode(&mut st, usage); + return; + } + if full { + st.fulls_remaining = st.fulls_remaining.saturating_sub(1); + if st.fulls_remaining == 0 { + // Fragmentation or the protected recent-block window can + // make two full observations unable to hit the target. + // End the bounded episode instead of collecting forever. + finish_episode(&mut st, usage); + } + } + return; + } + + if !in_trigger_band(usage) { + reset_low_streak(&mut st); + return; + } + + st.low_collections = st + .low_collections + .saturating_add(1) + .min(ARENA_RIGHT_SIZE_LOW_COLLECTIONS); + if full { + st.low_full_observations = st + .low_full_observations + .saturating_add(1) + .min(ARENA_RIGHT_SIZE_FULL_OBSERVATIONS); + } + if st.low_collections < ARENA_RIGHT_SIZE_LOW_COLLECTIONS { + return; + } + + st.episode_start_capacity_bytes = usage.capacity_bytes; + st.fulls_remaining = + ARENA_RIGHT_SIZE_FULL_OBSERVATIONS.saturating_sub(st.low_full_observations); + reset_low_streak(&mut st); + EPISODES.fetch_add(1, Ordering::Relaxed); + if st.fulls_remaining == 0 { + finish_episode(&mut st, usage); + } + }); +} + +/// Whether the idle reducer may start a full without new mutator collection +/// activity. Quiet-time and rate gates remain the reducer's responsibility. +pub(super) fn owed() -> bool { + STATE.with(|state| state.borrow().fulls_remaining != 0) +} + +/// Record that the idle reducer successfully opened a full for this debt. +pub(super) fn note_started() { + STARTS.fetch_add(1, Ordering::Relaxed); +} + +pub(super) fn snapshot() -> (u32, u8, bool, ArenaUsage) { + STATE.with(|state| { + let st = state.borrow(); + ( + st.low_collections, + st.fulls_remaining, + st.disarmed, + st.last_usage, + ) + }) +} + +/// `PERRY_GC_DIAG=1` exit line. +pub(super) fn emit_diag() { + let (low_collections, fulls_remaining, disarmed, usage) = snapshot(); + eprintln!( + "[gc-arena-right-size] episodes={} starts={} released_capacity_bytes={} \ + low_collections={} fulls_remaining={} disarmed={} arena_live={} arena_capacity={}", + arena_right_size_episodes(), + arena_right_size_starts(), + arena_right_size_released_capacity_bytes(), + low_collections, + fulls_remaining, + disarmed, + usage.live_bytes, + usage.capacity_bytes, + ); +} + +#[cfg(test)] +pub(super) mod test_support { + use super::*; + + pub(crate) fn set_test_usage(usage: Option) { + TEST_USAGE.with(|cell| cell.set(usage)); + } + + pub(crate) fn observe(usage: ArenaUsage, full: bool) { + set_test_usage(Some(usage)); + note_collection_finished(usage.live_bytes, full); + } + + pub(crate) fn reset_state() { + STATE.with(|state| *state.borrow_mut() = ArenaRightSizeState::default()); + } + + pub(crate) fn state_snapshot() -> (u32, u8, bool, ArenaUsage) { + snapshot() + } + + pub(crate) struct ArenaRightSizeTestGuard; + + impl ArenaRightSizeTestGuard { + pub(crate) fn new() -> Self { + reset_state(); + set_test_usage(Some(ArenaUsage { + live_bytes: ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES, + capacity_bytes: ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES, + })); + Self + } + } + + impl Drop for ArenaRightSizeTestGuard { + fn drop(&mut self) { + set_test_usage(None); + reset_state(); + } + } +} diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 05b42a59f6..6c53d857e8 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1739,7 +1739,7 @@ pub(super) fn run_copied_minor_attempt( // flip and the new active survivor holds only the copies, so from-space // live == from-space high-water. crate::arena::record_arena_live_census(arena_live_bytes, None); - note_collection_finished_arena_occupancy(); + note_collection_finished_arena_occupancy(false); // The same argument one trigger over: a young generation that did not die // is a heap growing by LIVE data, so arena-growth pacing must not read that // growth as garbage accumulating. Fed after publishing the census so the diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 7e4a8ce01d..01433cae8e 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1956,7 +1956,7 @@ impl GcCycleState { // #7865: arena-growth pacing tests a POST-collection occupancy, which // is the same kind of quantity as its post-full baseline. Recorded here // rather than per-kind because this is the one site both kinds reach. - super::policy::note_collection_finished_arena_occupancy(); + super::policy::note_collection_finished_arena_occupancy(self.minor.is_none()); if self.minor.is_none() { finish_full_old_reclaim_baseline(); } diff --git a/crates/perry-runtime/src/gc/idle_reclaim.rs b/crates/perry-runtime/src/gc/idle_reclaim.rs index 709658c86b..66d800961b 100644 --- a/crates/perry-runtime/src/gc/idle_reclaim.rs +++ b/crates/perry-runtime/src/gc/idle_reclaim.rs @@ -48,10 +48,12 @@ //! //! Three gates, all O(1), evaluated at every park: //! -//! 1. **Activity.** At least `2^backoff` collections the reducer did not start -//! itself have completed since its last full. A collection is the one -//! signal that the mutator allocated enough to matter; a heap nobody has -//! touched since the last idle full has nothing new for another to find. +//! 1. **Activity or arena debt.** Normally at least `2^backoff` collections +//! the reducer did not start itself have completed since its last full. A +//! collection is the signal that the mutator allocated enough to matter. +//! The exception is a bounded [`super::arena_right_size`] episode: arena +//! blocks need two full observations before their mappings can be returned, +//! and an idle heap cannot create the second through mutator activity. //! 2. **Quiet.** At least [`IDLE_RECLAIM_QUIET_MS`] since the last such //! collection was observed — a burst still in progress collects every few //! hundred milliseconds and must not be interleaved with a whole-heap mark. @@ -137,6 +139,25 @@ pub(crate) enum ParkVerdict { Park(u64), } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StartReason { + /// The existing memory-reducer signal: the mutator completed enough + /// collections and then went quiet. + Activity, + /// Sustained arena slack still needs full observations before empty blocks + /// can be returned, even though the mutator has done nothing new. + ArenaRightSize, +} + +impl StartReason { + fn as_str(self) -> &'static str { + match self { + StartReason::Activity => "activity", + StartReason::ArenaRightSize => "arena_right_size", + } + } +} + #[derive(Default)] struct IdleReclaimState { /// Collections not started by the reducer, as of the last observation. @@ -319,7 +340,7 @@ fn old_gen_occupancy() -> usize { /// Observe the external-collection counter and decide whether a reducer full /// is owed right now. Pure bookkeeping — no collector state is touched. -fn should_start(now: u64) -> bool { +fn start_reason(now: u64) -> Option { // Read the counters BEFORE taking the state borrow: `external_collections` // borrows the same cell. let external = external_collections(); @@ -330,22 +351,29 @@ fn should_start(now: u64) -> bool { st.last_seen_external = external; st.last_external_change_ms = now; } - let since_attempt = external.saturating_sub(st.external_at_last_attempt); - if since_attempt < (1u64 << st.backoff_shift) { - return false; - } if now.saturating_sub(st.last_external_change_ms) < IDLE_RECLAIM_QUIET_MS { - return false; + return None; } if st.attempts > 0 && now.saturating_sub(st.last_attempt_ms) < IDLE_RECLAIM_MIN_INTERVAL_MS { - return false; + return None; + } + // Arena capacity release itself needs multiple full observations. Once + // sustained low utilization has created that bounded debt, requiring + // another mutator collection here recreates #9709's deadlock: the + // mutator is idle precisely because there is no more activity. + if super::arena_right_size::owed() { + return Some(StartReason::ArenaRightSize); + } + let since_attempt = external.saturating_sub(st.external_at_last_attempt); + if since_attempt < (1u64 << st.backoff_shift) { + return None; } - true + Some(StartReason::Activity) }) } -fn note_started(now: u64) { +fn note_started(now: u64, reason: StartReason) { STATE.with(|s| { let mut st = s.borrow_mut(); st.attempts += 1; @@ -353,10 +381,23 @@ fn note_started(now: u64) { st.external_at_last_attempt = st.last_seen_external; st.old_in_use_at_start = old_gen_occupancy(); ATTEMPTS.fetch_add(1, Ordering::Relaxed); + if reason == StartReason::ArenaRightSize { + super::arena_right_size::note_started(); + } if gc_diag_enabled() { + let (_, right_size_fulls_remaining, _, usage) = super::arena_right_size::snapshot(); eprintln!( - "[gc-idle-reclaim] start attempt={} external_collections={} backoff_shift={} old_in_use={}", - st.attempts, st.last_seen_external, st.backoff_shift, st.old_in_use_at_start + "[gc-idle-reclaim] start attempt={} reason={} external_collections={} \ + backoff_shift={} old_in_use={} arena_live={} arena_capacity={} \ + right_size_fulls_remaining={}", + st.attempts, + reason.as_str(), + st.last_seen_external, + st.backoff_shift, + st.old_in_use_at_start, + usage.live_bytes, + usage.capacity_bytes, + right_size_fulls_remaining, ); } }); @@ -491,14 +532,14 @@ pub(crate) fn park_hook(budget_ms: u64) -> ParkVerdict { if super::idle_compact::maybe_compact(now) { return ParkVerdict::Resume; } - if !should_start(now) { + let Some(reason) = start_reason(now) else { return ParkVerdict::Park(budget_ms); - } + }; if !policy::gc_idle_reclaim_try_start() { START_BLOCKED.fetch_add(1, Ordering::Relaxed); return ParkVerdict::Park(budget_ms); } - note_started(now); + note_started(now, reason); drive_active_cycle(deadline) } @@ -580,6 +621,14 @@ pub(super) mod test_support { // before the hook runs. crate::event_pump::clear_main_thread_notified_for_test(); reset_state(); + super::super::arena_right_size::test_support::reset_state(); + super::super::arena_right_size::test_support::set_test_usage(Some( + super::super::arena_right_size::ArenaUsage { + live_bytes: super::super::arena_right_size::ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES, + capacity_bytes: + super::super::arena_right_size::ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES, + }, + )); set_test_now_ms(Some(now_ms)); set_test_enabled(Some(true)); set_test_max_slices(None); @@ -597,6 +646,8 @@ pub(super) mod test_support { set_test_slice_us(None); set_test_work_charge_ms(None); reset_state(); + super::super::arena_right_size::test_support::set_test_usage(None); + super::super::arena_right_size::test_support::reset_state(); } } } diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 9041e56360..68039d1e9f 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -47,6 +47,10 @@ mod heap_budget; pub(crate) use heap_budget::*; mod pressure; pub use pressure::*; +mod arena_right_size; +pub use arena_right_size::{ + arena_right_size_episodes, arena_right_size_released_capacity_bytes, arena_right_size_starts, +}; mod idle_compact; mod idle_reclaim; pub use idle_compact::{ @@ -1371,6 +1375,7 @@ fn emit_incremental_liveness_diag() { ); idle_reclaim::emit_diag(); idle_compact::emit_diag(); + arena_right_size::emit_diag(); emit_step_bounds_diag(); emit_gc_time_share_diag(); } diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index a81a6e3531..0977d5e70a 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1904,12 +1904,14 @@ pub(super) fn pacing_arena_in_use_bytes() -> usize { } /// Record the post-collection live arena bytes arena-growth pacing tests -/// against. Called once at the end of every cycle, minor and full alike. The -/// copying fast path publishes directly; non-copying cycles publish from +/// against, and feed the same exact census to the idle arena right-sizer. +/// Called once at the end of every cycle, minor and full alike. The copying +/// fast path publishes directly; non-copying cycles publish from /// `GcCycle::publish_reclaim_outcome` after their sweep census. -pub(super) fn note_collection_finished_arena_occupancy() { +pub(super) fn note_collection_finished_arena_occupancy(full: bool) { let bytes = pacing_arena_in_use_bytes(); GC_LAST_COLLECTION_POST_IN_USE_BYTES.with(|cell| cell.set(bytes)); + super::arena_right_size::note_collection_finished(bytes, full); } /// The arena reading [`arena_growth_full_escalation_due`] tests — see diff --git a/crates/perry-runtime/src/gc/tests/arena_right_size.rs b/crates/perry-runtime/src/gc/tests/arena_right_size.rs new file mode 100644 index 0000000000..07b36d67c1 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/arena_right_size.rs @@ -0,0 +1,170 @@ +//! Arena right-sizing policy: sustained-low-utilization detection, full-pass +//! accounting, and the hysteresis that bounds idle work (#9709). + +use super::super::arena_right_size::test_support::*; +use super::super::arena_right_size::*; + +const MIB: usize = 1024 * 1024; + +fn usage(live_bytes: usize, capacity_bytes: usize) -> ArenaUsage { + ArenaUsage { + live_bytes, + capacity_bytes, + } +} + +fn low_usage(capacity_bytes: usize) -> ArenaUsage { + usage( + capacity_bytes * ARENA_RIGHT_SIZE_TRIGGER_PCT / 100, + capacity_bytes, + ) +} + +#[test] +fn low_utilization_must_persist_and_the_capacity_floor_is_strict() { + let _guard = ArenaRightSizeTestGuard::new(); + let capacity = 100 * MIB; + + observe(low_usage(capacity), false); + assert!(!owed(), "one low collection is only a transient"); + assert_eq!(state_snapshot().0, 1); + + observe(low_usage(capacity), false); + assert!(owed(), "two consecutive low collections open an episode"); + assert_eq!(state_snapshot().1, ARENA_RIGHT_SIZE_FULL_OBSERVATIONS); + + reset_state(); + let floor = ARENA_RIGHT_SIZE_MIN_CAPACITY_BYTES; + observe(low_usage(floor), false); + observe(low_usage(floor), false); + assert!(!owed(), "capacity exactly at the floor is left alone"); + + reset_state(); + let one_over_trigger = usage(capacity * ARENA_RIGHT_SIZE_TRIGGER_PCT / 100 + 1, capacity); + observe(one_over_trigger, false); + observe(one_over_trigger, false); + assert!( + !owed(), + "one byte above the utilization trigger resets the streak" + ); +} + +#[test] +fn fulls_already_in_the_low_streak_count_toward_block_release() { + let _guard = ArenaRightSizeTestGuard::new(); + let low = low_usage(100 * MIB); + + observe(low, false); + observe(low, true); + assert_eq!( + state_snapshot().1, + ARENA_RIGHT_SIZE_FULL_OBSERVATIONS - 1, + "the full that established sustained slack is the first release observation" + ); + + reset_state(); + observe(low, false); + observe(low, false); + assert_eq!( + state_snapshot().1, + ARENA_RIGHT_SIZE_FULL_OBSERVATIONS, + "minor samples establish utilization but do not impersonate full sweeps" + ); + + reset_state(); + observe(low, true); + observe(low, true); + assert!( + !owed(), + "two full observations already paid the bounded episode" + ); + assert!(state_snapshot().2, "unchanged low utilization disarms it"); +} + +#[test] +fn an_episode_is_bounded_and_utilization_hysteresis_rearms_it() { + let _guard = ArenaRightSizeTestGuard::new(); + let capacity = 100 * MIB; + let low = low_usage(capacity); + + observe(low, false); + observe(low, false); + assert_eq!(state_snapshot().1, 2); + observe(low, true); + assert_eq!(state_snapshot().1, 1); + observe(low, true); + assert!(!owed()); + assert!(state_snapshot().2, "two fulls are the hard work bound"); + + for _ in 0..8 { + observe(low, false); + } + assert!( + !owed(), + "a stable low heap must not buy another episode on periodic minors" + ); + + let rearmed = usage(capacity * ARENA_RIGHT_SIZE_REARM_PCT / 100, capacity); + observe(rearmed, false); + assert!(!state_snapshot().2, "the re-arm watermark ends hysteresis"); + observe(low, false); + observe(low, false); + assert!(owed(), "a later low-utilization epoch gets its own episode"); +} + +#[test] +fn material_capacity_regrowth_rearms_without_requiring_a_large_live_set() { + let _guard = ArenaRightSizeTestGuard::new(); + let capacity = 100 * MIB; + let low = low_usage(capacity); + + // Two low FULL samples consume the episode immediately and disarm it at + // `capacity`, without needing to start a synthetic collector in this pure + // policy test. + observe(low, true); + observe(low, true); + assert!(state_snapshot().2); + + let growth = (capacity * ARENA_RIGHT_SIZE_REARM_GROWTH_PCT / 100) + .max(ARENA_RIGHT_SIZE_REARM_GROWTH_MIN_BYTES); + observe(low_usage(capacity + growth - 1), false); + assert!(state_snapshot().2, "one byte short of material growth"); + observe(low_usage(capacity + growth), false); + assert!(!state_snapshot().2, "a real new capacity peak re-arms"); + assert_eq!( + state_snapshot().0, + 1, + "the re-arm sample starts the new streak" + ); + observe(low_usage(capacity + growth), false); + assert!(owed()); +} + +#[test] +fn reaching_the_target_stops_early_and_records_capacity_released() { + let _guard = ArenaRightSizeTestGuard::new(); + let start_capacity = 100 * MIB; + let low = low_usage(start_capacity); + let released_before = arena_right_size_released_capacity_bytes(); + + observe(low, false); + observe(low, false); + assert!(owed()); + + let target_capacity = 70 * MIB; + let target = usage( + target_capacity * ARENA_RIGHT_SIZE_TARGET_PCT / 100, + target_capacity, + ); + observe(target, true); + + assert!(!owed(), "the target band cancels the remaining full"); + assert_eq!( + arena_right_size_released_capacity_bytes(), + released_before + (start_capacity - target_capacity) as u64 + ); + assert!( + state_snapshot().2, + "target is deliberately below the higher re-arm watermark" + ); +} diff --git a/crates/perry-runtime/src/gc/tests/idle_reclaim.rs b/crates/perry-runtime/src/gc/tests/idle_reclaim.rs index 4b37368bac..eacee0ba1f 100644 --- a/crates/perry-runtime/src/gc/tests/idle_reclaim.rs +++ b/crates/perry-runtime/src/gc/tests/idle_reclaim.rs @@ -120,7 +120,8 @@ fn idle_reclaim_runs_a_full_at_the_park_when_owed() { "rooted survivor intact and unmoved" ); - // Gate 1: activity. Nothing collected since — no second attempt, ever. + // Gate 1: activity. Nothing collected since, and this test guard keeps the + // arena right-size gate below its capacity floor, so no second attempt. set_test_now_ms(Some( IDLE_RECLAIM_QUIET_MS + IDLE_RECLAIM_MIN_INTERVAL_MS + 1, )); @@ -147,6 +148,81 @@ fn idle_reclaim_runs_a_full_at_the_park_when_owed() { drive_until_idle(t + IDLE_RECLAIM_QUIET_MS, 1000); } +/// #9709: general-arena block release deliberately needs two full collection +/// observations. One external collection plus the ordinary idle-reducer full +/// establishes sustained low utilization; that full is observation one, and +/// the right-sizer must grant observation two without demanding new mutator +/// activity. The episode then disarms, so the bypass cannot become a periodic +/// full-GC loop. +#[test] +fn sustained_arena_slack_gets_one_bounded_followup_without_mutator_activity() { + let _guard = CopyingNurseryTestGuard::new(1); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _reducer = IdleReclaimTestGuard::new(0); + let capacity = 100 * 1024 * 1024; + super::super::arena_right_size::test_support::set_test_usage(Some( + super::super::arena_right_size::ArenaUsage { + live_bytes: capacity / 2, + capacity_bytes: capacity, + }, + )); + let right_size_starts_before = arena_right_size_starts(); + + // First low sample, and the only mutator-driven collection in this test. + external_collection_observed_at(0); + assert_eq!( + super::super::arena_right_size::test_support::state_snapshot().0, + 1 + ); + + // The ordinary activity arm starts the first full. Its post-collection + // sample opens the episode and counts as the first full observation. + set_test_now_ms(Some(IDLE_RECLAIM_QUIET_MS)); + assert!(resumes(idle_reclaim_park_hook(1000))); + drive_until_idle(IDLE_RECLAIM_QUIET_MS, 1000); + assert_eq!(thread_attempts(), 1); + assert_eq!( + super::super::arena_right_size::test_support::state_snapshot().1, + 1, + "one full observation remains" + ); + + // The normal activity gate is not re-armed. The capacity debt alone must + // cross the same rate floor and start the second full. + set_test_now_ms(Some( + IDLE_RECLAIM_QUIET_MS + IDLE_RECLAIM_MIN_INTERVAL_MS - 1, + )); + assert!(parks(idle_reclaim_park_hook(1000))); + assert_eq!(thread_attempts(), 1, "the rate floor still applies"); + + let followup_at = IDLE_RECLAIM_QUIET_MS + IDLE_RECLAIM_MIN_INTERVAL_MS; + set_test_now_ms(Some(followup_at)); + assert!(resumes(idle_reclaim_park_hook(1000))); + assert_eq!(thread_attempts(), 2); + assert_eq!( + arena_right_size_starts(), + right_size_starts_before + 1, + "LIVE SUBJECT: the follow-up must identify the capacity debt as its reason" + ); + drive_until_idle(followup_at, 1000); + + let (_, fulls_remaining, disarmed, _) = + super::super::arena_right_size::test_support::state_snapshot(); + assert_eq!(fulls_remaining, 0); + assert!( + disarmed, + "unchanged low utilization ends the bounded episode" + ); + + set_test_now_ms(Some(followup_at + IDLE_RECLAIM_MIN_INTERVAL_MS + 1)); + assert!(parks(idle_reclaim_park_hook(1000))); + assert_eq!( + thread_attempts(), + 2, + "no third full without utilization or material-capacity hysteresis" + ); +} + #[test] fn idle_reclaim_rate_floor_holds_between_two_owed_fulls() { let _guard = CopyingNurseryTestGuard::new(1); diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 82a2f95b78..2a4551b1ad 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -1,4 +1,5 @@ mod alloc; +mod arena_right_size; mod array_pointer_slot_enumeration; mod barrier; mod barrier_arming; diff --git a/docs/src/internals/garbage-collector.md b/docs/src/internals/garbage-collector.md index f0c87744be..a8a2865dcd 100644 --- a/docs/src/internals/garbage-collector.md +++ b/docs/src/internals/garbage-collector.md @@ -208,7 +208,34 @@ of what it started with (or by less than 4 MiB (two to that power); a productive full resets the requirement to one. The same slicing finishes a budgeted cycle the pacer left open when the mutator went -quiet. `PERRY_GC_DIAG=1` reports the reducer's counters on the +quiet. + +Arena capacity has a second re-arm signal because returning a proven-empty +block deliberately takes two full collection observations, while the activity +clock above excludes the reducer's own fulls. Above a 32 MiB capacity floor, + +two consecutive post-collection live/capacity readings + +at or below 50 percent + +open a bounded right-size episode. It grants only enough idle fulls to reach +two full observations + +(counting any full already in the low-utilization streak), and stops early once +live data reaches 60 percent of capacity. + +After a bounded episode, stable low occupancy cannot immediately repeat it: a +new episode re-arms only after utilization reaches 70 percent + +or capacity grows by at least both 25 percent and 8 MiB. + + +That hysteresis preserves burst headroom and prevents an unreleasably +fragmented heap from buying a full every ten seconds. The right-size debt still +passes through the reducer's quiet, rate, wake, and work-budget gates. + + +`PERRY_GC_DIAG=1` reports the reducer's counters on the `[gc-idle-reclaim]` exit line, and `PERRY_GC_IDLE_RECLAIM=0` disables it. From e1198d2145d18b18738c27f83872aef96d5febd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 13:27:00 +0000 Subject: [PATCH 2/4] docs(changelog): record arena right-sizing --- changelog.d/9731-arena-right-sizing.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 changelog.d/9731-arena-right-sizing.md diff --git a/changelog.d/9731-arena-right-sizing.md b/changelog.d/9731-arena-right-sizing.md new file mode 100644 index 0000000000..78109a41f7 --- /dev/null +++ b/changelog.d/9731-arena-right-sizing.md @@ -0,0 +1,20 @@ +**Idle heaps now return arena capacity left behind by a burst.** General-arena +blocks deliberately need two full-GC observations before their mappings can be +released, but the idle reducer excluded its own collections from its activity +clock. A quiet heap therefore got one full and could stop forever with every +empty block only halfway through the page-return protocol (#9709). + +Two consecutive post-collection samples at or below 50% utilization, above a +32 MiB capacity floor, now open a bounded arena right-size episode. The existing +idle-reclaim and page-return paths supply only the full observations still +needed, stop early once live data reaches 60% of capacity, and remain subject to +the reducer's quiet-time, rate, wake, and work-budget gates. A completed episode +stays disarmed until utilization reaches 70% or capacity grows materially +(at least 25% and 8 MiB), so a retained low live set cannot turn the idle timer +into a periodic full-GC loop. + +On the compiled Claude Code 2.1.112 workload from the report, arena capacity +fell from 96.5 MiB before the episode to 36.7 MiB, then 35.7 MiB and 35.7 MiB +across a five-minute idle soak with about 23 MiB live. RSS fell from 401 MiB at +the first census to 132 MiB at the last, and exactly one idle full was attributed +to arena right-sizing. From 6d0a048ee57292db71cc0f1f4f3b6ba0ded01c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 13:34:09 +0000 Subject: [PATCH 3/4] fix(runtime): use hot TLS for arena right-sizing --- crates/perry-runtime/src/gc/arena_right_size.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/arena_right_size.rs b/crates/perry-runtime/src/gc/arena_right_size.rs index adcacd15c5..6b892dac61 100644 --- a/crates/perry-runtime/src/gc/arena_right_size.rs +++ b/crates/perry-runtime/src/gc/arena_right_size.rs @@ -82,7 +82,7 @@ struct ArenaRightSizeState { last_usage: ArenaUsage, } -thread_local! { +crate::perry_thread_local! { static STATE: RefCell = RefCell::new(ArenaRightSizeState::default()); #[cfg(test)] From bacadd474ed9e0f207faca8f553146fe42b4f395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 13:42:21 +0000 Subject: [PATCH 4/4] docs(changelog): correct workload units --- changelog.d/9731-arena-right-sizing.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/9731-arena-right-sizing.md b/changelog.d/9731-arena-right-sizing.md index 78109a41f7..d902f8937e 100644 --- a/changelog.d/9731-arena-right-sizing.md +++ b/changelog.d/9731-arena-right-sizing.md @@ -14,7 +14,7 @@ stays disarmed until utilization reaches 70% or capacity grows materially into a periodic full-GC loop. On the compiled Claude Code 2.1.112 workload from the report, arena capacity -fell from 96.5 MiB before the episode to 36.7 MiB, then 35.7 MiB and 35.7 MiB -across a five-minute idle soak with about 23 MiB live. RSS fell from 401 MiB at -the first census to 132 MiB at the last, and exactly one idle full was attributed +fell from 96.5 MB before the episode to 36.7 MB, then 35.7 MB and 35.7 MB +across a five-minute idle soak with about 23 MB live. RSS fell from 401 MB at +the first census to 132 MB at the last, and exactly one idle full was attributed to arena right-sizing.