From b603c49d96d0b0664c3cfac7e7a5b6097e99c670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 2 Sep 2026 12:24:21 +0200 Subject: [PATCH] perf(strings): per-site concat cache for "literal" + proven-small value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bench_object_property was the last suite row behind node after the concat memo (#9373) and its governor (#9397): 14 vs 12 ms. The memo made the `obj["field_" + j]` key allocation-free, but a memo hit is still a call into an ~850-instruction function (tag test, fract, range test, itoa, ASCII scan, hash, byte compare, governor bookkeeping); seven single-cause experiments on that function were all flat, and the standing verdict was that a revisit means a per-site redesign. This is it. Every `+` site whose left operand is a string literal and whose right operand is PROVEN to stay inside 0..=255 gets a private `[32 x i64] zeroinitializer` table. Slot k is either 0 or the NaN-boxed string `prefix + String(k)` — by construction, since the prefix cannot vary at the site and only the runtime fill arm writes the table — so a filled slot needs no verification. The emitted hot path is two ordered fcmps (every NaN-box fails them), fptosi/sitofp integrality (folds away for an i32 counter), one load and a non-zero test. The fill arm (`js_string_concat_site_value`) answers exactly what `js_string_concat_value_box` does, fills the slot and registers it through `js_gc_register_global_root`, the funnel string literals already use, so evacuation rewrites it; an SSO result is cached by value with no root. A value outside the table takes the original fused call directly. Admission is measured, not assumed: a gate on a value that sweeps past the table costs 1-2 ns per call (bench_gc_pressure's `"item_" + i` to 500k lost ~1 ms at min with an unconditional lane), a hit saves ~19 ns (210k calls, 4 ms). So the lane needs a proven bound — a loop counter's induction interval, an integer constant (literal, module constant, never-written `const`; `-`/`+`/`*` of those), or `x % C` with small C — exposed from the `loop_bounded_i32` collector as `LoopInductionFacts`. gc_pressure now emits no table and is byte-identical to the lane-off build. Mac mini, interleaved with node in one window, min/median ms: bench_object_property node 12/13 lane off 14/15 lane on 10/11 bench_gc_pressure node 12/13 lane off 12/13 lane on 12/13 bench_string_heavy / 08_string_concat unchanged (41/41, 5/5). Proof: runtime lifecycle tests (fill-once + one root, non-slot values leave the table alone, -0 is slot 0, SSO cached by value with no root, a copied minor rewrites a filled slot — sabotage-run: removing the registration fails the named assertion); perry integration tests (lane fires, node-exact plain and under forced verified evacuation across every slot-rule edge, admission pin with exact table count, kill switch PERRY_CONCAT_SITE_CACHE=0 restores the plain helper); gc store-site gate passes; string/concat suites green. --- .../perry-codegen/src/collectors/hir_facts.rs | 14 + .../src/collectors/loop_bounded_i32.rs | 65 ++++- crates/perry-codegen/src/concat_site_cache.rs | 269 ++++++++++++++++++ crates/perry-codegen/src/lib.rs | 1 + .../perry-codegen/src/lower_string_concat.rs | 7 + .../src/runtime_decls/strings.rs | 1 + .../perry-runtime/src/gc/tests/concat_site.rs | 155 ++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + .../perry-runtime/src/string/concat_site.rs | 105 +++++++ crates/perry-runtime/src/string/mod.rs | 2 + crates/perry/tests/concat_site_cache.rs | 257 +++++++++++++++++ 11 files changed, 864 insertions(+), 13 deletions(-) create mode 100644 crates/perry-codegen/src/concat_site_cache.rs create mode 100644 crates/perry-runtime/src/gc/tests/concat_site.rs create mode 100644 crates/perry-runtime/src/string/concat_site.rs create mode 100644 crates/perry/tests/concat_site_cache.rs diff --git a/crates/perry-codegen/src/collectors/hir_facts.rs b/crates/perry-codegen/src/collectors/hir_facts.rs index 6b2e59cc7e..0257f43348 100644 --- a/crates/perry-codegen/src/collectors/hir_facts.rs +++ b/crates/perry-codegen/src/collectors/hir_facts.rs @@ -70,6 +70,13 @@ pub(crate) struct RepresentationFacts { /// proof as `loop_bounded_i32_locals`, weaker conclusion — it changes no /// storage decision, only an FMF flag. See `collectors/loop_bounded_i32.rs`. pub reassociable_f64_accumulators: HashSet, + /// The intervals and integer constants behind `loop_bounded_i32_locals`, + /// for a consumer that needs the numbers rather than the verdict: + /// `concat_site_cache.rs` gives a `"literal" + value` site a per-site + /// table only when the value is proven small. Runs independently of the + /// canonical-i32 gate for the same reason as + /// `reassociable_f64_accumulators`: it is not a storage decision. + pub loop_induction: super::loop_bounded_i32::LoopInductionFacts, /// Locals whose canonical-i32 promotion is PROVABLE but not PROFITABLE /// (#7128): written after declaration, no i32-consuming read anywhere in /// the body, and at least one double-consuming read inside a loop — so the @@ -234,6 +241,10 @@ impl TypeFacts { &self.representation.reassociable_f64_accumulators } + pub(crate) fn loop_induction(&self) -> &super::loop_bounded_i32::LoopInductionFacts { + &self.representation.loop_induction + } + pub(crate) fn unprofitable_canonical_i32_locals(&self) -> &HashSet { &self.representation.unprofitable_canonical_i32_locals } @@ -559,6 +570,8 @@ pub(crate) fn collect_type_facts( stmts, compile_time_constants, ); + let loop_induction = + super::loop_bounded_i32::collect_loop_induction_facts(stmts, compile_time_constants); // #7123: this set now includes accumulators whose integer-ness and full // range were proved together (for example `sum += i % 1000`). The older // integer provenance collector deliberately does not accept bare `%`, so @@ -735,6 +748,7 @@ pub(crate) fn collect_type_facts( int_valued_ta_locals, loop_bounded_i32_locals, reassociable_f64_accumulators, + loop_induction, unprofitable_canonical_i32_locals, number_by_construction_locals, }, diff --git a/crates/perry-codegen/src/collectors/loop_bounded_i32.rs b/crates/perry-codegen/src/collectors/loop_bounded_i32.rs index e2f67b8e46..52d185a46d 100644 --- a/crates/perry-codegen/src/collectors/loop_bounded_i32.rs +++ b/crates/perry-codegen/src/collectors/loop_bounded_i32.rs @@ -105,10 +105,11 @@ struct GuardedLevel { extreme: i64, } +/// A closed integer interval a local is proven never to leave. #[derive(Clone, Copy, Debug)] -struct IntInterval { - lo: i64, - hi: i64, +pub(crate) struct IntInterval { + pub(crate) lo: i64, + pub(crate) hi: i64, } /// Analysis state, accumulated over one whole function body. @@ -216,6 +217,48 @@ pub fn collect_reassociable_f64_accumulators( stmts: &[Stmt], compile_time_constants: &HashMap, ) -> HashSet { + let st = analysed_state(stmts, compile_time_constants); + let induction_intervals = induction_intervals(&st); + collect_bounded_accumulator_locals( + stmts, + &st, + &induction_intervals, + AccumulatorMode::ReassocF64, + ) +} + +/// What the induction proof knows about integer values, for a consumer that +/// needs the numbers rather than the i32 verdict: `concat_site_cache.rs` +/// gives a `"literal" + value` site a per-site table only when the value is +/// proven small, because the table's inline gate is pure cost on a value +/// that sweeps past it. +#[derive(Clone, Debug, Default)] +pub(crate) struct LoopInductionFacts { + /// Every admissible counter's closed interval, both endpoints in i32. + pub(crate) intervals: HashMap, + /// Locals that are integer constants: module-level compile-time + /// constants plus this body's never-written `const`/`let` bindings with + /// an integer-literal initialiser — the same set the loop guard `v < B` + /// accepts as `B`. + pub(crate) integer_constants: HashMap, +} + +pub fn collect_loop_induction_facts( + stmts: &[Stmt], + compile_time_constants: &HashMap, +) -> LoopInductionFacts { + let st = analysed_state(stmts, compile_time_constants); + let intervals = induction_intervals(&st); + let mut integer_constants = st.module_consts.clone(); + integer_constants.extend(st.const_ints.iter().map(|(&id, &v)| (id, v))); + LoopInductionFacts { + intervals, + integer_constants, + } +} + +/// Run the whole-function walk once; both interval consumers start here. +fn analysed_state(stmts: &[Stmt], compile_time_constants: &HashMap) -> State { let mut st = State::default(); st.module_consts = compile_time_constants .iter() @@ -228,9 +271,12 @@ pub fn collect_reassociable_f64_accumulators( collect_const_ints(stmts, &mut st); let empty: HashMap = HashMap::new(); walk_stmts(stmts, &empty, &mut st); + st +} - let induction_intervals: HashMap = st - .bounds +/// Every admissible local's closed interval, both endpoints inside i32. +fn induction_intervals(st: &State) -> HashMap { + st.bounds .iter() .filter_map(|(&id, bound)| { if st.disqualified.contains(&id) || st.bad_decl.contains(&id) { @@ -249,14 +295,7 @@ pub fn collect_reassociable_f64_accumulators( }; (fits_i32(interval.lo) && fits_i32(interval.hi)).then_some((id, interval)) }) - .collect(); - - collect_bounded_accumulator_locals( - stmts, - &st, - &induction_intervals, - AccumulatorMode::ReassocF64, - ) + .collect() } fn fits_i32(n: i64) -> bool { diff --git a/crates/perry-codegen/src/concat_site_cache.rs b/crates/perry-codegen/src/concat_site_cache.rs new file mode 100644 index 0000000000..7785d989c1 --- /dev/null +++ b/crates/perry-codegen/src/concat_site_cache.rs @@ -0,0 +1,269 @@ +//! Per-site inline cache for `"literal" + value` — the `obj["field_" + j]` +//! key shape of `bench_object_property`, where node was still 2 ms ahead +//! (14 vs 12) with the process-wide concat memo in place. +//! +//! The memo (`perry-runtime/src/string/concat.rs`, 512 slots keyed by content +//! hash) already made the key concat allocation-free, but its hit is still a +//! call into an ~850-instruction function: tag test, `fract`, range test, +//! itoa into a stack buffer, ASCII scan, hash, byte compare, governor +//! bookkeeping. This lane keeps the hot key off the runtime entirely. +//! +//! Every site whose left operand is a source string literal gets a private +//! `[CONCAT_SITE_SLOTS x i64] zeroinitializer` table. Slot `k` is either 0 or +//! the NaN-boxed heap string `prefix + String(k)` — by construction, because +//! the prefix cannot vary at the site and only the runtime miss arm writes +//! the table — so a filled slot needs no verification: +//! +//! * **gate**: `0.0 <= r < SLOTS` as ordered `fcmp`s, which reject every +//! NaN and therefore every NaN-boxed non-number, and dominate the +//! `fptosi` (poison out of range); +//! * **probe**: `k = fptosi r`, `sitofp k == r` (integral), load slot `k`, +//! non-zero → the cached handle. The load is in-bounds for any gated `r`, +//! so integrality and emptiness fold into one `and`; +//! * **fill arm** (gated value, empty or non-integral slot): +//! `js_string_concat_site_value(table, prefix, r)`, which answers exactly +//! what `js_string_concat_value_box` would, fills the slot when the result +//! is a heap string, and registers the slot as a global root through the +//! same funnel string literals use; +//! * **plain arm** (value outside the table, e.g. `"item_" + i` past 31): +//! the fused `js_string_concat_value_box` call this lane replaced, so a +//! site whose values mostly miss pays two `fcmp`s and a branch over the +//! old cost — not an extra call level. bench_gc_pressure's 500k-iteration +//! key site measured that level at ~1 ms before the split. +//! +//! For a loop counter already proven i32 the round trip +//! `sitofp(fptosi(sitofp k))` folds away, leaving the gate and one load. +//! +//! ## Admission +//! +//! The gate is pure cost on a site whose values sweep past the table: +//! bench_gc_pressure's `"item_" + i` runs to 500k, and its two compares and +//! branch per call measured ~0.5-1 ms over 501k calls (1-2 ns each), against +//! a hit that saves ~19 ns (bench_object_property: 210k calls, 4 ms). So a +//! site gets a table only when the right operand is PROVEN small — a loop +//! counter with a proven induction interval (`loop_bounded_i32`), a +//! compile-time integer (literal, module constant, `-`/`+`/`*` of those), or +//! `x % C` with a small `C` — inside `0..=CONCAT_SITE_ADMIT_MAX`. A sweep to +//! 255 still hits one call in eight (2.4 ns saved against 1.75 ns of gate); +//! an unproven operand keeps the plain fused call and the process-wide memo. +//! +//! The table is emitted through `typed_parse_rodata`, the per-function +//! deferred raw-global sink every lowering context already drains. +//! `PERRY_CONCAT_SITE_CACHE=0` removes the lane at build time. + +use anyhow::Result; +use perry_hir::{BinaryOp, Expr, UnaryOp}; + +use crate::expr::FnCtx; +use crate::lower_string_concat::str_operand_handle_tag_dispatched; +use crate::nanbox::double_literal; +use crate::types::{DOUBLE, I1, I32, I64}; + +/// Must match `perry-runtime/src/string/concat_site.rs::CONCAT_SITE_SLOTS`. +pub(crate) const CONCAT_SITE_SLOTS: usize = 32; +/// The table's LLVM type, spelled out because the block builder keeps the +/// `gep` type string by reference. +const CONCAT_SITE_TABLE_TY: &str = "[32 x i64]"; +const _: () = assert!( + CONCAT_SITE_SLOTS == 32, + "CONCAT_SITE_TABLE_TY must spell CONCAT_SITE_SLOTS" +); + +/// Largest right-operand value a site may be proven to reach and still get a +/// table (see the admission paragraph in the module docs). +const CONCAT_SITE_ADMIT_MAX: i64 = 255; + +/// `PERRY_CONCAT_SITE_CACHE=0` kill switch (default on). +fn concat_site_cache_enabled() -> bool { + match std::env::var("PERRY_CONCAT_SITE_CACHE") { + Ok(v) => !matches!(v.as_str(), "0" | "off" | "false" | "OFF" | "FALSE"), + Err(_) => true, + } +} + +/// An integer `f64` inside i32 range, as an `i64`. +fn int_of(f: f64) -> Option { + (f.is_finite() && f.fract() == 0.0 && f.abs() <= i32::MAX as f64).then_some(f as i64) +} + +/// Compile-time integer value of `e`: an integer literal (HIR spells `19` +/// as `Expr::Integer`, `1e1` as `Expr::Number`), an integer-constant local +/// (module constant or a never-written `const` with a literal initialiser — +/// the loop proof's own set), unary minus, or `+`/`-`/`*` of those. `-0` +/// folds to 0, which is the slot JS prints it as. +fn const_int(ctx: &FnCtx<'_>, e: &Expr) -> Option { + match e { + Expr::Integer(n) => (n.unsigned_abs() <= i32::MAX as u64).then_some(*n), + Expr::Number(f) => int_of(*f), + Expr::LocalGet(id) => ctx + .native_facts + .loop_induction() + .integer_constants + .get(id) + .copied(), + Expr::Unary { + op: UnaryOp::Neg, + operand, + } => const_int(ctx, operand).and_then(|v| v.checked_neg()), + Expr::Binary { op, left, right } => { + let a = const_int(ctx, left)?; + let b = const_int(ctx, right)?; + match op { + BinaryOp::Add => a.checked_add(b), + BinaryOp::Sub => a.checked_sub(b), + BinaryOp::Mul => a.checked_mul(b), + _ => None, + } + } + _ => None, + } +} + +/// Whether the right operand is proven to stay inside `0..=ADMIT_MAX` (or, +/// for `x % C`, inside `(-C, C)` with `C - 1 <= ADMIT_MAX`; negative +/// remainders take the plain arm at runtime). +fn right_operand_proven_small(ctx: &FnCtx<'_>, right: &Expr) -> bool { + let small = |v: i64| (0..=CONCAT_SITE_ADMIT_MAX).contains(&v); + if let Some(v) = const_int(ctx, right) { + return small(v); + } + match right { + Expr::LocalGet(id) => ctx + .native_facts + .loop_induction() + .intervals + .get(id) + .is_some_and(|iv| iv.lo >= 0 && small(iv.hi)), + Expr::Binary { + op: BinaryOp::Mod, + right: modulus, + .. + } => const_int(ctx, modulus).is_some_and(|c| c > 0 && small(c - 1)), + _ => false, + } +} + +fn concat_site_global_name(ctx: &FnCtx<'_>, site_id: u32) -> String { + let prefix = ctx.strings.module_prefix(); + if prefix.is_empty() { + format!("perry_concat_site_{site_id}") + } else { + format!("perry_concat_site_{prefix}__{site_id}") + } +} + +/// If `left` is a string literal and `right` is proven small, emit the +/// per-site cached concat of `left + right` (operands already lowered to +/// `l_box` / `r_box`) and return the NaN-boxed string result; otherwise +/// `Ok(None)` so the caller keeps the plain fused helper call. Both cold arms +/// recompute the literal's handle (an inline `bitcast; and` for a literal) so +/// the hot path carries nothing but the gate and the probe. +pub(crate) fn try_lower_concat_site_cached( + ctx: &mut FnCtx<'_>, + left: &Expr, + right: &Expr, + l_box: &str, + r_box: &str, +) -> Result> { + if !concat_site_cache_enabled() || !matches!(left, Expr::String(_)) { + return Ok(None); + } + if !right_operand_proven_small(ctx, right) { + return Ok(None); + } + + let site_id = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let table_name = concat_site_global_name(ctx, site_id); + ctx.typed_parse_rodata.push(format!( + "@{table_name} = private global {CONCAT_SITE_TABLE_TY} zeroinitializer" + )); + let table_ref = format!("@{table_name}"); + + let chk_idx = ctx.new_block("csite.chk"); + let hit_idx = ctx.new_block("csite.hit"); + let fill_idx = ctx.new_block("csite.fill"); + let plain_idx = ctx.new_block("csite.plain"); + let merge_idx = ctx.new_block("csite.merge"); + let chk_label = ctx.block_label(chk_idx); + let hit_label = ctx.block_label(hit_idx); + let fill_label = ctx.block_label(fill_idx); + let plain_label = ctx.block_label(plain_idx); + let merge_label = ctx.block_label(merge_idx); + + // ---- gate: 0.0 <= r < SLOTS (ordered, so every NaN-box fails) ---- + { + let blk = ctx.block(); + let lo = blk.fcmp("oge", r_box, &double_literal(0.0)); + let hi = blk.fcmp("olt", r_box, &double_literal(CONCAT_SITE_SLOTS as f64)); + let in_range = blk.and(I1, &lo, &hi); + blk.cond_br(&in_range, &chk_label, &plain_label); + } + + // ---- probe: integral value and a filled slot ---- + ctx.current_block = chk_idx; + let cached = { + let blk = ctx.block(); + let k = blk.fptosi(DOUBLE, r_box, I32); + let back = blk.sitofp(I32, &k, DOUBLE); + let is_int = blk.fcmp("oeq", &back, r_box); + let k64 = blk.sext(I32, &k, I64); + let cell = blk.gep(CONCAT_SITE_TABLE_TY, &table_ref, &[(I64, "0"), (I64, &k64)]); + let cached = blk.load(I64, &cell); + let filled = blk.icmp_ne(I64, &cached, "0"); + let hit = blk.and(I1, &is_int, &filled); + blk.cond_br(&hit, &hit_label, &fill_label); + cached + }; + + // ---- hit: the slot IS the NaN-boxed result ---- + ctx.current_block = hit_idx; + let (hit_val, hit_end) = { + let blk = ctx.block(); + let val = blk.bitcast_i64_to_double(&cached); + let end = blk.label.clone(); + blk.br(&merge_label); + (val, end) + }; + + // ---- fill: gated value, slot empty (or value non-integral) ---- + ctx.current_block = fill_idx; + let fill_handle = str_operand_handle_tag_dispatched(ctx, left, l_box); + let (fill_val, fill_end) = { + let blk = ctx.block(); + let table_i64 = blk.ptrtoint(&table_ref, I64); + let val = blk.call( + DOUBLE, + "js_string_concat_site_value", + &[(I64, &table_i64), (I64, &fill_handle), (DOUBLE, r_box)], + ); + let end = blk.label.clone(); + blk.br(&merge_label); + (val, end) + }; + + // ---- plain: value outside the table — the call this lane replaced ---- + ctx.current_block = plain_idx; + let plain_handle = str_operand_handle_tag_dispatched(ctx, left, l_box); + let (plain_val, plain_end) = { + let blk = ctx.block(); + let val = blk.call( + DOUBLE, + "js_string_concat_value_box", + &[(I64, &plain_handle), (DOUBLE, r_box)], + ); + let end = blk.label.clone(); + blk.br(&merge_label); + (val, end) + }; + + ctx.current_block = merge_idx; + Ok(Some(ctx.block().phi( + DOUBLE, + &[ + (hit_val.as_str(), hit_end.as_str()), + (fill_val.as_str(), fill_end.as_str()), + (plain_val.as_str(), plain_end.as_str()), + ], + ))) +} diff --git a/crates/perry-codegen/src/lib.rs b/crates/perry-codegen/src/lib.rs index 46b144be39..1dfa523537 100644 --- a/crates/perry-codegen/src/lib.rs +++ b/crates/perry-codegen/src/lib.rs @@ -25,6 +25,7 @@ pub(crate) mod lower_array_method; pub(crate) mod lower_call; pub(crate) mod lower_conditional; pub(crate) mod lower_string_concat; +pub(crate) mod concat_site_cache; pub(crate) mod lower_string_method; pub mod module; pub mod nanbox; diff --git a/crates/perry-codegen/src/lower_string_concat.rs b/crates/perry-codegen/src/lower_string_concat.rs index 0594c822cb..e1d6cca0e0 100644 --- a/crates/perry-codegen/src/lower_string_concat.rs +++ b/crates/perry-codegen/src/lower_string_concat.rs @@ -567,6 +567,13 @@ fn coerce_concat_body( &[(DOUBLE, l_box), (DOUBLE, r_box)], )); } + // Literal prefix + proven-small value: the per-site table keeps the + // hot key off the runtime entirely (`concat_site_cache.rs`). + if let Some(value) = + crate::concat_site_cache::try_lower_concat_site_cached(ctx, left, right, l_box, r_box)? + { + return Ok(value); + } // Issue #214: SSO-safe unbox; repsel Phase 3a: inline `bitcast+and` // for proven-heap operands (string literals — the `"user_" + i` // shape) and tag-dispatch for canonical-Str locals. diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index abb3879d17..48d75e0462 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -31,6 +31,7 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { // `js_string_concat_value(prefix_handle, value_f64) -> handle` // `js_value_concat_string(value_f64, suffix_handle) -> handle` module.declare_function("js_string_concat_value", I64, &[I64, DOUBLE]); + module.declare_function("js_string_concat_site_value", DOUBLE, &[I64, I64, DOUBLE]); module.declare_function("js_value_concat_string", I64, &[DOUBLE, I64]); // NaN-box-returning twin: SSO immediate for ≤5-ASCII-byte results, so // `"k" + i` computed keys get content-stable bits (dyn-IC/stub hits) diff --git a/crates/perry-runtime/src/gc/tests/concat_site.rs b/crates/perry-runtime/src/gc/tests/concat_site.rs new file mode 100644 index 0000000000..466700213a --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/concat_site.rs @@ -0,0 +1,155 @@ +//! Lifecycle proof for the per-site concat cache (`string/concat_site.rs`). +//! +//! The table's contract ("a non-zero slot `k` is the live heap string +//! `prefix + String(k)`") rests on one call: the filling arm registering the +//! slot as a global root. Drop that call and +//! `test_filled_slot_is_rewritten_by_a_copied_minor` fails — the string moves +//! and the slot keeps its pre-move address. + +use super::super::*; +use super::support::*; +use crate::string::{js_string_concat_site_value, js_string_from_bytes, CONCAT_SITE_SLOTS}; +use crate::value::{POINTER_MASK, STRING_TAG}; + +fn fresh_table() -> &'static mut [u64; CONCAT_SITE_SLOTS] { + // Leaked on purpose: a filled slot's address becomes a GC root, and the + // guard's `reset_global_roots` runs at drop, after the test body. + Box::leak(Box::new([0u64; CONCAT_SITE_SLOTS])) +} + +fn is_heap_string(bits: u64) -> bool { + bits & !POINTER_MASK == STRING_TAG +} + +fn heap_string_bytes(bits: u64) -> Vec { + assert!(is_heap_string(bits), "expected a heap string handle"); + let ptr = (bits & POINTER_MASK) as *const crate::string::StringHeader; + unsafe { + let len = (*ptr).byte_len as usize; + std::slice::from_raw_parts(crate::string::string_data(ptr), len).to_vec() + } +} + +fn global_root_count() -> usize { + GLOBAL_ROOTS.with(|roots| roots.borrow().len()) +} + +/// A cacheable integer fills its slot once with the returned handle and every +/// later call answers that identical handle; every value that selects no slot +/// (fractional, negative, past the table, NaN) is answered correctly and +/// leaves the table alone; `-0` is slot 0; an SSO result is cached by value +/// and, holding no pointer, is not registered as a root. +#[test] +fn test_site_slot_is_filled_once_and_answers_identically() { + let _guard = CopyingNurseryTestGuard::new(0); + let table = fresh_table(); + let prefix = js_string_from_bytes(b"field_".as_ptr(), 6); + + let roots_before = global_root_count(); + let first = js_string_concat_site_value(table.as_mut_ptr(), prefix, 7.0); + assert_eq!(heap_string_bytes(first.to_bits()), b"field_7"); + assert_eq!( + global_root_count(), + roots_before + 1, + "a heap handle's slot is registered as exactly one global root" + ); + assert_eq!( + table[7], + first.to_bits(), + "a cacheable heap result fills its slot" + ); + + let again = js_string_concat_site_value(table.as_mut_ptr(), prefix, 7.0); + assert_eq!( + again.to_bits(), + first.to_bits(), + "a filled slot answers the identical handle" + ); + + for (value, expect) in [ + (7.5, &b"field_7.5"[..]), + (-1.0, &b"field_-1"[..]), + (CONCAT_SITE_SLOTS as f64, &b"field_32"[..]), + (f64::NAN, &b"field_NaN"[..]), + ] { + let out = js_string_concat_site_value(table.as_mut_ptr(), prefix, value); + assert_eq!(heap_string_bytes(out.to_bits()), expect); + assert_eq!( + table.iter().filter(|&&s| s != 0).count(), + 1, + "a value that selects no slot must not touch the table" + ); + assert_eq!(table[7], first.to_bits()); + } + + let zero = js_string_concat_site_value(table.as_mut_ptr(), prefix, -0.0); + assert_eq!(heap_string_bytes(zero.to_bits()), b"field_0"); + assert_eq!( + table[0], + zero.to_bits(), + "-0 selects slot 0, as JS prints it" + ); + + let short_prefix = js_string_from_bytes(b"k".as_ptr(), 1); + let sso_table = fresh_table(); + let roots_before = global_root_count(); + let sso = js_string_concat_site_value(sso_table.as_mut_ptr(), short_prefix, 4.0); + assert!( + !is_heap_string(sso.to_bits()), + "test premise: \"k4\" is an SSO immediate" + ); + assert_eq!( + sso_table[4], + sso.to_bits(), + "an SSO immediate is cached by value" + ); + assert_eq!( + global_root_count(), + roots_before, + "an SSO slot holds no pointer and must not be registered as a root" + ); + let sso_again = js_string_concat_site_value(sso_table.as_mut_ptr(), short_prefix, 4.0); + assert_eq!(sso_again.to_bits(), sso.to_bits()); +} + +/// A filled slot is a strong root the collector rewrites: after a copied +/// minor moves the cached string, the slot holds the new address and still +/// reads the same bytes. Fails if the filling arm loses its +/// `js_gc_register_global_root` call. +/// +/// A shadow-stack root holds the same string independently, so the string +/// is kept alive and moved whether or not the table registered its slot — +/// the premise cannot pass or fail on the behaviour under test, and a +/// missing registration fails on the assertion that names it (sabotage-run +/// while writing this: without the call the slot keeps the pre-move address +/// while the shadow root shows the new one). +#[test] +fn test_filled_slot_is_rewritten_by_a_copied_minor() { + let _guard = CopyingNurseryTestGuard::new(1); + let table = fresh_table(); + let prefix = js_string_from_bytes(b"field_".as_ptr(), 6); + + let handle = js_string_concat_site_value(table.as_mut_ptr(), prefix, 7.0); + let old_addr = (handle.to_bits() & POINTER_MASK) as usize; + assert_eq!( + table[7], + handle.to_bits(), + "test premise: the slot is filled" + ); + js_shadow_slot_set(0, handle.to_bits()); + + let _ = gc_collect_minor(); + + let moved_bits = js_shadow_slot_get(0); + let new_addr = (moved_bits & POINTER_MASK) as usize; + assert_ne!( + new_addr, old_addr, + "test premise: the cached string must actually move" + ); + assert_eq!( + table[7], moved_bits, + "the slot must follow the moved string — is the filled slot still \ + registered as a global root?" + ); + assert_eq!(heap_string_bytes(table[7]), b"field_7"); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index d2c7e84ff9..078e0dcadc 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -7,6 +7,7 @@ mod block_pool_pressure; mod budgeted_step_api; mod buffer_bound_method_name; mod buffer_side_tables; +mod concat_site; mod contract; mod copying; mod copying_side_tables; diff --git a/crates/perry-runtime/src/string/concat_site.rs b/crates/perry-runtime/src/string/concat_site.rs new file mode 100644 index 0000000000..239eb923b2 --- /dev/null +++ b/crates/perry-runtime/src/string/concat_site.rs @@ -0,0 +1,105 @@ +//! Per-site result cache for `"literal" + smallInt` — the +//! `obj["field_" + j]` key shape of `bench_object_property`. +//! +//! Codegen (`perry-codegen/src/concat_site_cache.rs`) gives every +//! literal-prefix `+ value` site a private zero-initialised +//! `[CONCAT_SITE_SLOTS x i64]` table and probes it inline. Slot `k` is either +//! 0 or the NaN-boxed heap string `prefix + String(k)` — by construction: the +//! prefix is a source literal, so it never varies at a site, and only this +//! helper writes the table. A hit therefore needs no hashing, no byte compare +//! and no governor. That is the whole gain over the process-wide +//! [`super::concat`] memo (512 slots keyed by content hash, verified by a byte +//! compare, throttled by a windowed governor), whose hit path is still a call +//! into an ~850-instruction function. +//! +//! Slots are write-once. The miss arm fills a slot with exactly the value +//! [`js_string_concat_value_box`] returns and registers the slot's address +//! through [`crate::gc::js_gc_register_global_root`] — the funnel +//! module-global string literals already use — so an evacuating collection +//! rewrites the slot instead of leaving it holding a moved string's old +//! address. An SSO result (≤5 ASCII bytes, e.g. `"k" + 4`) is an immediate +//! with content-stable bits and is cached by value, without a registration: +//! it carries no pointer, and leaving it uncached would send every call of +//! such a site through the fill arm forever. +//! +//! Sharing a result between callers is what the memo does too. Strings are +//! immutable; in-place append (`s += x`) is only taken on values codegen +//! proves uniquely owned, which a handle read back from a table never is. +//! +//! Like module-global roots, a table is process-global while `GLOBAL_ROOTS` +//! is per-thread; compiled module code runs on the thread that registers it. + +use super::concat::js_string_concat_value_box; +use crate::string::StringHeader; +use crate::value::JSValue; + +/// Slots per site. Must match `CONCAT_SITE_SLOTS` in +/// `perry-codegen/src/concat_site_cache.rs`. `"field_" + j` for `j < 20` is +/// the motivating shape; 32 keeps a site's table to 256 bytes of BSS. +pub const CONCAT_SITE_SLOTS: usize = 32; + +/// Which slot a right operand selects: a non-negative integer below the slot +/// count and nothing else. Ordered comparisons reject every NaN, and every +/// NaN-boxed non-number is a NaN. `-0` selects slot 0, which is right: JS +/// prints `-0` as `"0"`. +#[inline] +fn concat_site_slot(value: f64) -> Option { + if !(value >= 0.0 && value < CONCAT_SITE_SLOTS as f64) { + return None; + } + let k = value as usize; + if k as f64 != value { + return None; + } + Some(k) +} + +/// Fill arm of the per-site concat cache. +/// +/// Answers exactly what [`js_string_concat_value_box`]`(prefix, value)` does +/// for every input, so callers may route any value here; codegen only routes +/// values inside the table (an out-of-range value takes the plain fused call +/// directly, paying no extra call level). As a side effect, when `value` +/// selects a slot, the empty slot is filled with the string result — rooted +/// when it is a heap handle, by value when it is an SSO immediate; a slot +/// that is already filled is answered from the table (the emitted probe +/// checked it first, so this is the write-once guarantee, not a second fast +/// path). +/// +/// # Safety +/// `table` must point at `CONCAT_SITE_SLOTS` writable `u64` words that live +/// for the rest of the process — codegen emits a private global for each +/// site, and a filled slot's address is handed to the GC as a root. +#[no_mangle] +pub extern "C" fn js_string_concat_site_value( + table: *mut u64, + prefix: *const StringHeader, + value: f64, +) -> f64 { + let slot = concat_site_slot(value); + if let Some(k) = slot { + let cached = unsafe { *table.add(k) }; + if cached != 0 { + return f64::from_bits(cached); + } + } + let result = js_string_concat_value_box(prefix, value); + if let Some(k) = slot { + let bits = result.to_bits(); + let boxed = JSValue::from_bits(bits); + if boxed.is_any_string() { + unsafe { + let cell = table.add(k); + // GC_STORE_AUDIT(ROOT): a heap handle's cell is registered as + // a global root right below, which also applies the root + // heap-word barrier to the value just stored; an SSO + // immediate holds no pointer and needs neither. + std::ptr::write(cell, bits); + if boxed.is_string() { + crate::gc::js_gc_register_global_root(cell as i64); + } + } + } + } + result +} diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index df6b0d86a5..21e7520372 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -106,6 +106,7 @@ mod base64_codec; mod char_ops; mod compare; pub(crate) mod concat; +pub(crate) mod concat_site; mod format; mod html; mod intern; @@ -163,6 +164,7 @@ pub use concat::{ js_string_concat_chain, js_string_concat_value, js_value_add_string, js_value_concat_string, scan_concat_memo_roots, scan_concat_memo_roots_mut, }; +pub use concat_site::{js_string_concat_site_value, CONCAT_SITE_SLOTS}; pub(crate) use format::fix_exponent_format; pub(crate) use format::js_format_f64; pub use format::{ diff --git a/crates/perry/tests/concat_site_cache.rs b/crates/perry/tests/concat_site_cache.rs new file mode 100644 index 0000000000..4070b54857 --- /dev/null +++ b/crates/perry/tests/concat_site_cache.rs @@ -0,0 +1,257 @@ +//! Per-site concat cache for `"literal" + value` +//! (`perry-codegen/src/concat_site_cache.rs`, +//! `perry-runtime/src/string/concat_site.rs`). +//! +//! Node is the oracle for every value below; a hand-computed expectation has +//! been wrong before while perry was right. The IR pins say the lane FIRES +//! (a program can be correct because the lane never ran), and the kill switch +//! proves the plain fused helper still answers the same program. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +fn perry_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_perry")) +} + +/// The bench_object_property key shape, then every edge of the slot rule: +/// a proven bound past the table (32..39 take the plain arm through the +/// lane), a `+=` on a handle the cache handed out, `-0`, fractional / +/// negative / NaN / huge right operands, dynamic operands of number, string +/// and null type at a literal-prefix site, and the SSO twin. +const SOURCE: &str = r#" +const OBJECTS = 200; +const FIELDS = 20; +let checksum = 0; +for (let i = 0; i < OBJECTS; i++) { + const obj: any = {}; + for (let j = 0; j < FIELDS; j++) { + obj["field_" + j] = i * FIELDS + j; + } + checksum += obj["field_0"] + obj["field_" + (FIELDS - 1)]; +} +console.log("checksum:" + checksum); + +const parts: string[] = []; +for (let j = 0; j < 40; j++) { + parts.push("field_" + j); +} +let s = "field_" + 3; +s += "!"; +parts.push(s); +parts.push("field_" + 3); +parts.push("field_" + (-0)); +parts.push("field_" + 1.5); +parts.push("field_" + (-1)); +parts.push("field_" + NaN); +parts.push("field_" + 1e21); +let dyn: any = 7; +parts.push("field_" + dyn); +dyn = "x"; +parts.push("field_" + dyn); +dyn = null; +parts.push("field_" + dyn); +parts.push("k" + 4); +console.log(parts.join(",")); +"#; + +fn compile(dir: &Path, source: &str, extra_env: &[(&str, &str)]) -> (PathBuf, String) { + let entry = dir.join("main.ts"); + let output = dir.join("main_bin"); + std::fs::write(&entry, source).expect("write entry"); + let mut cmd = Command::new(perry_bin()); + cmd.current_dir(dir) + .arg("compile") + .arg(&entry) + .arg("-o") + .arg(&output) + .env("PERRY_NO_CACHE", "1") + .env("PERRY_LLVM_KEEP_IR", "1"); + for (k, v) in extra_env { + cmd.env(k, v); + } + let compile = cmd.output().expect("run perry compile"); + assert!( + compile.status.success(), + "perry compile failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + ( + output, + String::from_utf8_lossy(&compile.stderr).into_owned(), + ) +} + +/// CALL sites of `name`, not the `declare` line every module emits for every +/// runtime symbol (a bare-substring presence test passes vacuously and a +/// bare-substring absence test can never pass). +fn call_count(ir: &str, name: &str) -> usize { + let needle = format!("@{name}("); + ir.lines() + .filter(|l| l.contains(&needle) && l.contains("call ")) + .count() +} + +fn kept_ir(stderr: &str) -> String { + let path = stderr + .lines() + .find_map(|line| line.split("kept LLVM IR: ").nth(1)) + .map(str::trim) + .map(PathBuf::from) + .unwrap_or_else(|| panic!("PERRY_LLVM_KEEP_IR did not report an IR path\n{stderr}")); + std::fs::read_to_string(path).expect("read kept LLVM IR") +} + +fn node_oracle(dir: &Path) -> String { + let node = Command::new("node") + .current_dir(dir) + .arg("--experimental-strip-types") + .arg(dir.join("main.ts")) + .output() + .expect("run node"); + assert!( + node.status.success(), + "node failed on the oracle fixture\n{}", + String::from_utf8_lossy(&node.stderr) + ); + String::from_utf8_lossy(&node.stdout).into_owned() +} + +fn run(bin: &Path, dir: &Path, gc_stress: bool) -> Output { + let mut command = Command::new(bin); + command.current_dir(dir); + if gc_stress { + command + .env("PERRY_GC_HEAP_LIMIT", "8") + .env("PERRY_GC_FORCE_EVACUATE", "1") + .env("PERRY_GC_VERIFY_EVACUATION", "1"); + } + command.output().expect("run compiled binary") +} + +fn assert_matches_node(bin: &Path, dir: &Path, expected: &str, label: &str) { + for stress in [false, true] { + let out = run(bin, dir, stress); + assert!( + out.status.success(), + "{label}: binary failed (gc_stress={stress})\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&out.stdout), + expected, + "{label}: output differs from node (gc_stress={stress})" + ); + } +} + +/// The lane fires on every literal-prefix site, and the program is +/// node-exact — including under forced, verified evacuation, which is the +/// arm that matters for a cache whose entries are roots that must be +/// rewritten when the cached string moves. +#[test] +fn site_cache_fires_and_matches_node_under_evacuation() { + let dir = tempfile::tempdir().expect("tempdir"); + let (bin, stderr) = compile(dir.path(), SOURCE, &[]); + let ir = kept_ir(&stderr); + assert!( + call_count(&ir, "js_string_concat_site_value") > 0, + "the per-site lane's miss arm must be CALLED — the lane did not fire" + ); + assert!( + ir.contains("@perry_concat_site_"), + "the emitted probe must read a per-site table" + ); + let expected = node_oracle(dir.path()); + assert_matches_node(&bin, dir.path(), &expected, "site-cache"); +} + +/// Admission follows the proven bound: a counter that sweeps to 100k gets no +/// table (its gate would be pure cost, ~1-2 ns per call, and bench_gc_pressure +/// measured exactly that), while a counter bounded by a small module constant +/// does. +#[test] +fn admission_follows_the_proven_bound() { + const LARGE: &str = r#" +let big = 0; +for (let i = 0; i < 100000; i++) { + big += ("big_" + i).length; +} +console.log("big:" + big); +"#; + // Three admitted sites: the bounded counter, a constant expression over a + // module constant, and an integer literal. `"n:" + n` is not one (`n` is + // an accumulator with no proven interval), so the table count is exact. + const SMALL: &str = r#" +const FIELDS = 20; +let n = 0; +for (let j = 0; j < FIELDS; j++) { + n += ("field_" + j).length; + n += ("field_" + (FIELDS - 1)).length; + n += ("field_" + 19).length; +} +console.log("n:" + n); +"#; + let dir = tempfile::tempdir().expect("tempdir"); + let (bin, stderr) = compile(dir.path(), LARGE, &[]); + let ir = kept_ir(&stderr); + assert_eq!( + call_count(&ir, "js_string_concat_site_value"), + 0, + "a counter proven to sweep far past the table must not get one" + ); + assert!( + !ir.contains("@perry_concat_site_"), + "no per-site table may be emitted for the large-bound site" + ); + assert!( + call_count(&ir, "js_string_concat_value_box") > 0, + "vacuity guard: the large-bound site must still reach the fused arm" + ); + let expected = node_oracle(dir.path()); + assert_matches_node(&bin, dir.path(), &expected, "large-bound"); + + let dir = tempfile::tempdir().expect("tempdir"); + let (bin, stderr) = compile(dir.path(), SMALL, &[]); + let ir = kept_ir(&stderr); + let tables = ir + .lines() + .filter(|l| l.contains("@perry_concat_site_") && l.contains("= private global")) + .count(); + assert_eq!( + tables, 3, + "a small-bound counter, a constant expression over a module constant \ + and an integer literal must each get a table (and the unbounded \ + accumulator must not)" + ); + assert!( + call_count(&ir, "js_string_concat_site_value") > 0, + "the admitted sites' fill arm must be CALLED" + ); + let expected = node_oracle(dir.path()); + assert_matches_node(&bin, dir.path(), &expected, "small-bound"); +} + +/// Kill switch: `PERRY_CONCAT_SITE_CACHE=0` at build time removes the lane, +/// the same sites go back to the plain fused helper (vacuity guard: the +/// shape must still reach that arm), and the answers are unchanged. +#[test] +fn kill_switch_restores_the_plain_helper_and_stays_correct() { + let dir = tempfile::tempdir().expect("tempdir"); + let (bin, stderr) = compile(dir.path(), SOURCE, &[("PERRY_CONCAT_SITE_CACHE", "0")]); + let ir = kept_ir(&stderr); + assert_eq!( + call_count(&ir, "js_string_concat_site_value"), + 0, + "kill switch must remove the per-site lane" + ); + assert!( + call_count(&ir, "js_string_concat_value_box") > 0, + "fixture no longer reaches the fused literal-prefix arm — the absence \ + assertion above would pass vacuously" + ); + let expected = node_oracle(dir.path()); + assert_matches_node(&bin, dir.path(), &expected, "kill-switch"); +}