From c890b6a16dc6e737461ebb1c4ece76d81f7048a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 14:41:08 +0200 Subject: [PATCH 1/9] fix(runtime): dynamic Number toString uses NumberToString, not Rust Display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dynamically dispatched `x["toString"]()` on a number reached three arms of the native-method tower that formatted with a bare `f64::to_string()`. That is Rust's Display: it never switches to scientific notation and spells the infinities `inf`, so `2.2e-308` printed ~308 decimal digits and `Infinity` printed `inf` — while the same value's four static renderings were correct in the same program. The three arms are the plain-number and boxed-`Number` `toString` in `dispatch_common` and the boxed-`Number` `toString`/`toLocaleString` in `dispatch_primitive`. All now call `js_number_to_string`, which carries the spec's `|n| >= 1e21 || |n| < 1e-6` switch and its own integer fast path. This is the same mistake #3987 fixed in the string-concat fast paths; these arms were not part of that sweep. A neighbouring defect in the same arms rides along: a boxed receiver dropped an explicit radix, so `new Number(255).toString(16)` answered "255". Both boxed arms now route an explicit radix through `js_jsvalue_to_string_radix`, as the unboxed arm already did. `toLocaleString`'s argument is a locale, not a radix, so it keeps ignoring it. Closes #9713 --- changelog.d/9728-dynamic-number-tostring.md | 52 ++++++++++++++ .../native_call_method/common_methods.rs | 41 +++++++---- .../native_call_method/primitive_methods.rs | 33 +++++++-- .../test_gap_9713_dynamic_number_tostring.ts | 68 +++++++++++++++++++ 4 files changed, 172 insertions(+), 22 deletions(-) create mode 100644 changelog.d/9728-dynamic-number-tostring.md create mode 100644 test-files/test_gap_9713_dynamic_number_tostring.ts diff --git a/changelog.d/9728-dynamic-number-tostring.md b/changelog.d/9728-dynamic-number-tostring.md new file mode 100644 index 0000000000..143bbbebac --- /dev/null +++ b/changelog.d/9728-dynamic-number-tostring.md @@ -0,0 +1,52 @@ +**A dynamically dispatched `x["toString"]()` on a number now produces +`NumberToString`, not Rust's `f64` Display** (#9713). It printed `inf` for +`Infinity` and the full decimal expansion past the exponential thresholds, so +the same value stringified four static ways and once dynamically disagreed +inside one program: + +```ts +const a = 2.2e-308; +a.toString(); // 2.2e-308 (all four static forms) +((x: any, m: string) => x[m]())(a, "toString"); // 0.000…00022 — ~308 digits +``` + +Three arms of the native-method tower — the plain-number and boxed-`Number` +`toString` in `dispatch_common`, and the boxed-`Number` +`toString`/`toLocaleString` in `dispatch_primitive` — formatted with + +```rust +if n.fract() == 0.0 && n.abs() < INT_EXACT_FASTPATH_LIMIT { (n as i64).to_string() } else { n.to_string() } +``` + +`f64::to_string()` is Rust's shortest-round-trip Display, which never switches +to scientific notation and renders the infinities as `inf`. It is the exact +mistake `js_format_f64`'s doc comment already warns about — #3987 replaced the +same `format!("{}", n)` in the string-concat fast paths and these three arms +were not part of that sweep. They now call `js_number_to_string`, which carries +the spec's `|n| >= 1e21 || |n| < 1e-6` switch, the `Infinity` / `NaN` / `-0` +spellings, and its own (safer) integer fast path — `js_format_f64` cuts over to +the shortest-round-trip formatter at 1e15 rather than 2^53, so it also avoids +the `2**58` → `…744` vs `…740` divergence the local fast path could reach. + +Measured against node 26.5.1, previously wrong and now correct: `1e21`, +`1e-7`, `-2.5e-9`, `2.2e-308`, `Number.MAX_VALUE`, `Number.MIN_VALUE`, +`Number.EPSILON`, `±Infinity`, and every one of those again through +`new Number(x).toString()`. + +One neighbouring defect in the same arms rides along: a boxed receiver dropped +an explicit radix entirely, so `new Number(255).toString(16)` answered `"255"` +instead of `"ff"`. Both boxed arms now route an explicit radix through +`js_jsvalue_to_string_radix` the way the unboxed arm already did (which also +means an out-of-range radix throws `RangeError` there, as the spec requires). +`toLocaleString` keeps ignoring its argument — that one is a locale, not a +radix. + +`test-files/test_gap_9713_dynamic_number_tostring.ts` pins 18 values across the +thresholds in all seven renderings plus the radix and `toFixed` / +`toPrecision` / `toExponential` forms. Unpatched it differs from node on 12 +lines; patched it is byte-identical. + +Not fixed here, and filed separately: `toString(radix)` above 2^53 for a +non-power-of-two radix still emits exact digits rather than V8's shortest +round-trip (#9725) — that one reproduces from a plain static call and is a +different formatter. diff --git a/crates/perry-runtime/src/object/native_call_method/common_methods.rs b/crates/perry-runtime/src/object/native_call_method/common_methods.rs index c7782d7d60..12a9e001eb 100644 --- a/crates/perry-runtime/src/object/native_call_method/common_methods.rs +++ b/crates/perry-runtime/src/object/native_call_method/common_methods.rs @@ -802,15 +802,28 @@ pub(super) unsafe fn dispatch_common( } else { payload }; - let s = if n.fract() == 0.0 - && n.abs() < crate::builtins::INT_EXACT_FASTPATH_LIMIT - { - (n as i64).to_string() - } else { - n.to_string() - }; - let str_ptr = - crate::string::js_string_from_bytes(s.as_ptr(), s.len() as u32); + // #9713: `NumberToString`, not Rust's `{}`. A bare + // `f64::to_string()` prints `inf` for Infinity and the + // full decimal expansion past the exponential + // thresholds (`1e21` → `1000000000000000000000`, + // `Number.EPSILON` → `0.000…0002220446049250313`). + // `js_number_to_string` carries the spec's + // `|n| >= 1e21 || |n| < 1e-6` switch and its own + // integer fast path, so the local one is redundant too. + // + // A boxed receiver takes a radix like an unboxed one + // (`new Number(255).toString(16)` is "ff"); this arm + // dropped the argument entirely and answered "255". + // `js_jsvalue_to_string_radix` already accepts a boxed + // Number receiver, so hand it the box, not the payload. + let radix_arg = refreshed_args().first().copied(); + if let Some(r) = radix_arg { + if !JSValue::from_bits(r.to_bits()).is_undefined() { + let str_ptr = crate::value::js_jsvalue_to_string_radix(object, r); + return Some(f64::from_bits(JSValue::string_ptr(str_ptr).bits())); + } + } + let str_ptr = crate::string::js_number_to_string(n); return Some(f64::from_bits(JSValue::string_ptr(str_ptr).bits())); } Some("Boolean") => { @@ -861,12 +874,10 @@ pub(super) unsafe fn dispatch_common( crate::value::js_jsvalue_to_string_radix(object, radix_arg.unwrap()); return Some(f64::from_bits(JSValue::string_ptr(str_ptr).bits())); } - let s = if n.fract() == 0.0 && n.abs() < crate::builtins::INT_EXACT_FASTPATH_LIMIT { - (n as i64).to_string() - } else { - n.to_string() - }; - let str_ptr = crate::string::js_string_from_bytes(s.as_ptr(), s.len() as u32); + // #9713: same as the boxed-Number arm above — `NumberToString`, + // not Rust's `f64` Display. This is the arm a dynamic + // `x["toString"]()` on a plain number reaches. + let str_ptr = crate::string::js_number_to_string(n); return Some(f64::from_bits(JSValue::string_ptr(str_ptr).bits())); } else if jsval.is_bool() { let s = if jsval.as_bool() { "true" } else { "false" }; diff --git a/crates/perry-runtime/src/object/native_call_method/primitive_methods.rs b/crates/perry-runtime/src/object/native_call_method/primitive_methods.rs index 0a082e013d..39f7e6d8bb 100644 --- a/crates/perry-runtime/src/object/native_call_method/primitive_methods.rs +++ b/crates/perry-runtime/src/object/native_call_method/primitive_methods.rs @@ -222,15 +222,34 @@ pub(super) unsafe fn dispatch_primitive( } else { payload }; - let s = if n.fract() == 0.0 - && n.abs() < crate::builtins::INT_EXACT_FASTPATH_LIMIT - { - (n as i64).to_string() + // #9713: `NumberToString`, not Rust's `{}`. A bare + // `f64::to_string()` prints `inf` for Infinity and the + // full decimal expansion past the exponential + // thresholds (`1e21` → `1000000000000000000000`, + // `Number.EPSILON` → `0.000…0002220446049250313`). + // `js_number_to_string` carries the spec's + // `|n| >= 1e21 || |n| < 1e-6` switch and its own + // integer fast path, so the local one is redundant too. + // + // A boxed receiver takes a radix like an unboxed one + // (`new Number(255).toString(16)` is "ff"); this arm + // dropped the argument entirely and answered "255". + // `js_jsvalue_to_string_radix` already accepts a boxed + // Number receiver, so hand it the box, not the payload. + // `toLocaleString`'s argument is a locale, not a + // radix, so only `toString` consumes it here. + let radix_arg = if method_name == "toString" { + refreshed_args().first().copied() } else { - n.to_string() + None }; - let str_ptr = - crate::string::js_string_from_bytes(s.as_ptr(), s.len() as u32); + if let Some(r) = radix_arg { + if !JSValue::from_bits(r.to_bits()).is_undefined() { + let str_ptr = crate::value::js_jsvalue_to_string_radix(object, r); + return Some(f64::from_bits(JSValue::string_ptr(str_ptr).bits())); + } + } + let str_ptr = crate::string::js_number_to_string(n); return Some(f64::from_bits(JSValue::string_ptr(str_ptr).bits())); } Some("Boolean") => { diff --git a/test-files/test_gap_9713_dynamic_number_tostring.ts b/test-files/test_gap_9713_dynamic_number_tostring.ts new file mode 100644 index 0000000000..89bce343d5 --- /dev/null +++ b/test-files/test_gap_9713_dynamic_number_tostring.ts @@ -0,0 +1,68 @@ +// #9713: a dynamically dispatched `x["toString"]()` on a number reached the +// native-method tower's own formatter — a bare Rust `f64::to_string()` — instead +// of ECMA-262 NumberToString. That prints `inf` for Infinity and the full +// decimal expansion past the exponential thresholds, so the same value +// stringified four static ways and once dynamically disagreed inside one +// program. Boxed `new Number(x)` receivers took the same wrong arm. +// +// Every row prints all the renderings so a future divergence shows which path +// moved, not just that something changed. `toLocaleString` is deliberately +// absent: node applies locale grouping there (`1,000,000,000,000,000,000,000`, +// `\u221e`) and perry does not, which is a separate gap this fixture should not +// be entangled with. + +function dynCall(x: any, m: string): any { return x[m](); } +function dynCall1(x: any, m: string, a: any): any { return x[m](a); } + +const values: [string, number][] = [ + ["1e21", 1e21], + ["1e20", 1e20], + ["1e-6", 1e-6], + ["1e-7", 1e-7], + ["-2.5e-9", -2.5e-9], + ["2.2e-308", 2.2e-308], + ["1e-310", 1e-310], + ["MAX_VALUE", Number.MAX_VALUE], + ["MIN_VALUE", Number.MIN_VALUE], + ["EPSILON", Number.EPSILON], + ["Infinity", Infinity], + ["-Infinity", -Infinity], + ["NaN", NaN], + ["-0", -0], + ["0.1", 0.1], + ["255", 255], + ["2**53", 9007199254740992], + ["2**58", 288230376151711744], +]; + +for (const [label, n] of values) { + const parts = [ + "static=" + n.toString(), + "String=" + String(n), + "tpl=" + `${n}`, + "concat=" + (n + ""), + "dyn=" + dynCall(n, "toString"), + "boxed=" + dynCall(new Number(n), "toString"), + "boxedValueOf=" + String(dynCall(new Number(n), "valueOf")), + ]; + console.log(label + " :: " + parts.join(" | ")); +} + +// An explicit radix must still reach the radix formatter, and an explicit +// `undefined` radix must behave like no argument at all. +console.log("radix16=" + dynCall1(255, "toString", 16)); +console.log("radix2=" + dynCall1(5, "toString", 2)); +// Radix values stay at or below 2^53: above it perry's non-power-of-two radix +// formatter emits exact digits where V8 emits the shortest round-trip form +// (`(1e21).toString(36)` → `5v1j4f4ds7c4ks` vs `5v1j4f4ds7c000`), statically as +// well as dynamically. That is a separate defect and not what this pins. +console.log("radix36=" + dynCall1(9007199254740992, "toString", 36)); +console.log("radix7=" + dynCall1(255, "toString", 7)); +console.log("radixUndef=" + dynCall1(1e21, "toString", undefined)); +console.log("boxedRadix16=" + dynCall1(new Number(255), "toString", 16)); + +// Sibling numeric methods on the same dynamic route, so a shared regression in +// the tower's number handling is visible here too. +console.log("toFixed=" + dynCall1(3.14159, "toFixed", 2)); +console.log("toPrecision=" + dynCall1(1234.5678, "toPrecision", 6)); +console.log("toExponential=" + dynCall1(1e21, "toExponential", 3)); From 7d735c4e5ecbcc472f3ae1228045a513f68a0182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 14:44:02 +0200 Subject: [PATCH 2/9] perf(ic): allocate inline caches per used site behind an 8-byte slot (#9708) Every inline-cache site owned a `[12 x i64] zeroinitializer` global, 96 B of __bss per site whether or not the program executed it; on the Claude Code bundle that was 262k caches and 18.7 MB dirty at idle. A site now owns `@perry_ic_N = private global ptr null`; the runtime allocates the cache words from an arena on the site's first priming miss and publishes them into the slot with a CAS. Hit paths load the slot, fold `!= null` into the receiver guard they already evaluate, and read the words through the pointer; every miss entry takes the slot's address. Cache layout and prime/evict policy are unchanged; IC hit counters are identical. Claude-Session: https://claude.ai/code/session_014RVEmbrpNKHwaQdrcMgMHc --- crates/perry-codegen/src/codegen/closure.rs | 6 +- crates/perry-codegen/src/codegen/entry.rs | 12 +- crates/perry-codegen/src/codegen/function.rs | 6 +- crates/perry-codegen/src/codegen/method.rs | 12 +- crates/perry-codegen/src/expr/index_get.rs | 16 +- .../expr/index_get/inline_dyn_typed_array.rs | 16 +- crates/perry-codegen/src/expr/mod.rs | 50 +++- .../src/expr/property_get/composed_ics.rs | 48 ++- .../src/expr/property_get/generic_dispatch.rs | 50 +++- .../src/expr/property_get/tests.rs | 35 ++- .../perry-codegen/src/expr/proxy_reflect.rs | 59 +++- .../property_get/imported_object.rs | 14 +- crates/perry-codegen/src/module.rs | 13 +- .../src/runtime_decls/objects.rs | 3 +- .../src/stmt/cached_field_index_return.rs | 9 +- crates/perry-runtime/src/array/mod.rs | 3 +- crates/perry-runtime/src/array/subclass.rs | 107 ++----- .../src/array/subclass_packed_index.rs | 121 ++++++++ .../perry-runtime/src/array/subclass_tests.rs | 21 +- crates/perry-runtime/src/gc/census.rs | 1 + .../src/gc/tests/handle_bound_method_name.rs | 4 +- .../src/node_submodules/tests.rs | 8 +- .../perry-runtime/src/object/field_get_set.rs | 14 +- .../src/object/field_get_set/ic_miss.rs | 32 +- .../field_get_set/ic_miss/c3c_pic_tests.rs | 29 +- .../ic_miss_array_length_tests.rs | 14 +- .../src/object/field_get_set/ic_slot.rs | 281 ++++++++++++++++++ crates/perry-runtime/src/object/with_env.rs | 2 +- crates/perry-runtime/src/proxy.rs | 13 +- crates/perry-runtime/src/proxy/put_value.rs | 111 +++++-- crates/perry-runtime/src/symbol.rs | 2 +- crates/perry-runtime/src/symbol/get.rs | 80 +++-- crates/perry-runtime/src/typed_feedback.rs | 1 + .../src/typed_feedback/guards.rs | 29 +- .../perry-runtime/src/typed_feedback/tests.rs | 27 +- .../perry-runtime/src/value/dynamic_object.rs | 37 ++- crates/perry-runtime/src/value/mod.rs | 2 +- docs/src/internals/memory-model.md | 11 + .../test_gap_9708_lazy_inline_cache_slots.ts | 192 ++++++++++++ 39 files changed, 1192 insertions(+), 299 deletions(-) create mode 100644 crates/perry-runtime/src/array/subclass_packed_index.rs create mode 100644 crates/perry-runtime/src/object/field_get_set/ic_slot.rs create mode 100644 test-files/test_gap_9708_lazy_inline_cache_slots.ts diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 57f15b9af3..e530881f23 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -1403,11 +1403,7 @@ pub(super) fn compile_closure( llmod.declare_function(&name, ret, ¶ms); } for ic_name in &ic_globals { - llmod.add_raw_global(format!( - "@{} = private global [{} x i64] zeroinitializer", - ic_name, - crate::expr::property_get::generic_dispatch::PIC_CACHE_WORDS - )); + llmod.add_raw_global(crate::expr::inline_cache_global_definition(ic_name)); } for raw in &typed_parse_rodata { llmod.add_raw_global(raw.clone()); diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index b063835dc1..4564eb9046 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -1426,11 +1426,7 @@ pub(super) fn compile_module_entry( llmod.declare_function(&name, ret, ¶ms); } for ic_name in &ic_globals { - llmod.add_raw_global(format!( - "@{} = private global [{} x i64] zeroinitializer", - ic_name, - crate::expr::property_get::generic_dispatch::PIC_CACHE_WORDS - )); + llmod.add_raw_global(crate::expr::inline_cache_global_definition(ic_name)); } for raw in &typed_parse_rodata { llmod.add_raw_global(raw.clone()); @@ -1959,11 +1955,7 @@ pub(super) fn compile_module_entry( // A dylib's top-level plugin exports live in its entry module, and the // three symbols must be defined exactly once per shared library. for ic_name in &ic_globals { - llmod.add_raw_global(format!( - "@{} = private global [{} x i64] zeroinitializer", - ic_name, - crate::expr::property_get::generic_dispatch::PIC_CACHE_WORDS - )); + llmod.add_raw_global(crate::expr::inline_cache_global_definition(ic_name)); } for raw in &typed_parse_rodata { llmod.add_raw_global(raw.clone()); diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 8e58e0805b..21eb811fb3 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1454,11 +1454,7 @@ pub(super) fn compile_function( llmod.declare_function(&name, ret, ¶ms); } for ic_name in &ic_globals { - llmod.add_raw_global(format!( - "@{} = private global [{} x i64] zeroinitializer", - ic_name, - crate::expr::property_get::generic_dispatch::PIC_CACHE_WORDS - )); + llmod.add_raw_global(crate::expr::inline_cache_global_definition(ic_name)); } for raw in &typed_parse_rodata { llmod.add_raw_global(raw.clone()); diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index e4464d2d8e..ff861e76f8 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -1336,11 +1336,7 @@ pub(super) fn compile_method( llmod.declare_function(&name, ret, ¶ms); } for ic_name in &ic_globals { - llmod.add_raw_global(format!( - "@{} = private global [{} x i64] zeroinitializer", - ic_name, - crate::expr::property_get::generic_dispatch::PIC_CACHE_WORDS - )); + llmod.add_raw_global(crate::expr::inline_cache_global_definition(ic_name)); } for raw in &typed_parse_rodata { llmod.add_raw_global(raw.clone()); @@ -1870,11 +1866,7 @@ pub(super) fn compile_static_method( llmod.declare_function(&name, ret, ¶ms); } for ic_name in &ic_globals { - llmod.add_raw_global(format!( - "@{} = private global [{} x i64] zeroinitializer", - ic_name, - crate::expr::property_get::generic_dispatch::PIC_CACHE_WORDS - )); + llmod.add_raw_global(crate::expr::inline_cache_global_definition(ic_name)); } for raw in &typed_parse_rodata { llmod.add_raw_global(raw.clone()); diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index ee04a90858..0297f1acc5 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -72,15 +72,27 @@ pub(crate) fn lower_symbol_property_get_ic( ctx.ic_site_counter += 1; let cache_name = super::inline_cache_global_name(ctx, site_id); ctx.ic_globals.push(cache_name.clone()); - let cache_ref = format!("@{cache_name}"); + let probe_idx = ctx.new_block("symic.probe"); let hit_idx = ctx.new_block("symic.hit"); let miss_idx = ctx.new_block("symic.miss"); let merge_idx = ctx.new_block("symic.merge"); + let probe_label = ctx.block_label(probe_idx); let hit_label = ctx.block_label(hit_idx); let miss_label = ctx.block_label(miss_idx); let merge_label = ctx.block_label(merge_idx); + // #9708: the cache sits behind a pointer slot that the miss handler fills + // on the first prime. The probe's three loads go through the pointer, so + // an absent cache branches straight to the miss — the edge a fresh + // (all-zero) global took anyway, since a zero epoch never matches. + let ic_slot = super::emit_inline_cache_slot(ctx, &cache_name); + let cache_ref = ic_slot.cache.clone(); + let cache_slot_ref = ic_slot.slot_ref.clone(); + ctx.block() + .cond_br(&ic_slot.present, &probe_label, &miss_label); + + ctx.current_block = probe_idx; let epoch = ctx .block() .load_atomic_acquire(I64, "@PERRY_SYMBOL_PROPERTY_IC_EPOCH", 8); @@ -110,7 +122,7 @@ pub(crate) fn lower_symbol_property_get_ic( let miss_value = ctx.block().call( DOUBLE, "js_object_get_symbol_property_ic_miss", - &[(DOUBLE, obj_box), (DOUBLE, sym_box), (PTR, &cache_ref)], + &[(DOUBLE, obj_box), (DOUBLE, sym_box), (PTR, &cache_slot_ref)], ); let miss_end = ctx.block().label.clone(); ctx.block().br(&merge_label); diff --git a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs index 2113d0370b..1ca6db0f2b 100644 --- a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs +++ b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs @@ -340,7 +340,17 @@ pub(super) fn lower_inline_dyn_typed_array_get( ctx.ic_site_counter += 1; let cache_name = super::super::inline_cache_global_name(ctx, site_id); ctx.ic_globals.push(cache_name.clone()); - let cache_ref = format!("@{cache_name}"); + // #9708: the cache sits behind a pointer slot the runtime fills on the + // first shape-carried prime. `arrlike.ic.shape` reads word 0 inside a + // flat predicate, so it reads through `key_cache`: the real cache when + // present, else the slot itself — 8 bytes of null, i.e. a zero identity, + // which fails `key_nonzero` exactly as the all-zero global did. Every + // later word is read only past that edge, through the real pointer. + let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + let cache_ref = ic_slot.cache.clone(); + let key_cache = ctx + .block() + .select(I1, &ic_slot.present, PTR, &cache_ref, &ic_slot.slot_ref); let object_header_idx = ctx.new_block("arrlike.ic.header"); let object_brand_idx = ctx.new_block("arrlike.ic.brand"); @@ -613,7 +623,7 @@ pub(super) fn lower_inline_dyn_typed_array_get( let shape64 = ctx.block().zext(I32, &shape_id, I64); let class_high = ctx.block().shl(I64, &class64, "32"); let live_key = ctx.block().or(I64, &class_high, &shape64); - let cached_key_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); + let cached_key_ptr = ctx.block().gep(I64, &key_cache, &[(I64, "0")]); let cached_key = ctx.block().load(I64, &cached_key_ptr); let key_nonzero = ctx.block().icmp_ne(I64, &cached_key, "0"); let object_ok = ctx.block().and(I1, &is_object, &key_nonzero); @@ -843,7 +853,7 @@ pub(super) fn lower_inline_dyn_typed_array_get( let slow_raw = ctx.block().call( DOUBLE, "js_packed_arraylike_index_get", - &[(DOUBLE, obj_box), (DOUBLE, idx_d), (PTR, &cache_ref)], + &[(DOUBLE, obj_box), (DOUBLE, idx_d), (PTR, &ic_slot.slot_ref)], ); // In a number context, coerce the (possibly boxed) slow result here so the // merge phi is uniformly a Number and the arithmetic caller skips its own diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index dc6f0e332d..a360d47452 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -1517,8 +1517,10 @@ pub(crate) struct FnCtx<'a> { pub ic_site_counter: u32, /// (Issue #51) Names of IC globals created during lowering. After - /// the function is emitted, the caller emits `@ = private - /// global [2 x i64] zeroinitializer` for each entry. + /// the function is emitted, the caller emits one + /// [`inline_cache_global_definition`] — `@ = private global ptr + /// null`, an 8-byte slot the runtime fills with an arena cache on the + /// site's first priming miss (#9708) — for each entry. pub ic_globals: Vec, /// Region-scoped cache selected by a guarded statement fusion. Generic @@ -2235,6 +2237,50 @@ pub(crate) fn inline_cache_global_name(ctx: &FnCtx<'_>, site_id: u32) -> String inline_cache_global_name_for_prefix(ctx.strings.module_prefix(), site_id) } +/// The definition emitted for every name in `FnCtx::ic_globals`. +/// +/// #9708: an inline-cache site owns an 8-byte pointer **slot**, not its cache +/// words. The slot is zero-initialised (so it lands in `__bss` and costs +/// nothing until touched) and stays null until the runtime's miss handler +/// primes the site, at which point it publishes a cache allocated from the +/// runtime's IC arena (`perry_runtime::object::pic_slot_resolve`). A site +/// the program never executes therefore costs 8 bytes of zero-fill instead +/// of a 96-byte cache that dirtied a resident page on first touch. Every +/// inline hit path loads the slot through [`emit_inline_cache_slot`] and +/// proves it non-null before reading a cache word; every runtime miss entry +/// takes the slot's address. +pub(crate) fn inline_cache_global_definition(name: &str) -> String { + format!("@{name} = private global ptr null") +} + +/// A site's inline-cache slot, loaded in the current block (#9708). +/// +/// `slot_ref` is the `@perry_ic_N` global — what the runtime miss entries +/// take. `cache` is the `ptr` loaded from it and `present` the `i1` proving +/// it non-null: a site must branch (or fold `present` into a guard that +/// dominates) before it emits any load through `cache`, because the slot is +/// null until the site's first priming miss. +pub(crate) struct InlineCacheSlot { + pub slot_ref: String, + pub cache: String, + pub present: String, +} + +/// Load `@`'s cache pointer in the current block and test it. +/// The load has no dependency on the receiver, so it is free to issue early; +/// folding `present` into the receiver guard the site already evaluates costs +/// a single fused compare on the hit path. +pub(crate) fn emit_inline_cache_slot(ctx: &mut FnCtx<'_>, cache_name: &str) -> InlineCacheSlot { + let slot_ref = format!("@{cache_name}"); + let cache = ctx.block().load(PTR, &slot_ref); + let present = ctx.block().icmp_ne(PTR, &cache, "null"); + InlineCacheSlot { + slot_ref, + cache, + present, + } +} + /// Record a cold-arm bailout for a compiler-private versioned-loop callback. /// The stack context is `[counter_slot_ptr, original_bound, resume_index]` as /// three i64 words. The first cold arm stores `counter + 1` and poisons the diff --git a/crates/perry-codegen/src/expr/property_get/composed_ics.rs b/crates/perry-codegen/src/expr/property_get/composed_ics.rs index 4171ce5130..38ecc9ecdb 100644 --- a/crates/perry-codegen/src/expr/property_get/composed_ics.rs +++ b/crates/perry-codegen/src/expr/property_get/composed_ics.rs @@ -42,27 +42,42 @@ pub(super) fn lower_symbol_then_named_property_ic( let symbol_site = ctx.ic_site_counter; ctx.ic_site_counter += 1; - let symbol_cache = super::super::inline_cache_global_name(ctx, symbol_site); - ctx.ic_globals.push(symbol_cache.clone()); - let symbol_cache = format!("@{symbol_cache}"); + let symbol_cache_name = super::super::inline_cache_global_name(ctx, symbol_site); + ctx.ic_globals.push(symbol_cache_name.clone()); let field_site = ctx.ic_site_counter; ctx.ic_site_counter += 1; - let field_cache = super::super::inline_cache_global_name(ctx, field_site); - ctx.ic_globals.push(field_cache.clone()); - let field_cache = format!("@{field_cache}"); + let field_cache_name = super::super::inline_cache_global_name(ctx, field_site); + ctx.ic_globals.push(field_cache_name.clone()); + let probe_idx = ctx.new_block("symfield.probe"); let identity_idx = ctx.new_block("symfield.identity"); let hit_idx = ctx.new_block("symfield.hit"); let live_idx = ctx.new_block("symfield.live"); let miss_idx = ctx.new_block("symfield.miss"); let merge_idx = ctx.new_block("symfield.merge"); + let probe_label = ctx.block_label(probe_idx); let identity_label = ctx.block_label(identity_idx); let hit_label = ctx.block_label(hit_idx); let live_label = ctx.block_label(live_idx); let miss_label = ctx.block_label(miss_idx); let merge_label = ctx.block_label(merge_idx); + // #9708: both caches sit behind pointer slots the miss handler fills + // on the first prime. The probe, identity and hit blocks read through + // the two pointers, so they are reached only once both are present; + // an absent cache is the miss edge a zero epoch always took. + let symbol_slot = crate::expr::emit_inline_cache_slot(ctx, &symbol_cache_name); + let field_slot = crate::expr::emit_inline_cache_slot(ctx, &field_cache_name); + let symbol_cache = symbol_slot.cache.clone(); + let field_cache = field_slot.cache.clone(); + let both_present = ctx + .block() + .and(I1, &symbol_slot.present, &field_slot.present); + ctx.block() + .cond_br(&both_present, &probe_label, &miss_label); + + ctx.current_block = probe_idx; let epoch = ctx .block() .load_atomic_acquire(I64, "@PERRY_SYMBOL_PROPERTY_IC_EPOCH", 8); @@ -144,8 +159,8 @@ pub(super) fn lower_symbol_then_named_property_ic( (DOUBLE, &symbol_box), (PTR, &key_ptr), (I64, &feedback_site_id), - (PTR, &symbol_cache), - (PTR, &field_cache), + (PTR, &symbol_slot.slot_ref), + (PTR, &field_slot.slot_ref), ], ); let miss_end = ctx.block().label.clone(); @@ -175,10 +190,10 @@ pub(super) fn emit_array_subclass_length_ic( ctx.ic_site_counter += 1; let cache_name = super::super::inline_cache_global_name(ctx, site_id); ctx.ic_globals.push(cache_name.clone()); - let cache_ref = format!("@{cache_name}"); let header_idx = ctx.new_block("plen.ic.header"); let shape_idx = ctx.new_block("plen.ic.shape"); + let shape_probe_idx = ctx.new_block("plen.ic.shape.probe"); let identity_idx = ctx.new_block("plen.ic.identity"); let exact_idx = ctx.new_block("plen.ic.exact"); let family_meta_idx = ctx.new_block("plen.ic.family_meta"); @@ -192,6 +207,7 @@ pub(super) fn emit_array_subclass_length_ic( let merge_idx = ctx.new_block("plen.ic.merge"); let header_label = ctx.block_label(header_idx); let shape_label = ctx.block_label(shape_idx); + let shape_probe_label = ctx.block_label(shape_probe_idx); let identity_label = ctx.block_label(identity_idx); let exact_label = ctx.block_label(exact_idx); let family_meta_label = ctx.block_label(family_meta_idx); @@ -219,6 +235,14 @@ pub(super) fn emit_array_subclass_length_ic( let below_ceiling = ctx.block().icmp_ult(I64, recv_handle, &heap_ceiling); let in_heap = ctx.block().and(I1, &pointer_tag, &above_floor); let in_heap = ctx.block().and(I1, &in_heap, &below_ceiling); + // #9708: the cache sits behind a pointer slot the runtime fills on the + // first shape-carried prime. It is loaded here, off the receiver's + // dependency chain, and tested at `plen.ic.shape` — NOT folded into the + // header guard, because the elements-backed arm between them serves + // `length` without ever publishing a cache, and a site whose receivers + // are all elements-backed must keep that arm with a slot that stays null. + let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + let cache_ref = ic_slot.cache.clone(); ctx.block().cond_br(&in_heap, &header_label, &miss_label); ctx.current_block = header_idx; @@ -281,6 +305,10 @@ pub(super) fn emit_array_subclass_length_ic( ctx.block().br(&merge_label); ctx.current_block = shape_idx; + ctx.block() + .cond_br(&ic_slot.present, &shape_probe_label, &miss_label); + + ctx.current_block = shape_probe_idx; let object_ptr = ctx.block().inttoptr(I64, recv_handle); let class_id = ctx.block().load(I32, &object_ptr); let shape_addr = ctx.block().add(I64, recv_handle, "4"); @@ -395,7 +423,7 @@ pub(super) fn emit_array_subclass_length_ic( let miss_length = ctx.block().call( DOUBLE, "js_value_length_property_ic_f64", - &[(DOUBLE, recv_box), (PTR, &cache_ref)], + &[(DOUBLE, recv_box), (PTR, &ic_slot.slot_ref)], ); let miss_end = ctx.block().label.clone(); ctx.block().br(&merge_label); diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 1cdb2c8424..29040b9c65 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -13,15 +13,19 @@ use perry_hir::Expr; use crate::nanbox::POINTER_MASK_I64; use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; -/// Words in a per-site `@perry_ic_N` property-read cache global. +/// Words in a per-site property-read cache. /// -/// **Must equal `perry_runtime::object::field_get_set::PIC_CACHE_WORDS`** — -/// the runtime writes this memory through a `*mut [i64; PIC_CACHE_WORDS]`, so a -/// smaller global here is an out-of-bounds store. perry-codegen does not depend -/// on perry-runtime (the same reason `INLINE_SLOT_FLOOR` is duplicated in +/// **Must equal `perry_runtime::object::field_get_set::PIC_CACHE_WORDS`.** +/// Since #9708 codegen emits only the 8-byte slot (`@perry_ic_N = private +/// global ptr null`) and the runtime allocates the words itself, sized from +/// its own `PicCache` — so the constant is no longer an emission width, but +/// the emitted way GEPs (`PIC_WAY_BASE + PIC_WAYS * 2` words) must still +/// land inside that allocation. perry-codegen does not depend on +/// perry-runtime (the same reason `INLINE_SLOT_FLOOR` is duplicated in /// `target_layout`), so the pairing is held by `pic_cache_layout_matches_runtime` /// here and `pic_cache_words_match_codegen` in the runtime: change one and both /// fail. +#[cfg(test)] pub(crate) const PIC_CACHE_WORDS: usize = 12; /// First word of the polymorphic way array (words 0..2 are the MRU entry and /// word 3 is the gate). Mirrors the runtime's `PIC_WAY_BASE`. @@ -136,7 +140,9 @@ pub(crate) fn lower_generic_property_get( // unchanged. let cache_name = overridden_cache_name(ctx, object, property) .unwrap_or_else(|| allocate_property_cache(ctx)); - let cache_ref = format!("@{}", cache_name); + // #9708: the helper takes the site's SLOT and resolves the cache + // itself; nothing is read inline here, so no load is emitted. + let cache_slot_ref = format!("@{}", cache_name); let key_handle = emit_key_handle(ctx, &key_handle_global); let val = ctx.block().call( DOUBLE, @@ -145,7 +151,7 @@ pub(crate) fn lower_generic_property_get( (I64, &obj_bits), (I64, &key_handle), (I64, &feedback_site_id), - (PTR, &cache_ref), + (PTR, &cache_slot_ref), ], ); return Ok(val); @@ -354,7 +360,6 @@ pub(crate) fn lower_generic_property_get( // // Threshold matches `js_native_call_method`'s small-handle // detection (raw_ptr < 0x100000). - let cache_ref = format!("@{}", cache_name); let is_real_ptr = ctx.block().icmp_ugt(I64, &obj_handle, "1048575"); // 0x100000 // #7883: the hit/miss/merge blocks are minted here so the guard chain @@ -462,7 +467,21 @@ pub(crate) fn lower_generic_property_get( let reserved = ctx.block().load(crate::types::I16, &reserved_ptr); let has_desc = ctx.block().and(crate::types::I16, &reserved, "2048"); // OBJ_FLAG_HAS_DESCRIPTORS (0x800) let no_desc = ctx.block().icmp_eq(crate::types::I16, &has_desc, "0"); + // #9708: the site's cache lives behind a pointer slot that is null until + // the first priming miss. The slot load does not depend on the receiver, + // so it issues alongside the header loads, and its non-null test joins + // the flat header predicate as one more fused compare. Both edges that + // read a cache word (`pic.token` and the descriptor prefix path) require + // `cache_present`; the slot itself is what the miss handler takes, so a + // fresh site goes straight to it. `cache_ref` is the LOADED pointer from + // here on, never the global: every GEP below goes through it, and only + // the runtime calls take `cache_slot_ref`. + let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + let cache_ref = ic_slot.cache.clone(); + let cache_slot_ref = ic_slot.slot_ref.clone(); + let cache_present = ic_slot.present.clone(); let is_plain_object = ctx.block().and(I1, &is_object_kind, &no_desc); + let is_plain_object = ctx.block().and(I1, &is_plain_object, &cache_present); // #7883: first exit. The header predicates above are kept as one flat // `and` on purpose — they are loads from the same cache line and LLVM @@ -485,8 +504,15 @@ pub(crate) fn lower_generic_property_get( // is unrelated to all class-declared named fields and arm word 2. Keep // this classification off the ordinary descriptor-free hit path. ctx.current_block = desc_classify_idx; - ctx.block() - .cond_br(&is_object_kind, &desc_prefix_guard_label, &cold_label); + // #9708: the descriptor prefix path reads cache word 2, so it needs the + // same non-null proof `pic.token` has; without a cache the receiver is + // simply a cold miss. + let desc_object_with_cache = ctx.block().and(I1, &is_object_kind, &cache_present); + ctx.block().cond_br( + &desc_object_with_cache, + &desc_prefix_guard_label, + &cold_label, + ); ctx.current_block = tok_idx; // The receiver token is derived solely from its authoritative ShapeId. @@ -578,7 +604,7 @@ pub(crate) fn lower_generic_property_get( (I64, &obj_handle), (I64, &ovf_key_handle), (I32, &ovf_slot_i32), - (PTR, &cache_ref), + (PTR, &cache_slot_ref), ], ); let ovf_end_label = ctx.block().label.clone(); @@ -931,7 +957,7 @@ pub(crate) fn lower_generic_property_get( &[ (I64, &obj_handle), (I64, &miss_key_handle), - (PTR, &cache_ref), + (PTR, &cache_slot_ref), ], ); let miss_end_label = ctx.block().label.clone(); diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 72542af871..5196d9c092 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -276,12 +276,13 @@ fn fs_parent_promises_property_installs_before_resolution() { /// #7753, paired with `pic_cache_words_match_codegen` in /// `perry-runtime/src/object/field_get_set/ic_miss.rs`. /// -/// The runtime writes a `@perry_ic_N` global through `*mut [i64; -/// PIC_CACHE_WORDS]`. If codegen emits a NARROWER global, `pic_prime_get`'s way -/// stores run past the end of it into whatever global the linker placed next — -/// silent memory corruption that no property-read test would notice. This pins -/// the emitted width to the constant both sides share, and pins the constant -/// itself so the runtime's copy cannot drift. +/// The runtime writes a site's cache through `*mut [i64; PIC_CACHE_WORDS]` +/// and the emitted ways read words up to `PIC_WAY_BASE + PIC_WAYS * 2`. Since +/// #9708 the cache words are allocated by the runtime (`pic_slot_resolve` +/// sizes them from its own `PicCache`), so the two constants pinned here are +/// what keeps the emitted way GEPs inside that allocation. The emitted global +/// itself is the 8-byte SLOT, never the words: a `[N x i64]` IC global would +/// be the pre-#9708 shape coming back, with its 96 B of zero-fill per site. #[test] fn pic_cache_layout_matches_runtime() { use crate::expr::property_get::generic_dispatch::{ @@ -301,11 +302,25 @@ fn pic_cache_layout_matches_runtime() { "runtime PicCache word 2 carries the Array-subclass named-prefix token" ); let ir = emit(false, None); + let ic_defs: Vec<&str> = ir + .lines() + .filter(|l| l.starts_with("@perry_ic_") && l.contains(" = ")) + .collect(); assert!( - ir.contains(&format!( - "= private global [{PIC_CACHE_WORDS} x i64] zeroinitializer" - )), - "every @perry_ic_N must be emitted at the width the runtime writes:\n{ir}" + !ic_defs.is_empty(), + "test premise: the generic read emits a per-site cache slot:\n{ir}" + ); + for def in &ic_defs { + assert!( + def.ends_with(" = private global ptr null"), + "every @perry_ic_N must be an 8-byte null pointer slot the runtime \ + fills on the first prime (#9708), got:\n{def}\n\nIR:\n{ir}" + ); + } + assert!( + ir.contains("load ptr, ptr @perry_ic_") && ir.contains("icmp ne ptr "), + "the hit path must load the slot and prove it non-null before reading \ + a cache word:\n{ir}" ); } diff --git a/crates/perry-codegen/src/expr/proxy_reflect.rs b/crates/perry-codegen/src/expr/proxy_reflect.rs index 23ea00e5a4..e4e84e19d7 100644 --- a/crates/perry-codegen/src/expr/proxy_reflect.rs +++ b/crates/perry-codegen/src/expr/proxy_reflect.rs @@ -510,13 +510,21 @@ fn lower_put_value_static_write_ic( ctx.pending_declares .push((format!("__ic_decl_{}", site_id), DOUBLE, vec![])); ctx.ic_globals.push(cache_name.clone()); - let cache_ref = format!("@{}", cache_name); // Keep the first four ways inline. Shapes 5–8 use a separate cache in a // compact outlined helper, avoiding four more copies of the generated // receiver guards while preventing the fourth inline way from thrashing. let tail_cache_name = format!("{}_poly_tail", cache_name); ctx.ic_globals.push(tail_cache_name.clone()); - let tail_cache_ref = format!("@{}", tail_cache_name); + // #9708: both caches sit behind pointer slots. The inline ways read the + // primary cache through the pointer loaded here, so its non-null test + // joins `heap_candidate` (every way guard is dominated by that edge); a + // site that has never primed goes to the way-0 miss, which allocates. The + // tail is only ever handed to the runtime, which resolves it itself — + // so a tail cache is not allocated until a fifth shape actually arrives. + let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + let cache_ref = ic_slot.cache.clone(); + let cache_slot_ref = ic_slot.slot_ref.clone(); + let tail_cache_slot_ref = format!("@{}", tail_cache_name); // Branch before the first header load so primitives, forged non-pointer // bit patterns, and native handle ids can never be dereferenced by the @@ -525,6 +533,7 @@ fn lower_put_value_static_write_ic( let pointer_tag = ctx.block().icmp_eq(I64, &target_tag, "32765"); // 0x7FFD let above_handles = ctx.block().icmp_ugt(I64, &target_handle, "1048575"); // 0x100000 let heap_candidate = ctx.block().and(I1, &pointer_tag, &above_handles); + let heap_candidate = ctx.block().and(I1, &heap_candidate, &ic_slot.present); let guard_idx = ctx.new_block("put.pic.guard"); let guard2_idx = ctx.new_block("put.pic.guard2"); let guard3_idx = ctx.new_block("put.pic.guard3"); @@ -854,7 +863,8 @@ fn lower_put_value_static_write_ic( (I64, &key_handle), (DOUBLE, &stored_value), (I32, strict_i32), - (PTR, &cache_ref), + (PTR, &cache_slot_ref), + (I32, "0"), ], ); let deleted_end_label = ctx.block().label.clone(); @@ -869,7 +879,8 @@ fn lower_put_value_static_write_ic( (I64, &key_handle), (DOUBLE, &stored_value), (I32, strict_i32), - (PTR, &cache_ref), + (PTR, &cache_slot_ref), + (I32, "0"), ], ); let miss_end_label = ctx.block().label.clone(); @@ -884,7 +895,8 @@ fn lower_put_value_static_write_ic( (I64, &key_handle), (DOUBLE, &stored_value), (I32, strict_i32), - (PTR, &cached2_token_ptr), + (PTR, &cache_slot_ref), + (I32, "1"), ], ); let miss2_end_label = ctx.block().label.clone(); @@ -899,7 +911,8 @@ fn lower_put_value_static_write_ic( (I64, &key_handle), (DOUBLE, &stored_value), (I32, strict_i32), - (PTR, &cached3_token_ptr), + (PTR, &cache_slot_ref), + (I32, "2"), ], ); let miss3_end_label = ctx.block().label.clone(); @@ -914,7 +927,8 @@ fn lower_put_value_static_write_ic( (I64, &key_handle), (DOUBLE, &stored_value), (I32, strict_i32), - (PTR, &cached4_token_ptr), + (PTR, &cache_slot_ref), + (I32, "3"), ], ); let miss4_end_label = ctx.block().label.clone(); @@ -925,7 +939,7 @@ fn lower_put_value_static_write_ic( DOUBLE, "js_put_value_set_ic_poly_tail", &[ - (PTR, &tail_cache_ref), + (PTR, &tail_cache_slot_ref), (DOUBLE, &target_value), (I64, &key_handle), (DOUBLE, &stored_value), @@ -974,7 +988,25 @@ fn lower_put_value_dyn_ic_inline( ctx.ic_site_counter += 1; let cache_name = super::inline_cache_global_name(ctx, site_id); ctx.ic_globals.push(cache_name.clone()); - let cache_ref = format!("@{}", cache_name); + // #9708: the site cache sits behind a pointer slot that is null until the + // miss handler's first prime. The guard block below reads word 0 in a + // flat predicate, so it reads through `token_cache`: the real cache when + // present, else the SLOT ITSELF — an 8-byte null, i.e. a zero token, which + // is exactly what the all-zero global used to read as. A zero token fails + // `token_nonzero`, so the ways (which read words 1..6 through the real + // pointer) are unreachable for an absent cache, and the transition probe + // is reached on the same edge it always was. The outlined slow entry and + // the miss handler take the slot. + let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + let cache_ref = ic_slot.cache.clone(); + let cache_slot_ref = ic_slot.slot_ref.clone(); + let token_cache = ctx.block().select( + I1, + &ic_slot.present, + crate::types::PTR, + &cache_ref, + &cache_slot_ref, + ); // #9287: this thread's transition-cache base, loaded once per function // (the table is thread-local; a link-time constant would alias one @@ -1094,7 +1126,7 @@ fn lower_put_value_dyn_ic_inline( let shape_token = ctx .block() .select(I1, &has_shape_id, I64, &shape_id_token, "0"); - let cached_token_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); + let cached_token_ptr = ctx.block().gep(I64, &token_cache, &[(I64, "0")]); let cached_token = ctx.block().load(I64, &cached_token_ptr); let token_match = ctx.block().icmp_eq(I64, &shape_token, &cached_token); let token_nonzero = ctx.block().icmp_ne(I64, &shape_token, "0"); @@ -1433,7 +1465,7 @@ fn lower_put_value_dyn_ic_inline( DOUBLE, "js_put_value_set_dyn_ic", &[ - (crate::types::PTR, &cache_ref), + (crate::types::PTR, &cache_slot_ref), (DOUBLE, t), (DOUBLE, k), (DOUBLE, v), @@ -2049,12 +2081,13 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { ctx.ic_site_counter += 1; let cache_name = super::inline_cache_global_name(ctx, site_id); ctx.ic_globals.push(cache_name.clone()); - let cache_ref = format!("@{}", cache_name); + // #9708: the outlined entry takes the site's slot. + let cache_slot_ref = format!("@{}", cache_name); Ok(ctx.block().call( DOUBLE, "js_put_value_set_dyn_ic", &[ - (crate::types::PTR, &cache_ref), + (crate::types::PTR, &cache_slot_ref), (DOUBLE, &t), (DOUBLE, &k), (DOUBLE, &v), diff --git a/crates/perry-codegen/src/lower_call/property_get/imported_object.rs b/crates/perry-codegen/src/lower_call/property_get/imported_object.rs index 058558d7af..925e5e5076 100644 --- a/crates/perry-codegen/src/lower_call/property_get/imported_object.rs +++ b/crates/perry-codegen/src/lower_call/property_get/imported_object.rs @@ -60,7 +60,6 @@ fn emit_cached_own_method_guard( ctx.ic_site_counter += 1; let cache_name = crate::expr::inline_cache_global_name(ctx, cache_site); ctx.ic_globals.push(cache_name.clone()); - let cache_ref = format!("@{cache_name}"); let deref_idx = ctx.new_block("object_method_cache.deref"); let fast_idx = ctx.new_block("object_method_cache.fast"); @@ -85,6 +84,15 @@ fn emit_cached_own_method_guard( let below_ceiling = ctx.block().icmp_ult(I64, &recv_handle, &heap_ceiling); let in_range = ctx.block().and(I1, &above_floor, &below_ceiling); let safe = ctx.block().and(I1, &tagged, &in_range); + // #9708: the token lives behind a pointer slot the miss handler fills on + // the first prime. `deref` reads it in a flat predicate, so it reads + // through `token_cache`: the real cache when present, else the slot itself + // — 8 bytes of null, a zero token, which fails `cache_populated` exactly as + // the all-zero global did and takes the same revalidate edge. + let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + let token_cache = + ctx.block() + .select(I1, &ic_slot.present, PTR, &ic_slot.cache, &ic_slot.slot_ref); ctx.block().cond_br(&safe, &deref_label, miss_label); ctx.current_block = deref_idx; @@ -96,7 +104,7 @@ fn emit_cached_own_method_guard( .and(I32, &gc_header, GC_OBJECT_METHOD_GUARD_MASK_I32); let gc_header_ok = ctx.block().icmp_eq(I32, &guarded_gc_bits, GC_TYPE_OBJECT); let live_class_shape = ctx.block().load(I64, &object_ptr); - let cache_token_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); + let cache_token_ptr = ctx.block().gep(I64, &token_cache, &[(I64, "0")]); let cached_class_shape = ctx.block().load(I64, &cache_token_ptr); let cache_populated = ctx.block().icmp_ne(I64, &cached_class_shape, "0"); let shape_matches = ctx @@ -137,7 +145,7 @@ fn emit_cached_own_method_guard( (PTR, &bytes_global), (I64, &name_len), (PTR, &format!("@{closure_symbol}")), - (PTR, &cache_token_ptr), + (PTR, &ic_slot.slot_ref), ], ); let cold_passes = ctx.block().icmp_ne(I64, &cold_handle, "0"); diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index 8ed7f9c547..4839d3264e 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -1447,14 +1447,15 @@ mod tests { // `zeroinitializer` global that only ONE unit defines moves it out of // zerofill `__DATA,__bss` and writes its zeros into the file — 25.16 MB // (8.2%) of the Claude Code binary, all of it per-site inline caches - // (`[12 x i64] zeroinitializer`, one per property-access site, each - // referenced by exactly one function and so by exactly one unit). + // (`[12 x i64] zeroinitializer` then; an 8-byte `ptr null` slot since + // #9708 — one per property-access site either way, each referenced by + // exactly one function and so by exactly one unit). // Promote only what a link would otherwise see defined twice; ELF/COFF // are unaffected either way (their BSS choice ignores linkage). let mut m = LlModule::new("arm64-apple-macosx15.0.0"); m.declare_function("js_ic_touch", VOID, &[PTR]); - m.add_raw_global("@perry_ic_m__0 = private global [12 x i64] zeroinitializer".to_string()); - m.add_raw_global("@perry_ic_m__1 = private global [12 x i64] zeroinitializer".to_string()); + m.add_raw_global(crate::expr::inline_cache_global_definition("perry_ic_m__0")); + m.add_raw_global(crate::expr::inline_cache_global_definition("perry_ic_m__1")); m.add_internal_global("perry_class_keys_m__C", I64, "0"); m.add_global("perry_class_shape_id_m__C", I32, "0"); @@ -1481,9 +1482,7 @@ mod tests { for ic in ["@perry_ic_m__0", "@perry_ic_m__1"] { let defs: Vec<&String> = units .iter() - .filter(|u| { - u.contains(&format!("{ic} = private global [12 x i64] zeroinitializer")) - }) + .filter(|u| u.contains(&format!("{ic} = private global ptr null"))) .collect(); assert_eq!( defs.len(), diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index ef42376207..5a578f4c87 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -527,10 +527,11 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { DOUBLE, &[PTR, DOUBLE, DOUBLE, DOUBLE, I32], ); + // #9708: takes the site's cache SLOT plus the way index to prime. module.declare_function( "js_put_value_set_ic_miss", DOUBLE, - &[DOUBLE, I64, DOUBLE, I32, PTR], + &[DOUBLE, I64, DOUBLE, I32, PTR, I32], ); // #9287: validate-and-store for a constant-key IC hit whose slot word // carries the overflow bit (property lives in the spill buffer). diff --git a/crates/perry-codegen/src/stmt/cached_field_index_return.rs b/crates/perry-codegen/src/stmt/cached_field_index_return.rs index 4cbd480574..709d085f83 100644 --- a/crates/perry-codegen/src/stmt/cached_field_index_return.rs +++ b/crates/perry-codegen/src/stmt/cached_field_index_return.rs @@ -119,7 +119,6 @@ pub(super) fn try_emit_cached_field_index_return( } let cache_name = allocate_shared_cache(ctx, &candidate); - let cache_ref = format!("@{cache_name}"); let base_box = lower_expr(ctx, &Expr::LocalGet(candidate.base_local_id))?; let index_i32 = ctx.block().load(I32, &index_slot); @@ -169,7 +168,15 @@ pub(super) fn try_emit_cached_field_index_return( let gc_flags = ctx.block().load(I8, &gc_flags_ptr); let forwarded_bits = ctx.block().and(I8, &gc_flags, "128"); let not_forwarded = ctx.block().icmp_eq(I8, &forwarded_bits, "0"); + // #9708: the shared cache sits behind a pointer slot that the generic + // property-get miss handler fills on the first prime. Every cache read + // below (`exact_token`, `prefix_meta`, `field_load`) is dominated by this + // edge, so the non-null test joins the header predicate; a site whose + // cache is still unallocated simply takes the normal lowering. + let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + let cache_ref = ic_slot.cache.clone(); let object_ok = ctx.block().and(I1, &is_object, ¬_forwarded); + let object_ok = ctx.block().and(I1, &object_ok, &ic_slot.present); ctx.block() .cond_br(&object_ok, &exact_or_prefix_label, &normal_label); diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 71ac00ae83..99359c9d92 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -201,7 +201,8 @@ pub(crate) use self::sort::object_prototype_has_index_prop; pub(crate) use self::sort::object_prototype_index_get as sort_object_prototype_index_get; pub(crate) use self::sort::object_prototype_index_get_with_receiver as sort_object_prototype_index_get_with_receiver; pub use self::subclass::{ - array_subclass_dense_snapshot, array_subclass_has_iterator_override, is_array_subclass_instance, + array_subclass_dense_snapshot, array_subclass_has_iterator_override, + is_array_subclass_instance, ArrayLikePicCache, ArrayLikePicCacheSlot, ARRAYLIKE_PIC_WORDS, }; #[cfg(test)] pub(crate) use indexing_support::test_swap_array_index_fast_path_invalidated; diff --git a/crates/perry-runtime/src/array/subclass.rs b/crates/perry-runtime/src/array/subclass.rs index 061309fe17..28074a6a12 100644 --- a/crates/perry-runtime/src/array/subclass.rs +++ b/crates/perry-runtime/src/array/subclass.rs @@ -16,6 +16,11 @@ use crate::value::JSValue; #[path = "subclass_loop_guard.rs"] pub(super) mod loop_guard; +#[path = "subclass_packed_index.rs"] +pub(super) mod packed_index; +#[cfg(test)] +pub(super) use packed_index::js_packed_arraylike_index_get; +pub use packed_index::{ArrayLikePicCache, ArrayLikePicCacheSlot, ARRAYLIKE_PIC_WORDS}; // The loop-guard entry points are exported C symbols; only the unit tests // reach them through Rust paths. #[cfg(test)] @@ -905,8 +910,15 @@ pub(crate) fn array_subclass_fast_length(value: f64) -> Option { /// Array-subclass named-prefix token used by the dense indexed-read IC. The /// payload is published before the identity, and no managed pointer escapes /// into the cache, so moving GC needs neither a root nor a rewrite hook. +/// +/// `cache_slot` is the site's slot (#9708); the cache is allocated only on +/// the publishing path below, so the elements-backed early return never +/// allocates one. #[inline] -pub(crate) fn array_subclass_fast_length_with_ic(value: f64, cache: *mut u64) -> Option { +pub(crate) fn array_subclass_fast_length_with_ic( + value: f64, + cache_slot: *mut crate::value::LengthPicCacheSlot, +) -> Option { if let Some(elements) = validated_object_receiver_for_value(value) .and_then(|r| super::subclass_elements::elements_for_validated(&r)) { @@ -916,7 +928,10 @@ pub(crate) fn array_subclass_fast_length_with_ic(value: f64, cache: *mut u64) -> } let (obj, layout) = dense_layout_for_value(value)?; let result = f64::from_bits(layout_length_value(obj, layout).bits()); - if !cache.is_null() { + if !cache_slot.is_null() { + // SAFETY: a non-null slot is the emitted pointer global or a test's + // stack slot; resolving it allocates the cache on the first prime. + let cache = unsafe { crate::object::pic_slot_resolve(cache_slot) } as *mut u64; let family_token = if crate::object::object_spill_enabled() { unsafe { array_subclass_named_prefix_token_for_slot(obj, layout.length_slot as usize) } } else { @@ -1531,94 +1546,6 @@ fn canonical_u32_index(value: f64) -> Option { .then_some(value as u32) } -/// Unknown-receiver numeric read used by codegen's guarded typed-array miss -/// block. Stable real arrays and Array subclasses terminate here; every other -/// receiver/key keeps the established tag-aware dispatcher as a cold side -/// exit. Keeping that call behind this ABI boundary removes `js_dyn_index_get` -/// from the emitted hot-loop artifact without weakening its semantics. -#[no_mangle] -/// The five optional IC words are -/// scalar layout facts, never heap pointers: -/// `(class_id, ShapeId)`, length slot, element base, dense prefix, inline bound. -/// The emitted hit path reloads the live object/meta/spill pointers, so moving -/// GC never has to trace or rewrite this cache. -pub extern "C" fn js_packed_arraylike_index_get(receiver: f64, index: f64, cache: *mut u64) -> f64 { - if let Some(index_u32) = canonical_u32_index(index) { - let js = JSValue::from_bits(receiver.to_bits()); - if js.is_pointer() { - let raw = js.as_pointer::(); - if let Some(header) = - unsafe { crate::value::addr_class::try_read_gc_header(raw as usize) } - { - if matches!( - header.obj_type, - crate::gc::GC_TYPE_ARRAY | crate::gc::GC_TYPE_LAZY_ARRAY - ) { - return crate::array::js_array_get_f64( - raw as *const crate::array::ArrayHeader, - index_u32, - ); - } - if header.obj_type == crate::gc::GC_TYPE_OBJECT - && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 - { - let obj = raw.cast::(); - // Elements-backed instance: an in-bounds non-hole element - // answers directly; a hole continues to the complete - // dispatcher (prototype chain). - let elements = unsafe { super::subclass_elements::elements_of(obj) }; - if !elements.is_null() { - if let Some(value) = - super::subclass_elements::elements_index_get(elements, index_u32) - { - return value; - } - } else if let Some(layout) = dense_layout_for_validated_object(obj) { - // The codegen hit path handles both inline and - // object-owned spill slots. In spill mode, publish a - // class-wide dense-tail identity when the owner has - // proved one. Exact push/pop transitions preserve - // that move-stable token, so a lifecycle loop does not - // miss once for every historical tail ShapeId. The - // cached dense-prefix word remains the admitted high - // water mark: a generic `length` grow beyond it still - // side-exits and re-establishes the complete proof. - if !cache.is_null() && crate::object::object_spill_enabled() { - let family_token = unsafe { - array_subclass_named_prefix_token_for_slot( - obj, - layout.length_slot as usize, - ) - }; - unsafe { - // GC_STORE_AUDIT(POINTER_FREE): generated IC - // words are scalar layout facts, not heap edges. - cache.add(1).write(layout.length_slot as u64); - cache.add(2).write(layout.element_base as u64); - cache.add(3).write(layout.dense_prefix_len as u64); - cache.add(4).write(layout.live_inline_slots as u64); - cache.write(if family_token != 0 { - family_token - } else { - dense_cache_key((*obj).class_id, (*obj).parent_class_id) - }); - } - } - if let Some(value) = dense_index_get_with_layout(obj, layout, index_u32) { - return value; - } - } - } - } - } - } - crate::value::js_dyn_index_get(receiver, index) -} -#[cfg(feature = "keepalive-anchors")] -#[used] -static KEEP_JS_PACKED_ARRAYLIKE_INDEX_GET: extern "C" fn(f64, f64, *mut u64) -> f64 = - js_packed_arraylike_index_get; - /// True when `class_id` is a user class that extends `Array` (the reserved /// parent id `0xFFFF0024` appears in its class chain), i.e. `class X extends /// Array`. Such instances are plain `ObjectHeader`s, so the array-like engines diff --git a/crates/perry-runtime/src/array/subclass_packed_index.rs b/crates/perry-runtime/src/array/subclass_packed_index.rs new file mode 100644 index 0000000000..2d50e3eef1 --- /dev/null +++ b/crates/perry-runtime/src/array/subclass_packed_index.rs @@ -0,0 +1,121 @@ +//! The packed array-like numeric read codegen's guarded typed-array miss +//! block calls, with its per-site index cache (#9708: allocated on the first +//! prime, behind an emitted pointer slot). +//! +//! Child module of `subclass.rs`, split out to stay under the 2,000-line file +//! gate; `use super::*` keeps the parent's private layout helpers reachable. + +use super::*; + +/// Words in a per-site packed array-like index cache: `(identity, length +/// slot, element base, dense prefix, inline bound)`. +pub const ARRAYLIKE_PIC_WORDS: usize = 5; +/// A per-site packed array-like index cache, as the emitted slot resolves it. +pub type ArrayLikePicCache = [u64; ARRAYLIKE_PIC_WORDS]; +/// The emitted `@perry_ic_N = private global ptr null` for such a site: null +/// until the site's first priming read (#9708). +pub type ArrayLikePicCacheSlot = *mut ArrayLikePicCache; + +/// Unknown-receiver numeric read used by codegen's guarded typed-array miss +/// block. Stable real arrays and Array subclasses terminate here; every other +/// receiver/key keeps the established tag-aware dispatcher as a cold side +/// exit. Keeping that call behind this ABI boundary removes `js_dyn_index_get` +/// from the emitted hot-loop artifact without weakening its semantics. +/// +/// The five optional IC words are +/// scalar layout facts, never heap pointers: +/// `(class_id, ShapeId)`, length slot, element base, dense prefix, inline bound. +/// The emitted hit path reloads the live object/meta/spill pointers, so moving +/// GC never has to trace or rewrite this cache. +/// +/// `cache_slot` is the site's [`ArrayLikePicCacheSlot`] address; the cache is +/// allocated on the first prime (#9708), so a site whose receivers are plain +/// Arrays or elements-backed instances never allocates one. +#[no_mangle] +pub extern "C" fn js_packed_arraylike_index_get( + receiver: f64, + index: f64, + cache_slot: *mut ArrayLikePicCacheSlot, +) -> f64 { + if let Some(index_u32) = canonical_u32_index(index) { + let js = JSValue::from_bits(receiver.to_bits()); + if js.is_pointer() { + let raw = js.as_pointer::(); + if let Some(header) = + unsafe { crate::value::addr_class::try_read_gc_header(raw as usize) } + { + if matches!( + header.obj_type, + crate::gc::GC_TYPE_ARRAY | crate::gc::GC_TYPE_LAZY_ARRAY + ) { + return crate::array::js_array_get_f64( + raw as *const crate::array::ArrayHeader, + index_u32, + ); + } + if header.obj_type == crate::gc::GC_TYPE_OBJECT + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + { + let obj = raw.cast::(); + // Elements-backed instance: an in-bounds non-hole element + // answers directly; a hole continues to the complete + // dispatcher (prototype chain). + let elements = unsafe { crate::array::subclass_elements::elements_of(obj) }; + if !elements.is_null() { + if let Some(value) = + crate::array::subclass_elements::elements_index_get(elements, index_u32) + { + return value; + } + } else if let Some(layout) = dense_layout_for_validated_object(obj) { + // The codegen hit path handles both inline and + // object-owned spill slots. In spill mode, publish a + // class-wide dense-tail identity when the owner has + // proved one. Exact push/pop transitions preserve + // that move-stable token, so a lifecycle loop does not + // miss once for every historical tail ShapeId. The + // cached dense-prefix word remains the admitted high + // water mark: a generic `length` grow beyond it still + // side-exits and re-establishes the complete proof. + if !cache_slot.is_null() && crate::object::object_spill_enabled() { + // SAFETY: a non-null slot is the emitted pointer + // global or a test's stack slot. + let cache = + unsafe { crate::object::pic_slot_resolve(cache_slot) } as *mut u64; + let family_token = unsafe { + array_subclass_named_prefix_token_for_slot( + obj, + layout.length_slot as usize, + ) + }; + unsafe { + // GC_STORE_AUDIT(POINTER_FREE): generated IC + // words are scalar layout facts, not heap edges. + cache.add(1).write(layout.length_slot as u64); + cache.add(2).write(layout.element_base as u64); + cache.add(3).write(layout.dense_prefix_len as u64); + cache.add(4).write(layout.live_inline_slots as u64); + cache.write(if family_token != 0 { + family_token + } else { + dense_cache_key((*obj).class_id, (*obj).parent_class_id) + }); + } + } + if let Some(value) = dense_index_get_with_layout(obj, layout, index_u32) { + return value; + } + } + } + } + } + } + crate::value::js_dyn_index_get(receiver, index) +} +#[cfg(feature = "keepalive-anchors")] +#[used] +static KEEP_JS_PACKED_ARRAYLIKE_INDEX_GET: extern "C" fn( + f64, + f64, + *mut ArrayLikePicCacheSlot, +) -> f64 = js_packed_arraylike_index_get; diff --git a/crates/perry-runtime/src/array/subclass_tests.rs b/crates/perry-runtime/src/array/subclass_tests.rs index fc2f186f7d..b540f8ff4c 100644 --- a/crates/perry-runtime/src/array/subclass_tests.rs +++ b/crates/perry-runtime/src/array/subclass_tests.rs @@ -366,9 +366,10 @@ fn array_subclass_length_ic_publishes_only_scalar_exact_or_family_facts() { let receiver = crate::value::js_nanbox_pointer(obj as i64); crate::node_stream::js_array_subclass_init(receiver, 0.0); - let mut cache = [0_u64; 3]; + let mut cache: crate::value::LengthPicCache = [0; crate::value::LENGTH_PIC_WORDS]; + let mut cache_slot: crate::value::LengthPicCacheSlot = &mut cache; assert_eq!( - array_subclass_fast_length_with_ic(receiver, cache.as_mut_ptr()), + array_subclass_fast_length_with_ic(receiver, &mut cache_slot), Some(0.0) ); if crate::object::object_spill_enabled() { @@ -392,7 +393,7 @@ fn array_subclass_length_ic_publishes_only_scalar_exact_or_family_facts() { js_array_push_f64(obj as *mut ArrayHeader, 11.0); assert_eq!( - array_subclass_fast_length_with_ic(receiver, cache.as_mut_ptr()), + array_subclass_fast_length_with_ic(receiver, &mut cache_slot), Some(1.0) ); if crate::object::object_spill_enabled() { @@ -677,7 +678,8 @@ fn array_subclass_named_prefix_token_survives_only_exact_numeric_tail_transition js_array_push_f64(obj as *mut ArrayHeader, 22.0); let mask_key = crate::string::js_string_from_bytes(b"mask".as_ptr(), 4); let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; - crate::object::js_object_get_field_ic_miss(obj, mask_key, &mut cache); + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; + crate::object::js_object_get_field_ic_miss(obj, mask_key, &mut cache_slot); let token = cache[2] as u64; assert_ne!(token, 0); assert!(unsafe { array_subclass_named_prefix_token_matches_class(obj, class_id) }); @@ -691,9 +693,10 @@ fn array_subclass_named_prefix_token_survives_only_exact_numeric_tail_transition "the IC miss must publish the same owner-side token it caches" ); - let mut index_ic = [0u64; 5]; + let mut index_ic: super::ArrayLikePicCache = [0; super::ARRAYLIKE_PIC_WORDS]; + let mut index_ic_slot: super::ArrayLikePicCacheSlot = &mut index_ic; assert_eq!( - js_packed_arraylike_index_get(receiver, 0.0, index_ic.as_mut_ptr()), + js_packed_arraylike_index_get(receiver, 0.0, &mut index_ic_slot), 11.0 ); assert_eq!( @@ -711,8 +714,9 @@ fn array_subclass_named_prefix_token_survives_only_exact_numeric_tail_transition let zero_key = crate::string::js_string_from_bytes(b"0".as_ptr(), 1); let mut element_cache = [0i64; crate::object::PIC_CACHE_WORDS]; + let mut element_cache_slot: crate::object::PicCacheSlot = &mut element_cache; assert_eq!( - crate::object::js_object_get_field_ic_miss(obj, zero_key, &mut element_cache), + crate::object::js_object_get_field_ic_miss(obj, zero_key, &mut element_cache_slot), 11.0 ); assert_eq!( @@ -857,7 +861,8 @@ fn empty_array_subclass_named_prefix_token_survives_warm_tail_cycle() { let change_key = crate::string::js_string_from_bytes(b"change".as_ptr(), 6); let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; - let via_ic = crate::object::js_object_get_field_ic_miss(obj, change_key, &mut cache); + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; + let via_ic = crate::object::js_object_get_field_ic_miss(obj, change_key, &mut cache_slot); let via_ladder = crate::object::js_object_get_field_by_name_f64(obj, change_key); assert_eq!(via_ic.to_bits(), via_ladder.to_bits()); let token = cache[2] as u64; diff --git a/crates/perry-runtime/src/gc/census.rs b/crates/perry-runtime/src/gc/census.rs index 5aeaad82a0..891035533f 100644 --- a/crates/perry-runtime/src/gc/census.rs +++ b/crates/perry-runtime/src/gc/census.rs @@ -559,6 +559,7 @@ fn side_tables() -> Vec { rows.extend(crate::object::shapes::shape_table_census()); rows.extend(crate::object::class_registry_census()); rows.extend(crate::object::object_tables_census()); + rows.extend(crate::object::pic_slot_census()); rows.extend(super::roots::stack_map_index_census()); let (slots, bytes) = crate::string::intern_table_census(); rows.push(("string.intern_table(fixed)", slots, bytes)); diff --git a/crates/perry-runtime/src/gc/tests/handle_bound_method_name.rs b/crates/perry-runtime/src/gc/tests/handle_bound_method_name.rs index 0fa6c7e051..a299a8d6ff 100644 --- a/crates/perry-runtime/src/gc/tests/handle_bound_method_name.rs +++ b/crates/perry-runtime/src/gc/tests/handle_bound_method_name.rs @@ -184,11 +184,11 @@ fn a_bound_timer_method_from_the_ic_miss_path_never_captures_the_key() { let id = live_timer(); let (key, key_interior) = heap_key("hasRef"); let mut cache = crate::object::PicCache::default(); - + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; let bits = crate::object::js_object_get_field_ic_miss( id as *const crate::ObjectHeader, key, - &mut cache, + &mut cache_slot, ); assert_names_the_literal( crate::value::JSValue::from_bits(bits.to_bits()), diff --git a/crates/perry-runtime/src/node_submodules/tests.rs b/crates/perry-runtime/src/node_submodules/tests.rs index 30523768b1..a6abfdede0 100644 --- a/crates/perry-runtime/src/node_submodules/tests.rs +++ b/crates/perry-runtime/src/node_submodules/tests.rs @@ -468,8 +468,12 @@ fn test_default_and_named_exports_share_the_self_alias() { crate::object::js_object_get_field_by_name_f64(closure as *const ObjectHeader, key); assert_eq!(property.to_bits(), default.to_bits()); let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; - let property = - crate::object::js_object_get_field_ic_miss(closure as *const ObjectHeader, key, &mut cache); + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; + let property = crate::object::js_object_get_field_ic_miss( + closure as *const ObjectHeader, + key, + &mut cache_slot, + ); assert_eq!(property.to_bits(), default.to_bits()); } diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 5a9a2bb98c..088f9d70d1 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -216,6 +216,7 @@ mod ic_miss; #[cfg(test)] #[path = "field_get_set/ic_miss_array_length_tests.rs"] mod ic_miss_array_length_tests; +mod ic_slot; mod map_set_receiver; mod probe_dispatch; /// #9131: per-instance `[[Prototype]]` override lookup, split out of @@ -304,8 +305,10 @@ pub use ic_miss::{ js_class_field_add, js_object_get_field_by_name_f64, js_object_get_field_by_property_id_f64, js_object_get_field_ic, js_object_get_field_ic_miss, js_object_set_field_by_property_id, js_private_brand_add, js_private_brand_check, js_private_field_add, js_private_guard, PicCache, - PIC_CACHE_WORDS, + PicCacheSlot, PIC_CACHE_WORDS, }; +pub(crate) use ic_slot::pic_slot_census; +pub use ic_slot::{pic_arena_bytes, pic_slot_peek, pic_slot_resolve, pic_slots_resolved}; #[cfg(test)] mod buffer_ic_miss_tests { @@ -337,24 +340,25 @@ mod buffer_ic_miss_tests { for len in [16usize, 24, 32] { let buf = secret_buffer(len); let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; - + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; let ty = js_object_get_field_ic_miss( buf as *const ObjectHeader, key(b"type"), - &mut cache, + &mut cache_slot, ); assert_eq!(string_value_bytes(ty), b"secret"); let size = js_object_get_field_ic_miss( buf as *const ObjectHeader, key(b"symmetricKeySize"), - &mut cache, + &mut cache_slot, ); assert_eq!(size, len as f64); let raw = dispatch_buffer_method(buf as usize, "export", std::ptr::null(), 0); let raw_addr = (raw.to_bits() & 0x0000_FFFF_FFFF_FFFF) as *const ObjectHeader; - let raw_len = js_object_get_field_ic_miss(raw_addr, key(b"length"), &mut cache); + let raw_len = + js_object_get_field_ic_miss(raw_addr, key(b"length"), &mut cache_slot); assert_eq!(raw_len, len as f64); } } diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index fdd60bce19..be3027adb6 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -239,6 +239,13 @@ pub const PIC_CACHE_WORDS: usize = 12; /// | 11 | round-robin victim index for the ways | pub type PicCache = [i64; PIC_CACHE_WORDS]; +/// The per-site slot codegen emits for a property-read cache — `@perry_ic_N = +/// private global ptr null` — holding null until the site's first priming +/// miss, then the arena cache `pic_slot_resolve` published (#9708). The +/// emitted hit path reads the cache words through this pointer; every runtime +/// entry that primes takes the slot's address and resolves it here. +pub type PicCacheSlot = *mut PicCache; + /// First word of the polymorphic way array. /// /// The ways start at 4, not 3, so that [`PIC_WAY_STATE`] can sit at word 3 — @@ -434,7 +441,7 @@ pub extern "C" fn js_object_get_field_ic_overflow_load( obj: *const ObjectHeader, key: *const crate::StringHeader, slot: i32, - cache: *mut PicCache, + cache_slot: *mut PicCacheSlot, ) -> f64 { let idx = (slot as u32 & !crate::proxy::IC_SLOT_OVERFLOW_BIT) as usize; if !obj.is_null() { @@ -444,7 +451,7 @@ pub extern "C" fn js_object_get_field_ic_overflow_load( } } } - js_object_get_field_ic_miss(obj, key, cache) + js_object_get_field_ic_miss(obj, key, cache_slot) } /// Monomorphic inline cache miss handler (issue #51). @@ -454,9 +461,13 @@ pub extern "C" fn js_object_get_field_ic_overflow_load( /// then populates the per-site cache so subsequent calls with the same shape /// hit the inline fast path (no function call, direct field load). /// -/// `cache` layout: see [`PicCache`]. Words 0..1 are the ShapeId-token MRU entry; -/// word 2 is reserved scratch, and words 3.. are the polymorphic ways filled by -/// [`pic_prime_get`] (#7753). +/// `cache_slot` is the address of the site's [`PicCacheSlot`]; the cache it +/// resolves to (allocated on the first priming miss, #9708) has the layout in +/// [`PicCache`]. Words 0..1 are the ShapeId-token MRU entry; word 2 is +/// reserved scratch, and words 3.. are the polymorphic ways filled by +/// [`pic_prime_get`] (#7753). A miss that cannot prime — SSO or proxy +/// receiver, a missing key, an accessor — never touches the slot, so a site +/// that only ever sees such receivers costs its 8-byte slot and nothing else. /// /// Only caches when: /// - obj is a valid ObjectHeader (not null, not handle, not string/array/etc.) @@ -469,7 +480,7 @@ pub extern "C" fn js_object_get_field_ic_overflow_load( pub extern "C" fn js_object_get_field_ic_miss( obj: *const ObjectHeader, key: *const crate::StringHeader, - cache: *mut PicCache, + cache_slot: *mut PicCacheSlot, ) -> f64 { // SSO receiver — never cacheable. Route through the SSO-aware // `js_object_get_field_by_name` which handles `.length` inline @@ -795,6 +806,7 @@ pub extern "C" fn js_object_get_field_ic_miss( // the prefix paths compute inline // addresses and must never fire from an // overflow-primed entry. + let cache = pic_slot_resolve(cache_slot); (*cache)[2] = 0; pic_prime_get( cache, @@ -844,6 +856,7 @@ pub extern "C" fn js_object_get_field_ic_miss( if has_own_descriptors && named_prefix_token == 0 { break; } + let cache = pic_slot_resolve(cache_slot); (*cache)[2] = named_prefix_token; pic_prime_get(cache, token, i as i64); let field_ptr = (obj as *const u8) @@ -877,13 +890,14 @@ pub extern "C" fn js_object_get_field_ic_miss( /// - `obj_bits`: the receiver's full (unmasked) NaN-box bits /// - `key`: the property-name `StringHeader`, already masked to a raw pointer /// - `site_id`: the typed-feedback site id -/// - `cache`: the per-site monomorphic IC cache global (primed by `..._ic_miss`) +/// - `cache_slot`: the per-site [`PicCacheSlot`] (resolved and primed by +/// `..._ic_miss`) #[no_mangle] pub extern "C" fn js_object_get_field_ic( obj_bits: i64, key: *const crate::StringHeader, site_id: u64, - cache: *mut PicCache, + cache_slot: *mut PicCacheSlot, ) -> f64 { // POINTER_MASK: lower 48 bits — strips the NaN-box tag to a raw heap pointer. const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; @@ -917,7 +931,7 @@ pub extern "C" fn js_object_get_field_ic( // is primed for any future inline sites sharing this global). if (tag & 0xFFFD) == 0x7FFD { crate::typed_feedback::js_typed_feedback_observe_property_get(site_id, obj_handle, key); - return js_object_get_field_ic_miss(obj_handle, key, cache); + return js_object_get_field_ic_miss(obj_handle, key, cache_slot); } // Invalid (non-pointer) receiver. `undefined`/`null` throw a TypeError (#462 — // matches the inline nullish path, which aborts with a node-shaped message); diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/c3c_pic_tests.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/c3c_pic_tests.rs index 05efbcfcbe..605e09471d 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/c3c_pic_tests.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/c3c_pic_tests.rs @@ -31,9 +31,11 @@ fn unrelated_accessor_does_not_poison_plain_receiver_pic() { }); let mut cache = [0i64; super::PIC_CACHE_WORDS]; + + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; assert_eq!( obj.with_mut_ptr( - |o| key.with_const_ptr(|k| super::js_object_get_field_ic_miss(o, k, &mut cache)) + |o| key.with_const_ptr(|k| super::js_object_get_field_ic_miss(o, k, &mut cache_slot)) ), 42.0 ); @@ -66,8 +68,10 @@ fn accessor_bearing_receiver_does_not_prime_plain_data_pic() { ); let mut cache = [0i64; super::PIC_CACHE_WORDS]; + + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; let via_pic = obj.with_mut_ptr(|o| { - key.with_const_ptr(|k| super::js_object_get_field_ic_miss(o, k, &mut cache)) + key.with_const_ptr(|k| super::js_object_get_field_ic_miss(o, k, &mut cache_slot)) }); let via_ladder = obj.with_mut_ptr(|o| key.with_const_ptr(|k| super::js_object_get_field_by_name_f64(o, k))); @@ -96,7 +100,9 @@ fn a_class_instance_primes_an_id_token_after_rung1() { assert_eq!((*obj).class_id, 0x6080, "test premise: a class instance"); let mut cache = [0i64; super::PIC_CACHE_WORDS]; - let v = super::js_object_get_field_ic_miss(obj, key, &mut cache); + + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; + let v = super::js_object_get_field_ic_miss(obj, key, &mut cache_slot); assert_eq!(v, 7.0); let stamp = crate::object::shapes::object_shape_stamp(obj); @@ -169,11 +175,17 @@ fn a_compacted_class_instance_primes_a_token_a_pristine_sibling_cannot_match() { ); let mut c_pristine = [0i64; super::PIC_CACHE_WORDS]; - let vp = super::js_object_get_field_ic_miss(pristine, key("picdel_c"), &mut c_pristine); + + let mut c_pristine_slot: crate::object::PicCacheSlot = &mut c_pristine; + let vp = + super::js_object_get_field_ic_miss(pristine, key("picdel_c"), &mut c_pristine_slot); assert_eq!(vp, 3.0, "pristine `c` is slot 2"); let mut c_compacted = [0i64; super::PIC_CACHE_WORDS]; - let vc = super::js_object_get_field_ic_miss(compacted, key("picdel_c"), &mut c_compacted); + + let mut c_compacted_slot: crate::object::PicCacheSlot = &mut c_compacted; + let vc = + super::js_object_get_field_ic_miss(compacted, key("picdel_c"), &mut c_compacted_slot); assert_eq!( vc, 3.0, "compacted `c` shifted to slot 1 and must still read 3" @@ -254,8 +266,10 @@ fn a_fresh_class_instance_computes_the_token_the_miss_handler_primed() { ); let mut cache = [0i64; super::PIC_CACHE_WORDS]; + + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; assert_eq!( - super::js_object_get_field_ic_miss(primed_from, key, &mut cache), + super::js_object_get_field_ic_miss(primed_from, key, &mut cache_slot), 5.0, "test premise: the miss handler resolved the field" ); @@ -279,7 +293,8 @@ fn a_fresh_class_instance_computes_the_token_the_miss_handler_primed() { // And the same must hold once the fresh one has itself resolved: // priming from either instance is interchangeable. let mut cache2 = [0i64; super::PIC_CACHE_WORDS]; - super::js_object_get_field_ic_miss(fresh, key, &mut cache2); + let mut cache2_slot: crate::object::PicCacheSlot = &mut cache2; + super::js_object_get_field_ic_miss(fresh, key, &mut cache2_slot); assert_eq!( cache2[0], cache[0], "two instances of one class primed two different tokens — the \ diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss_array_length_tests.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss_array_length_tests.rs index 24156b1ab8..271a8df9c2 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss_array_length_tests.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss_array_length_tests.rs @@ -21,7 +21,8 @@ fn array_length_short_circuit_agrees_with_the_full_ladder() { } let obj = arr as *const super::ObjectHeader; let mut cache = [0i64; super::PIC_CACHE_WORDS]; - let via_ic = super::js_object_get_field_ic_miss(obj, len_key, &mut cache); + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; + let via_ic = super::js_object_get_field_ic_miss(obj, len_key, &mut cache_slot); let via_ladder = super::js_object_get_field_by_name_f64(obj, len_key); assert_eq!( via_ic.to_bits(), @@ -32,7 +33,7 @@ fn array_length_short_circuit_agrees_with_the_full_ladder() { // A same-length key that is not `length` must not be captured // by the fast path. assert_eq!( - super::js_object_get_field_ic_miss(obj, other_key, &mut cache).to_bits(), + super::js_object_get_field_ic_miss(obj, other_key, &mut cache_slot).to_bits(), super::js_object_get_field_by_name_f64(obj, other_key).to_bits(), "a non-`length` key on an array must take the normal path" ); @@ -41,8 +42,9 @@ fn array_length_short_circuit_agrees_with_the_full_ladder() { // short-circuit — it is an ordinary (absent) property there. let plain = crate::object::js_object_alloc(0, 0); let mut cache = [0i64; super::PIC_CACHE_WORDS]; + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; assert_eq!( - super::js_object_get_field_ic_miss(plain, len_key, &mut cache).to_bits(), + super::js_object_get_field_ic_miss(plain, len_key, &mut cache_slot).to_bits(), super::js_object_get_field_by_name_f64(plain, len_key).to_bits(), "`length` on a plain object must keep its normal answer" ); @@ -70,7 +72,8 @@ fn array_subclass_length_short_circuit_preserves_object_semantics() { let len_key = crate::string::js_string_from_bytes(b"length".as_ptr(), 6); let mut cache = [0i64; super::PIC_CACHE_WORDS]; - let via_ic = super::js_object_get_field_ic_miss(obj, len_key, &mut cache); + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; + let via_ic = super::js_object_get_field_ic_miss(obj, len_key, &mut cache_slot); let via_ladder = super::js_object_get_field_by_name_f64(obj, len_key); assert_eq!(via_ic.to_bits(), via_ladder.to_bits()); assert_eq!(via_ic, 3.0, "the fast path must observe the live length"); @@ -78,8 +81,9 @@ fn array_subclass_length_short_circuit_preserves_object_semantics() { let plain = crate::object::js_object_alloc(0, 1); crate::object::js_object_set_field_by_name(plain, len_key, 123.0); let mut plain_cache = [0i64; super::PIC_CACHE_WORDS]; + let mut plain_cache_slot: crate::object::PicCacheSlot = &mut plain_cache; assert_eq!( - super::js_object_get_field_ic_miss(plain, len_key, &mut plain_cache), + super::js_object_get_field_ic_miss(plain, len_key, &mut plain_cache_slot), 123.0, "ordinary objects must retain their own `length` property semantics" ); diff --git a/crates/perry-runtime/src/object/field_get_set/ic_slot.rs b/crates/perry-runtime/src/object/field_get_set/ic_slot.rs new file mode 100644 index 0000000000..040e2e3b6a --- /dev/null +++ b/crates/perry-runtime/src/object/field_get_set/ic_slot.rs @@ -0,0 +1,281 @@ +//! #9708: per-site inline caches allocated on first miss, not per emitted site. +//! +//! Codegen used to emit every inline-cache site as its own +//! `[PIC_CACHE_WORDS x i64] zeroinitializer` global — 96 B per property-access +//! site, whether or not the program ever executes it. On a large bundle that +//! is hundreds of thousands of sites and tens of megabytes of `__bss` that +//! turn into dirty resident pages as soon as a neighbour on the same page is +//! touched: the `cc` build carried 262k caches (25 MB) and dirtied 18.7 MB of +//! them at idle while running a few thousand distinct hot sites. +//! +//! Each site now owns an 8-byte **slot** — `@perry_ic_N = private global ptr +//! null` — and the cache words live in a runtime-owned arena, allocated the +//! first time the site's miss handler runs. A site that never executes costs +//! its 8 bytes of zero-fill and nothing else; a site that does costs the same +//! `PIC_CACHE_WORDS` words it always did, packed next to the other caches +//! that were touched around the same time. +//! +//! The emitted hit path loads the slot, folds `!= null` into the receiver +//! guard it already evaluates, and reads the cache words through the loaded +//! pointer; the miss path hands the *slot* to the runtime, which resolves or +//! allocates the cache here. Only the words the runtime writes ever live in +//! the arena, so a cache's layout and every prime/evict policy are unchanged. +//! +//! Publication is a compare-and-swap on the slot so two `perry/thread` agents +//! missing the same site for the first time agree on one cache; the loser's +//! block stays in the arena unused (bounded by the number of such races, and +//! counted). The arena is never freed: the sites reference their caches for +//! the life of the process, exactly as the globals were. + +use std::alloc::{alloc_zeroed, handle_alloc_error, Layout}; +use std::ptr::null_mut; +use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; +use std::sync::Mutex; + +/// Bytes requested from the system allocator per arena refill. Holds 682 +/// twelve-word caches; a program that primes fewer sites than that pays one +/// zeroed allocation in total. +const PIC_ARENA_CHUNK_BYTES: usize = 64 * 1024; + +/// Every cache starts on a 32-byte boundary, so the four words the MRU hit +/// path and the way-state gate read (`tok0`, `slot0`, scratch, state) never +/// straddle a 64-byte line — the property the pre-#9708 layout relied on +/// when it parked the gate at word 3 (see `PIC_WAY_BASE`'s doc). +const PIC_CACHE_ALIGN: usize = 32; + +/// Bump-pointer state over the current chunk. Addresses are kept as `usize` +/// on purpose: the arena hands out cache words (tokens and slot indices), it +/// never holds a GC heap pointer, and `usize` keeps the static `Send` without +/// an `unsafe impl`. +struct PicArena { + cur: usize, + end: usize, +} + +static PIC_ARENA: Mutex = Mutex::new(PicArena { cur: 0, end: 0 }); + +/// Caches published into a slot (one per site that has missed at least once). +static PIC_SLOTS_RESOLVED: AtomicUsize = AtomicUsize::new(0); +/// Bytes requested from the system allocator for arena chunks. +static PIC_ARENA_BYTES: AtomicUsize = AtomicUsize::new(0); +/// Allocations that lost the publication race to another thread. +static PIC_SLOT_RACES: AtomicUsize = AtomicUsize::new(0); + +/// Bump-allocate `bytes` of zeroed, `PIC_CACHE_ALIGN`-aligned memory. +fn pic_arena_alloc(bytes: usize) -> *mut u8 { + let bytes = bytes.div_ceil(PIC_CACHE_ALIGN) * PIC_CACHE_ALIGN; + debug_assert!(bytes <= PIC_ARENA_CHUNK_BYTES); + let mut arena = PIC_ARENA.lock().unwrap_or_else(|e| e.into_inner()); + if arena.cur + bytes > arena.end { + // SAFETY: the layout is non-zero-sized and its alignment is a power of + // two; a zeroed chunk is exactly the initial state every cache expects. + let layout = Layout::from_size_align(PIC_ARENA_CHUNK_BYTES, 64) + .expect("PIC arena chunk layout is a constant"); + let chunk = unsafe { alloc_zeroed(layout) }; + if chunk.is_null() { + handle_alloc_error(layout); + } + PIC_ARENA_BYTES.fetch_add(PIC_ARENA_CHUNK_BYTES, Ordering::Relaxed); + arena.cur = chunk as usize; + arena.end = arena.cur + PIC_ARENA_CHUNK_BYTES; + } + let out = arena.cur; + arena.cur += bytes; + out as *mut u8 +} + +/// Resolve a per-site cache slot to its cache, allocating a zeroed one and +/// publishing it into the slot on the first miss. +/// +/// `slot` is the address of the emitted `@perry_ic_N` pointer global (or a +/// stack `*mut T` in tests); `T` is the cache's word array — `PicCache` for a +/// property read, the eight-word write cache, the four-word Symbol cache — +/// and decides how many bytes the arena hands out. A null `slot` resolves to +/// a null cache: the write PIC's poly tail passes that to mean "run `[[Set]]` +/// and prime nothing" and every consumer already checks for it. +/// +/// # Safety +/// `slot` must be null or point at a live, 8-byte-aligned pointer-sized +/// location that holds either null or a cache previously returned by this +/// function (or any live `T`). All-zero bytes must be a valid `T`. +#[inline] +pub unsafe fn pic_slot_resolve(slot: *mut *mut T) -> *mut T { + if slot.is_null() { + return null_mut(); + } + let atomic = &*(slot as *const AtomicPtr); + let cur = atomic.load(Ordering::Acquire); + if !cur.is_null() { + return cur; + } + pic_slot_publish(atomic) +} + +/// The cache `slot` currently holds, without allocating: null for a site +/// that has never primed. Runtime entries that only *read* a site's cache +/// (the outlined write IC's hit check, a Symbol miss's epoch reset) go +/// through here so a concurrent first prime on another thread is an atomic +/// publication rather than a torn read. +/// +/// # Safety +/// `slot` must be null or a live cache slot (see [`pic_slot_resolve`]). +#[inline] +pub unsafe fn pic_slot_peek(slot: *mut *mut T) -> *mut T { + if slot.is_null() { + return null_mut(); + } + (*(slot as *const AtomicPtr)).load(Ordering::Acquire) +} + +#[cold] +#[inline(never)] +unsafe fn pic_slot_publish(atomic: &AtomicPtr) -> *mut T { + let fresh = pic_arena_alloc(std::mem::size_of::()) as *mut T; + match atomic.compare_exchange(null_mut(), fresh, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => { + PIC_SLOTS_RESOLVED.fetch_add(1, Ordering::Relaxed); + fresh + } + Err(existing) => { + PIC_SLOT_RACES.fetch_add(1, Ordering::Relaxed); + existing + } + } +} + +/// Number of sites whose slot holds an arena cache. +pub fn pic_slots_resolved() -> usize { + PIC_SLOTS_RESOLVED.load(Ordering::Relaxed) +} + +/// Bytes the arena has requested from the system allocator. +pub fn pic_arena_bytes() -> usize { + PIC_ARENA_BYTES.load(Ordering::Relaxed) +} + +/// `PERRY_GC_CENSUS`: the lazily allocated inline caches as one side-table +/// row — `entries` is the number of resolved sites, `bytes` the arena's +/// footprint (chunk granularity, so it over-reports by at most one chunk). +pub(crate) fn pic_slot_census() -> Vec { + vec![("ic.lazy_caches", pic_slots_resolved(), pic_arena_bytes())] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn null_slot_resolves_to_null_cache() { + let cache = unsafe { pic_slot_resolve::<[i64; 12]>(null_mut()) }; + assert!(cache.is_null()); + } + + #[test] + fn first_resolve_allocates_a_zeroed_cache_and_publishes_it() { + let mut slot: *mut [i64; 12] = null_mut(); + let before = pic_slots_resolved(); + let cache = unsafe { pic_slot_resolve(&mut slot) }; + assert!(!cache.is_null(), "a fresh slot must resolve to a cache"); + assert_eq!(slot, cache, "the cache must be published into the slot"); + assert_eq!( + cache as usize % PIC_CACHE_ALIGN, + 0, + "caches are 32-byte aligned" + ); + assert!( + unsafe { (*cache).iter().all(|w| *w == 0) }, + "a fresh cache is all zeros" + ); + assert!( + pic_slots_resolved() > before, + "the census counter must observe the allocation" + ); + } + + #[test] + fn resolving_a_populated_slot_returns_the_same_cache_without_allocating() { + let mut slot: *mut [i64; 12] = null_mut(); + let first = unsafe { pic_slot_resolve(&mut slot) }; + unsafe { (*first)[0] = 0x4000_0000_0000_0001 }; + let resolved_before = pic_slots_resolved(); + let again = unsafe { pic_slot_resolve(&mut slot) }; + assert_eq!(first, again); + assert_eq!( + unsafe { (*again)[0] }, + 0x4000_0000_0000_0001, + "primed words survive" + ); + assert_eq!( + pic_slots_resolved(), + resolved_before, + "a second resolve of the same slot allocates nothing" + ); + } + + #[test] + fn peek_never_allocates() { + let mut slot: *mut [i64; 12] = null_mut(); + let before = pic_slots_resolved(); + assert!(unsafe { pic_slot_peek(&mut slot) }.is_null()); + assert!(slot.is_null(), "peek must not publish anything"); + assert_eq!(pic_slots_resolved(), before); + let cache = unsafe { pic_slot_resolve(&mut slot) }; + assert_eq!(unsafe { pic_slot_peek(&mut slot) }, cache); + assert!(unsafe { pic_slot_peek::<[i64; 12]>(null_mut()) }.is_null()); + } + + #[test] + fn a_stack_cache_pre_seeded_into_a_slot_is_honoured() { + let mut cache = [7i64; 8]; + let mut slot: *mut [i64; 8] = &mut cache; + let resolved = unsafe { pic_slot_resolve(&mut slot) }; + assert_eq!(resolved, &mut cache as *mut [i64; 8]); + } + + #[test] + fn consecutive_caches_are_packed_and_distinct() { + let mut a: *mut [i64; 12] = null_mut(); + let mut b: *mut [i64; 12] = null_mut(); + let ca = unsafe { pic_slot_resolve(&mut a) }; + let cb = unsafe { pic_slot_resolve(&mut b) }; + assert_ne!(ca, cb); + // Other tests allocate concurrently, so only the lower bound is exact: + // two caches can never overlap. + let (lo, hi) = if (ca as usize) < (cb as usize) { + (ca, cb) + } else { + (cb, ca) + }; + assert!(hi as usize - lo as usize >= 96, "caches must not overlap"); + } + + #[test] + fn eight_word_caches_fit_the_same_arena() { + let mut slot: *mut [i64; 8] = null_mut(); + let cache = unsafe { pic_slot_resolve(&mut slot) }; + assert!(!cache.is_null()); + assert_eq!(cache as usize % PIC_CACHE_ALIGN, 0); + unsafe { (*cache)[7] = 1 }; + assert_eq!(unsafe { (*slot)[7] }, 1); + } + + #[test] + fn concurrent_first_misses_agree_on_one_cache() { + use std::sync::atomic::AtomicUsize as Shared; + static SLOT: Shared = Shared::new(0); + let slot_addr = &SLOT as *const Shared as usize; + let handles: Vec<_> = (0..8) + .map(|_| { + std::thread::spawn(move || unsafe { + pic_slot_resolve(slot_addr as *mut *mut [i64; 12]) as usize + }) + }) + .collect(); + let results: Vec = handles.into_iter().map(|h| h.join().unwrap()).collect(); + assert!( + results.iter().all(|r| *r == results[0]), + "every thread must see one cache" + ); + assert_eq!(SLOT.load(Ordering::Relaxed), results[0]); + } +} diff --git a/crates/perry-runtime/src/object/with_env.rs b/crates/perry-runtime/src/object/with_env.rs index a7e2f03396..175765f3aa 100644 --- a/crates/perry-runtime/src/object/with_env.rs +++ b/crates/perry-runtime/src/object/with_env.rs @@ -101,7 +101,7 @@ pub extern "C" fn js_with_set_binding( if strict != 0 && !has_property(coerced, key) { crate::error::js_throw_reference_error_unresolvable_assignment(key_as_value(key)); } - crate::proxy::js_put_value_set_ic_miss(coerced, key, value, strict, std::ptr::null_mut()) + crate::proxy::js_put_value_set_ic_miss(coerced, key, value, strict, std::ptr::null_mut(), 0) } #[no_mangle] diff --git a/crates/perry-runtime/src/proxy.rs b/crates/perry-runtime/src/proxy.rs index a4e8213aeb..1975eedc69 100644 --- a/crates/perry-runtime/src/proxy.rs +++ b/crates/perry-runtime/src/proxy.rs @@ -34,6 +34,7 @@ pub use put_value::{js_proxy_set, js_put_value_set}; pub(crate) use put_value::{ js_put_value_set_ic_miss, proxy_set_with_receiver, IC_SLOT_OVERFLOW_BIT, }; +pub use put_value::{write_pic_way_entry, WritePicCache, WritePicCacheSlot, WRITE_PIC_WORDS}; mod json; mod metadata; mod own_keys; @@ -3280,8 +3281,10 @@ mod tests { let key_ptr = crate::string::js_string_from_bytes(b"n".as_ptr(), 1); let key_ptr = crate::string::js_string_intern(key_ptr, fnv1a(b"n")); - let mut cache = [0i64; 2]; - let stored = put_value::js_put_value_set_ic_miss(target, key_ptr, 7.0, 0, &mut cache); + let mut cache: put_value::WritePicCache = [0; put_value::WRITE_PIC_WORDS]; + let mut cache_slot: put_value::WritePicCacheSlot = &mut cache; + let stored = + put_value::js_put_value_set_ic_miss(target, key_ptr, 7.0, 0, &mut cache_slot, 0); assert_eq!(stored, 7.0); let expected_token = unsafe { crate::object::shapes::PIC_ID_TOKEN_BIT @@ -3303,13 +3306,15 @@ mod tests { (unmarked as *mut u8).sub(crate::gc::GC_HEADER_SIZE) as *mut crate::gc::GcHeader; (*header)._reserved &= !crate::gc::OBJ_FLAG_PLAIN_ORDINARY; } - let mut cache2 = [0i64; 2]; + let mut cache2: put_value::WritePicCache = [0; put_value::WRITE_PIC_WORDS]; + let mut cache2_slot: put_value::WritePicCacheSlot = &mut cache2; let stored = put_value::js_put_value_set_ic_miss( f64::from_bits(value.bits()), key_ptr, 9.0, 0, - &mut cache2, + &mut cache2_slot, + 0, ); assert_eq!(stored, 9.0, "the write itself still succeeds"); assert_eq!( diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index c41a52232b..1f4e8acdbb 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -328,13 +328,56 @@ pub extern "C" fn js_put_value_set( } value_handle.get_nanbox_f64() } + +/// Words in a per-site static-key write cache (`[shape_token, slot]` × the +/// four inline ways) and in its outlined poly tail. Both are views of the +/// same `PicCacheSlot` family the read PIC uses; the arena hands out exactly +/// this many words for them (#9708). +pub const WRITE_PIC_WORDS: usize = 8; + +/// A per-site write cache, as the emitted slot resolves it. +pub type WritePicCache = [i64; WRITE_PIC_WORDS]; +/// The emitted `@perry_ic_N = private global ptr null` for a write site: +/// null until the site's first priming miss (#9708). +pub type WritePicCacheSlot = *mut WritePicCache; + +/// Address of way `way` (a `[token, slot]` pair) inside the cache `slot` +/// resolves to, allocating the cache on first use. A null `slot` yields null: +/// the poly tail passes that to mean "prime nothing". +/// +/// # Safety +/// `slot` must be null or a live write-cache slot; `way` must index a pair +/// inside `WRITE_PIC_WORDS`. +#[inline] +pub unsafe fn write_pic_way_entry(slot: *mut WritePicCacheSlot, way: i32) -> *mut [i64; 2] { + if slot.is_null() { + return std::ptr::null_mut(); + } + debug_assert!( + (0..(WRITE_PIC_WORDS / 2) as i32).contains(&way), + "way {way} is outside the {WRITE_PIC_WORDS}-word write cache" + ); + let cache = crate::object::pic_slot_resolve(slot); + (cache as *mut i64).add(way as usize * 2) as *mut [i64; 2] +} + +/// Miss path for one way of the codegen-emitted polymorphic PutValue cache. +/// +/// The full strict/sloppy `[[Set]]` semantics run first. Only a successful +/// ordinary own-data overwrite may prime `[shape_token, slot]`; every +/// exotic, descriptor-bearing, frozen, or typed-layout-intact receiver +/// remains on the miss path. `cache_slot` is the site's +/// [`WritePicCacheSlot`] address and `way` the pair to prime; the cache is +/// allocated only when a prime actually happens (#9708), so a site that never +/// sees a primeable receiver never allocates. #[no_mangle] pub extern "C" fn js_put_value_set_ic_miss( target: f64, key: *const crate::StringHeader, value: f64, strict: i32, - cache: *mut [i64; 2], + cache_slot: *mut WritePicCacheSlot, + way: i32, ) -> f64 { let scope = crate::gc::RuntimeHandleScope::new(); let target_handle = scope.root_nanbox_f64(target); @@ -357,7 +400,7 @@ pub extern "C" fn js_put_value_set_ic_miss( ) }); - if cache.is_null() { + if cache_slot.is_null() { return result; } @@ -464,8 +507,9 @@ pub extern "C" fn js_put_value_set_ic_miss( // Publish the token last conceptually: a zero-initialized or stale // token cannot hit this slot until it matches this receiver's current // discriminated shape token. Perry's read PIC uses the same format. - (*cache)[1] = slot_word as i64; - (*cache)[0] = shape_token as i64; + let entry = write_pic_way_entry(cache_slot, way); + (*entry)[1] = slot_word as i64; + (*entry)[0] = shape_token as i64; // Rotating-key sites overflow the single-slot site cache immediately; // the global stub is what lets them hit. Key bits from the rooted key. // `key_handle` here roots a raw STRING pointer (this entry's key arrives @@ -497,23 +541,33 @@ const STATIC_PIC_TAIL_WAYS: usize = 4; /// semantics. Empty ways are filled in order and a full cache is never /// overwritten, so a stable eight-shape site settles instead of continuously /// replacing its fourth entry. +/// +/// `tail_slot` is the tail's own [`WritePicCacheSlot`]: a site reaches this +/// helper only once its four inline ways are full, so the tail cache is not +/// allocated before a fifth shape actually shows up (#9708). An unallocated +/// tail reads as four empty ways. #[no_mangle] pub extern "C" fn js_put_value_set_ic_poly_tail( - cache: *mut [i64; 8], + tail_slot: *mut WritePicCacheSlot, target: f64, key: *const crate::StringHeader, value: f64, strict: i32, ) -> f64 { - if !cache.is_null() { + if !tail_slot.is_null() { unsafe { + let cache = crate::object::pic_slot_peek(tail_slot); + if cache.is_null() { + return js_put_value_set_ic_miss(target, key, value, strict, tail_slot, 0); + } let c = &mut *cache; for way in 0..STATIC_PIC_TAIL_WAYS { let word = way * 2; let token = c[word] as u64; if token == 0 { - let entry = c.as_mut_ptr().add(word) as *mut [i64; 2]; - return js_put_value_set_ic_miss(target, key, value, strict, entry); + return js_put_value_set_ic_miss( + target, key, value, strict, tail_slot, way as i32, + ); } if let Some(result) = dyn_ic_try_store(target, token, c[word + 1] as u32, value) { return result; @@ -524,7 +578,7 @@ pub extern "C" fn js_put_value_set_ic_poly_tail( // More than eight stable shapes remain bounded and semantically correct: // execute the ordinary write without evicting a useful settled entry. - js_put_value_set_ic_miss(target, key, value, strict, std::ptr::null_mut()) + js_put_value_set_ic_miss(target, key, value, strict, std::ptr::null_mut(), 0) } // --------------------------------------------------------------------------- @@ -547,14 +601,21 @@ const DYN_IC_WAYS: usize = 3; /// Outlined dynamic-key PutValue with per-site cache. Fast path: shape token /// + key-bits match -> validated own-slot overwrite. Everything else falls /// through to the full `[[Set]]` semantics and re-primes. +/// +/// `cache_slot` is the site's [`WritePicCacheSlot`] address; the cache is read +/// through it here and allocated by the miss handler on the first prime +/// (#9708). #[no_mangle] pub extern "C" fn js_put_value_set_dyn_ic( - cache: *mut [i64; 8], + cache_slot: *mut WritePicCacheSlot, target: f64, key: f64, value: f64, strict: i32, ) -> f64 { + // SAFETY: a non-null slot is the emitted pointer global (or a test's + // stack slot); reading it is the same load the inline hit path performs. + let cache = unsafe { crate::object::pic_slot_peek(cache_slot) }; if !cache.is_null() { let hit = unsafe { let c = &*cache; @@ -585,7 +646,7 @@ pub extern "C" fn js_put_value_set_dyn_ic( return ret; } } - js_put_value_set_dyn_ic_miss(cache, target, key, value, strict) + js_put_value_set_dyn_ic_miss(cache_slot, target, key, value, strict) } /// Megamorphic stub cache for dynamic string-keyed WRITES — V8's answer to a @@ -832,8 +893,13 @@ unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Op // link would otherwise dead-strip the IC entry. #[cfg(feature = "keepalive-anchors")] #[used] -static KEEP_JS_PUT_VALUE_SET_DYN_IC: extern "C" fn(*mut [i64; 8], f64, f64, f64, i32) -> f64 = - js_put_value_set_dyn_ic; +static KEEP_JS_PUT_VALUE_SET_DYN_IC: extern "C" fn( + *mut WritePicCacheSlot, + f64, + f64, + f64, + i32, +) -> f64 = js_put_value_set_dyn_ic; /// #9287 transition-IC: the emitted hit's SPILL-APPEND arm. /// @@ -923,12 +989,16 @@ extern "C" fn transition_ic_report_shim() { /// its VALUE (SSO or heap) instead of requiring an interned pointer. #[no_mangle] pub extern "C" fn js_put_value_set_dyn_ic_miss( - cache: *mut [i64; 8], + cache_slot: *mut WritePicCacheSlot, target: f64, key: f64, value: f64, strict: i32, ) -> f64 { + // The slot is read once here; a null cache means the site has never + // primed, which the stub probe below treats as "no site token" exactly as + // it treated an all-zero global (#9708). + let cache = unsafe { crate::object::pic_slot_peek(cache_slot) }; // The compiled inline IC (`lower_put_value_dyn_ic_inline`) walks its three // ways in GENERATED code and calls straight here on a way miss — it never // enters `js_put_value_set_dyn_ic` above. A rotating-key site therefore @@ -1026,7 +1096,7 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( target_handle.get_nanbox_f64(), strict, ); - if cache.is_null() { + if cache_slot.is_null() { return result; } unsafe { @@ -1121,7 +1191,7 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( if idx >= alloc_limit { return result; } - let c = &mut *cache; + let c = &mut *crate::object::pic_slot_resolve(cache_slot); if c[0] as u64 != shape_token { // New shape at this site: restart the way set. *c = [0; 8]; @@ -1173,13 +1243,14 @@ mod tests { scope.root_raw_mut_ptr(crate::object::js_object_alloc(ANON_CLASS_ID, 0)); let user_class_handle = scope.root_raw_mut_ptr(crate::object::js_object_alloc(USER_CLASS_ID, 0)); - let mut cache = [0i64; 8]; + let mut cache: WritePicCache = [0; WRITE_PIC_WORDS]; + let mut cache_slot: WritePicCacheSlot = &mut cache; let first = first_handle.get_raw_mut_ptr::(); let learned_predecessor = unsafe { crate::object::shapes::object_shape_stamp(first) }; assert_eq!( js_put_value_set_dyn_ic_miss( - &mut cache, + &mut cache_slot, crate::value::js_nanbox_pointer(first as i64), key_handle.get_nanbox_f64(), 11.0, @@ -1209,7 +1280,7 @@ mod tests { ); assert_eq!( js_put_value_set_dyn_ic_miss( - &mut cache, + &mut cache_slot, crate::value::js_nanbox_pointer(second as i64), key_handle.get_nanbox_f64(), 29.0, @@ -1238,7 +1309,7 @@ mod tests { let user_class = user_class_handle.get_raw_mut_ptr::(); assert_eq!( js_put_value_set_dyn_ic_miss( - &mut cache, + &mut cache_slot, crate::value::js_nanbox_pointer(user_class as i64), key_handle.get_nanbox_f64(), 47.0, diff --git a/crates/perry-runtime/src/symbol.rs b/crates/perry-runtime/src/symbol.rs index c7ff66a031..3abb55867f 100644 --- a/crates/perry-runtime/src/symbol.rs +++ b/crates/perry-runtime/src/symbol.rs @@ -58,7 +58,7 @@ pub use properties::{ pub(crate) use get::{has_own_symbol_property, inherited_symbol_property, own_symbol_property}; pub use get::{ js_object_get_symbol_property, js_object_get_symbol_property_ic_miss, - js_object_get_symbol_then_field_ic_miss, + js_object_get_symbol_then_field_ic_miss, SymbolPicCache, SymbolPicCacheSlot, SYMBOL_PIC_WORDS, }; // Iterator protocol, getOwnPropertySymbols, ToPrimitive. diff --git a/crates/perry-runtime/src/symbol/get.rs b/crates/perry-runtime/src/symbol/get.rs index ba6aa95351..503d041ccf 100644 --- a/crates/perry-runtime/src/symbol/get.rs +++ b/crates/perry-runtime/src/symbol/get.rs @@ -88,6 +88,26 @@ pub(crate) unsafe fn own_symbol_property(obj_f64: f64, sym_f64: f64) -> Option *mut u64 { + crate::object::pic_slot_peek(slot) as *mut u64 +} + /// Miss path for the generated weak own-Symbol-property IC. /// /// Cache layout (all `u64`): epoch, receiver bits, symbol bits, value bits. @@ -99,14 +119,17 @@ pub(crate) unsafe fn own_symbol_property(obj_f64: f64, sym_f64: f64) -> Option f64 { let obj_bits = obj_f64.to_bits(); - if !cache.is_null() && (obj_bits >> 48) == 0x7FFD { + if !cache_slot.is_null() && (obj_bits >> 48) == 0x7FFD { let obj_key = (obj_bits & POINTER_MASK) as usize; if let Some(header) = crate::value::addr_class::try_read_gc_header(obj_key) { if header.obj_type == crate::gc::GC_TYPE_OBJECT { @@ -128,6 +151,10 @@ pub unsafe extern "C" fn js_object_get_symbol_property_ic_miss( if let Some(value_bits) = value_bits { let epoch = PERRY_SYMBOL_PROPERTY_IC_EPOCH .load(std::sync::atomic::Ordering::Acquire); + // #9708: the cache is allocated here, on the first + // successful prime, never for a miss that cannot + // prime. + let cache = crate::object::pic_slot_resolve(cache_slot) as *mut u64; // Publish identity/value first and epoch last. The // generated hit path acquire-loads this first word. *cache.add(1) = obj_bits; @@ -160,9 +187,12 @@ pub unsafe extern "C" fn js_object_get_symbol_then_field_ic_miss( sym_f64: f64, key: *const crate::StringHeader, feedback_site_id: u64, - symbol_cache: *mut u64, - field_cache: *mut crate::object::PicCache, + symbol_cache_slot: *mut SymbolPicCacheSlot, + field_cache_slot: *mut crate::object::PicCacheSlot, ) -> f64 { + // #9708: both caches sit behind slots. Each is re-read after the miss + // that may allocate it; before that, a null cache is simply "not primed". + let symbol_cache = symbol_cache_of(symbol_cache_slot); if key.is_null() { if !symbol_cache.is_null() { (&*(symbol_cache as *const std::sync::atomic::AtomicU64)) @@ -175,8 +205,10 @@ pub unsafe extern "C" fn js_object_get_symbol_then_field_ic_miss( // whose accessor/prototype fallback may allocate before the named read. let scope = crate::gc::RuntimeHandleScope::new(); let key_handle = scope.root_nanbox_f64(crate::value::js_nanbox_string(key as i64)); - let intermediate = js_object_get_symbol_property_ic_miss(obj_f64, sym_f64, symbol_cache); + let intermediate = js_object_get_symbol_property_ic_miss(obj_f64, sym_f64, symbol_cache_slot); let intermediate_handle = scope.root_nanbox_f64(intermediate); + let symbol_cache = symbol_cache_of(symbol_cache_slot); + let field_cache = crate::object::pic_slot_peek(field_cache_slot); // Do not mistake an entry retained from the previously cached // intermediate object for a prime performed by this miss. @@ -192,9 +224,11 @@ pub unsafe extern "C" fn js_object_get_symbol_then_field_ic_miss( current.to_bits() as i64, key_now, feedback_site_id, - field_cache, + field_cache_slot, ) }); + // The named read may have been the site's first prime, which allocates. + let field_cache = crate::object::pic_slot_peek(field_cache_slot); // `cache[3]` is dereferenced by generated code only after cache[0]'s // acquire-load succeeds. Leave that epoch published solely when both miss @@ -1352,8 +1386,9 @@ mod own_data_ic_tests { let first = 41.0_f64; super::properties::js_object_set_symbol_property(obj, sym, first); - let mut cache = [0_u64; 12]; - let got = js_object_get_symbol_property_ic_miss(obj, sym, cache.as_mut_ptr()); + let mut cache: SymbolPicCache = [0; SYMBOL_PIC_WORDS]; + let mut cache_slot: SymbolPicCacheSlot = &mut cache; + let got = js_object_get_symbol_property_ic_miss(obj, sym, &mut cache_slot); assert_eq!(got.to_bits(), first.to_bits()); assert_eq!(cache[1], obj.to_bits()); assert_eq!(cache[2], sym.to_bits()); @@ -1371,7 +1406,7 @@ mod own_data_ic_tests { PERRY_SYMBOL_PROPERTY_IC_EPOCH.load(Ordering::Acquire), "a Symbol data write must make the generated hit guard fail" ); - let got = js_object_get_symbol_property_ic_miss(obj, sym, cache.as_mut_ptr()); + let got = js_object_get_symbol_property_ic_miss(obj, sym, &mut cache_slot); assert_eq!(got.to_bits(), second.to_bits()); assert_eq!(cache[3], second.to_bits()); } @@ -1382,10 +1417,11 @@ mod own_data_ic_tests { let _global = crate::gc::global_side_table_test_lock(); unsafe { let sym = super::constructors::js_symbol_new_empty(); - let mut cache = [0_u64; 12]; - let got = js_object_get_symbol_property_ic_miss(7.0, sym, cache.as_mut_ptr()); + let mut cache: SymbolPicCache = [0; SYMBOL_PIC_WORDS]; + let mut cache_slot: SymbolPicCacheSlot = &mut cache; + let got = js_object_get_symbol_property_ic_miss(7.0, sym, &mut cache_slot); assert_eq!(got.to_bits(), TAG_UNDEFINED); - assert_eq!(cache, [0_u64; 12]); + assert_eq!(cache, [0; SYMBOL_PIC_WORDS]); } } @@ -1404,15 +1440,17 @@ mod own_data_ic_tests { crate::object::js_object_set_field_by_name(metadata_ptr, key, 41.0); super::properties::js_object_set_symbol_property(owner, sym, metadata); - let mut symbol_cache = [0_u64; 12]; + let mut symbol_cache: SymbolPicCache = [0; SYMBOL_PIC_WORDS]; + let mut symbol_cache_slot: SymbolPicCacheSlot = &mut symbol_cache; let mut field_cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + let mut field_cache_slot: crate::object::PicCacheSlot = &mut field_cache; let first = js_object_get_symbol_then_field_ic_miss( owner, sym, key, 0, - symbol_cache.as_mut_ptr(), - &mut field_cache, + &mut symbol_cache_slot, + &mut field_cache_slot, ); let epoch_before_named_write = symbol_cache[0]; let slot = field_cache[1] as usize; @@ -1430,8 +1468,8 @@ mod own_data_ic_tests { sym, key, 0, - symbol_cache.as_mut_ptr(), - &mut field_cache, + &mut symbol_cache_slot, + &mut field_cache_slot, ); let epoch_after_named_write = PERRY_SYMBOL_PROPERTY_IC_EPOCH.load(Ordering::Acquire); crate::gc::gc_unsuppress(); @@ -1459,15 +1497,17 @@ mod own_data_ic_tests { let sym = super::constructors::js_symbol_new_empty(); let key = js_string_from_bytes(b"id".as_ptr(), 2); super::properties::js_object_set_symbol_property(owner, sym, 7.0); - let mut symbol_cache = [0_u64; 12]; + let mut symbol_cache: SymbolPicCache = [0; SYMBOL_PIC_WORDS]; + let mut symbol_cache_slot: SymbolPicCacheSlot = &mut symbol_cache; let mut field_cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + let mut field_cache_slot: crate::object::PicCacheSlot = &mut field_cache; let got = js_object_get_symbol_then_field_ic_miss( owner, sym, key, 0, - symbol_cache.as_mut_ptr(), - &mut field_cache, + &mut symbol_cache_slot, + &mut field_cache_slot, ); crate::gc::gc_unsuppress(); diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index ebdac4c887..9535f9cad8 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -1153,6 +1153,7 @@ pub use guards::{ js_typed_feedback_class_field_get_guard, js_typed_feedback_class_field_set_guard, js_typed_feedback_closure_direct_call_guard, js_typed_feedback_method_direct_call_guard, js_typed_feedback_native_call_method, js_typed_feedback_native_call_method_apply, + MethodPicCache, MethodPicCacheSlot, METHOD_PIC_WORDS, }; #[path = "typed_feedback/trace.rs"] diff --git a/crates/perry-runtime/src/typed_feedback/guards.rs b/crates/perry-runtime/src/typed_feedback/guards.rs index 53f0121659..339bc3ccff 100644 --- a/crates/perry-runtime/src/typed_feedback/guards.rs +++ b/crates/perry-runtime/src/typed_feedback/guards.rs @@ -1213,6 +1213,15 @@ pub extern "C" fn js_closure_exact_func_guard( } } +/// Words in a per-site imported-object own-method cache: the one +/// `(ShapeId << 32 | class_id)` token the emitted guard compares. +pub const METHOD_PIC_WORDS: usize = 1; +/// A per-site own-method cache, as the emitted slot resolves it. +pub type MethodPicCache = [u64; METHOD_PIC_WORDS]; +/// The emitted `@perry_ic_N = private global ptr null` for such a site: null +/// until the site's first priming miss (#9708). +pub type MethodPicCacheSlot = *mut MethodPicCache; + /// Revalidate and prime the shape token for an own object-literal method. /// /// The exported adapter object may append ordinary state fields during @@ -1228,6 +1237,10 @@ pub extern "C" fn js_closure_exact_func_guard( /// or mints a semantic successor. A spill-only metadata record is allowed: /// appending past the object's inline birth width creates one even though the /// original method slot and its lookup semantics remain unchanged. +/// +/// `cache_slot` is the site's [`MethodPicCacheSlot`] address (#9708). A miss +/// that cannot prime clears an existing cache's token but never allocates +/// one; only the publishing tail below resolves the slot. #[no_mangle] pub unsafe extern "C" fn js_object_own_method_cache_miss( receiver: f64, @@ -1236,12 +1249,15 @@ pub unsafe extern "C" fn js_object_own_method_cache_miss( method_name_ptr: *const i8, method_name_len: usize, expected_func_ptr: *const u8, - cache_token: *mut u64, + cache_slot: *mut MethodPicCacheSlot, ) -> u64 { - if !cache_token.is_null() { - *cache_token = 0; + { + let cache = crate::object::pic_slot_peek(cache_slot); + if !cache.is_null() { + (*cache)[0] = 0; + } } - if expected_class_id == 0 || expected_func_ptr.is_null() || cache_token.is_null() { + if expected_class_id == 0 || expected_func_ptr.is_null() || cache_slot.is_null() { return 0; } let Some(method_bytes) = method_name_bytes(method_name_ptr, method_name_len) else { @@ -1303,7 +1319,8 @@ pub unsafe extern "C" fn js_object_own_method_cache_miss( if !crate::object::shapes::is_shape_id(shape_id) { return 0; } - *cache_token = ((shape_id as u64) << 32) | expected_class_id as u64; + let cache = crate::object::pic_slot_resolve(cache_slot); + (*cache)[0] = ((shape_id as u64) << 32) | expected_class_id as u64; closure as u64 } @@ -1339,7 +1356,7 @@ mod keep_guard_symbols { #[cfg(feature = "keepalive-anchors")] #[used] static G3B: extern "C" fn(f64, *const u8) -> u64 = js_closure_exact_func_guard; #[cfg(feature = "keepalive-anchors")] - #[used] static G3C: unsafe extern "C" fn(f64, u32, u32, *const i8, usize, *const u8, *mut u64) -> u64 = js_object_own_method_cache_miss; + #[used] static G3C: unsafe extern "C" fn(f64, u32, u32, *const i8, usize, *const u8, *mut MethodPicCacheSlot) -> u64 = js_object_own_method_cache_miss; #[cfg(feature = "keepalive-anchors")] #[used] static G4: unsafe extern "C" fn(f64, u32, u32, u32) -> i32 = js_method_direct_shape_guard; #[cfg(feature = "keepalive-anchors")] diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index 935caf01dd..c12ed2c0c5 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -1391,7 +1391,7 @@ fn representation_lowering_helpers_have_lto_keepalive_anchors() { ( guards, "static G3C", - "static G3C: unsafe extern \"C\" fn(f64, u32, u32, *const i8, usize, *const u8, *mut u64) -> u64", + "static G3C: unsafe extern \"C\" fn(f64, u32, u32, *const i8, usize, *const u8, *mut MethodPicCacheSlot) -> u64", "js_object_own_method_cache_miss", ), ( @@ -2517,7 +2517,8 @@ fn own_method_cache_accepts_appends_and_rejects_live_method_mutation() { let closure_value = crate::value::js_nanbox_pointer(closure as i64); crate::object::js_object_set_field_by_name(object, method_key, closure_value); let receiver = crate::value::js_nanbox_pointer(object as i64); - let mut cache = 0; + let mut cache: MethodPicCache = [0]; + let mut cache_slot: MethodPicCacheSlot = &mut cache; let first = unsafe { js_object_own_method_cache_miss( @@ -2527,12 +2528,12 @@ fn own_method_cache_accepts_appends_and_rejects_live_method_mutation() { b"method".as_ptr() as *const i8, 6, fn_ptr, - &mut cache, + &mut cache_slot, ) }; assert_eq!(first, closure as u64); - assert_ne!(cache, 0); - let initial_shape_token = cache; + assert_ne!(cache[0], 0); + let initial_shape_token = cache[0]; crate::object::js_object_set_field_by_name(object, extra_key, 42.0); let after_append = unsafe { @@ -2543,12 +2544,12 @@ fn own_method_cache_accepts_appends_and_rejects_live_method_mutation() { b"method".as_ptr() as *const i8, 6, fn_ptr, - &mut cache, + &mut cache_slot, ) }; assert_eq!(after_append, closure as u64); assert_ne!( - cache, initial_shape_token, + cache[0], initial_shape_token, "append must publish the live successor shape" ); @@ -2565,11 +2566,11 @@ fn own_method_cache_accepts_appends_and_rejects_live_method_mutation() { b"method".as_ptr() as *const i8, 6, fn_ptr, - &mut cache, + &mut cache_slot, ) }; assert_eq!(after_spilled_append, closure as u64); - assert_ne!(cache, 0); + assert_ne!(cache[0], 0); let replacement = crate::closure::js_closure_alloc_singleton(test_direct_method_ptr()); crate::object::js_object_set_field_by_name( @@ -2585,11 +2586,11 @@ fn own_method_cache_accepts_appends_and_rejects_live_method_mutation() { b"method".as_ptr() as *const i8, 6, fn_ptr, - &mut cache, + &mut cache_slot, ) }; assert_eq!(replaced, 0); - assert_eq!(cache, 0); + assert_eq!(cache[0], 0); crate::object::js_object_set_field_by_name(object, method_key, closure_value); crate::object::js_object_delete_field(object, method_key); @@ -2601,11 +2602,11 @@ fn own_method_cache_accepts_appends_and_rejects_live_method_mutation() { b"method".as_ptr() as *const i8, 6, fn_ptr, - &mut cache, + &mut cache_slot, ) }; assert_eq!(deleted, 0); - assert_eq!(cache, 0); + assert_eq!(cache[0], 0); } #[test] diff --git a/crates/perry-runtime/src/value/dynamic_object.rs b/crates/perry-runtime/src/value/dynamic_object.rs index 11b214975d..e8bc1203eb 100644 --- a/crates/perry-runtime/src/value/dynamic_object.rs +++ b/crates/perry-runtime/src/value/dynamic_object.rs @@ -255,15 +255,31 @@ pub extern "C" fn js_value_length_property_f64(value: f64) -> f64 { value_length_property_with_cache(value, std::ptr::null_mut()) } +/// Words in a per-site Array-subclass `length` cache: `(identity, length +/// slot, inline bound)`. See `array_subclass_fast_length_with_ic`. +pub const LENGTH_PIC_WORDS: usize = 3; +/// A per-site `length` cache, as the emitted slot resolves it. +pub type LengthPicCache = [u64; LENGTH_PIC_WORDS]; +/// The emitted `@perry_ic_N = private global ptr null` for a `length` site: +/// null until the site's first priming read (#9708). +pub type LengthPicCacheSlot = *mut LengthPicCache; + /// Property-semantic `.length` read whose Array-subclass arm primes the /// generated scalar IC. All non-subclass cases deliberately share the exact /// implementation used by `js_value_length_property_f64`. +/// +/// `cache_slot` is the site's [`LengthPicCacheSlot`] address; the cache is +/// allocated on the first prime (#9708), so an elements-backed receiver — +/// which never publishes shape words — never allocates one. #[no_mangle] -pub extern "C" fn js_value_length_property_ic_f64(value: f64, cache: *mut u64) -> f64 { - value_length_property_with_cache(value, cache) +pub extern "C" fn js_value_length_property_ic_f64( + value: f64, + cache_slot: *mut LengthPicCacheSlot, +) -> f64 { + value_length_property_with_cache(value, cache_slot) } -fn value_length_property_with_cache(value: f64, cache: *mut u64) -> f64 { +fn value_length_property_with_cache(value: f64, cache_slot: *mut LengthPicCacheSlot) -> f64 { let jsval = JSValue::from_bits(value.to_bits()); if jsval.is_undefined() || jsval.is_null() { crate::error::js_throw_type_error_property_access( @@ -278,7 +294,7 @@ fn value_length_property_with_cache(value: f64, cache: *mut u64) -> f64 { crate::builtins::boxed_primitive_to_string_tag(value), Some("String") ) { - return value_length_property_with_cache(payload, cache); + return value_length_property_with_cache(payload, cache_slot); } } @@ -287,7 +303,7 @@ fn value_length_property_with_cache(value: f64, cache: *mut u64) -> f64 { return crate::string::js_string_length(string) as f64; } - if let Some(length) = crate::array::array_subclass_fast_length_with_ic(value, cache) { + if let Some(length) = crate::array::array_subclass_fast_length_with_ic(value, cache_slot) { return length; } @@ -854,7 +870,7 @@ mod length_handle_band_tests { let console_ptr = crate::value::js_nanbox_get_pointer(console_ctor) as usize; let length_key = crate::string::js_string_from_bytes(b"length".as_ptr(), 6); let mut cache = [0_i64; crate::object::PIC_CACHE_WORDS]; - + let mut cache_slot: crate::object::PicCacheSlot = &mut cache; assert_eq!(js_value_length_f64(console_ctor), 1.0); assert_eq!( crate::closure::closure_get_dynamic_prop(console_ptr, "length"), @@ -864,7 +880,7 @@ mod length_handle_band_tests { crate::object::js_object_get_field_ic_miss( console_ptr as *const crate::ObjectHeader, length_key, - &mut cache, + &mut cache_slot, ), 1.0 ); @@ -872,9 +888,10 @@ mod length_handle_band_tests { #[test] fn property_length_preserves_missing_and_non_numeric_values() { - let mut cache = [0_u64; 3]; + let mut cache: LengthPicCache = [0; LENGTH_PIC_WORDS]; + let mut cache_slot: LengthPicCacheSlot = &mut cache; assert_eq!( - js_value_length_property_ic_f64(42.0, cache.as_mut_ptr()).to_bits(), + js_value_length_property_ic_f64(42.0, &mut cache_slot).to_bits(), crate::value::TAG_UNDEFINED, "a number has no length property" ); @@ -888,7 +905,7 @@ mod length_handle_band_tests { let boxed_obj = crate::value::js_nanbox_pointer(obj as i64); assert_eq!( - js_value_length_property_ic_f64(boxed_obj, cache.as_mut_ptr()).to_bits(), + js_value_length_property_ic_f64(boxed_obj, &mut cache_slot).to_bits(), seven_value.to_bits(), "a source-level property read must not coerce its value" ); diff --git a/crates/perry-runtime/src/value/mod.rs b/crates/perry-runtime/src/value/mod.rs index 44525e6c59..fe050ebc5f 100644 --- a/crates/perry-runtime/src/value/mod.rs +++ b/crates/perry-runtime/src/value/mod.rs @@ -150,5 +150,5 @@ pub use dynamic_array::{ pub use dynamic_object::{ js_collection_method_dispatch, js_dynamic_object_get_property, js_dynamic_object_keys, js_get_property, js_value_length_f64, js_value_length_property_f64, - js_value_length_property_ic_f64, + js_value_length_property_ic_f64, LengthPicCache, LengthPicCacheSlot, LENGTH_PIC_WORDS, }; diff --git a/docs/src/internals/memory-model.md b/docs/src/internals/memory-model.md index 984b0d9cd9..ddcffc59e3 100644 --- a/docs/src/internals/memory-model.md +++ b/docs/src/internals/memory-model.md @@ -291,6 +291,17 @@ macOS keeps such pages counted in RSS and `phys_footprint` until memory pressure, so headline RSS numbers overstate what the process would actually hold onto under pressure. +- **`__DATA` dirty is not the inline caches any more.** Every property-access + site used to own a 96-byte `[12 x i64]` cache global — 25 MB of `__bss` on + a Claude Code build, 18.7 MB of it dirty at idle because a page is dirtied + by the first cache touched on it (#9708). A site now owns an 8-byte pointer + **slot** (`@perry_ic_N = private global ptr null`); the cache words are + allocated from a runtime arena on the site's first *priming* miss and + published into the slot, so an unexecuted site costs its zero-filled slot + and nothing resident. The arena is one row of `PERRY_GC_CENSUS`'s + `side_tables` (`ic.lazy_caches`: resolved sites, arena bytes); a program + that reports `0` there executed no property site that could prime. + ## Source map Paths, not line numbers: a line number is a claim nothing re-derives, and every diff --git a/test-files/test_gap_9708_lazy_inline_cache_slots.ts b/test-files/test_gap_9708_lazy_inline_cache_slots.ts new file mode 100644 index 0000000000..2b7ec9cb76 --- /dev/null +++ b/test-files/test_gap_9708_lazy_inline_cache_slots.ts @@ -0,0 +1,192 @@ +// #9708: inline caches are allocated per USED site, behind an 8-byte pointer +// slot that starts null and is filled by the runtime on the site's first +// priming miss. Every inline-cache shape codegen emits has to keep resolving +// correctly across that null → allocated transition, and across the shapes +// that never prime at all (a site whose receivers are all proxies, strings, +// or small native handles keeps a null slot for the life of the program). +// +// The receivers are deliberately `any`-typed so the reads and writes go +// through the generic towers (the per-site PIC), not a class-field fast path. + +// --------------------------------------------------------------------------- +// Property reads: monomorphic, polymorphic (fits the ways), megamorphic. +// --------------------------------------------------------------------------- +function readX(o: any): number { + return o.x; +} + +const mono: any = { x: 1, y: 2 }; +let monoSum = 0; +for (let i = 0; i < 1000; i++) monoSum += readX(mono); +console.log("mono", monoSum); + +// Five shapes with `x` at five different slots: the MRU entry plus four ways. +const polyShapes: any[] = [ + { x: 10 }, + { a: 0, x: 20 }, + { a: 0, b: 0, x: 30 }, + { a: 0, b: 0, c: 0, x: 40 }, + { a: 0, b: 0, c: 0, d: 0, x: 50 }, +]; +let polySum = 0; +for (let i = 0; i < 1000; i++) polySum += readX(polyShapes[i % polyShapes.length]); +console.log("poly", polySum); + +// Twenty shapes: wider than the ways hold, so the site latches megamorphic +// and recovers; every read must still resolve. +const megaShapes: any[] = []; +for (let s = 0; s < 20; s++) { + const o: any = {}; + for (let k = 0; k < s; k++) o["p" + k] = k; + o.x = s; + megaShapes.push(o); +} +let megaSum = 0; +for (let i = 0; i < 2000; i++) megaSum += readX(megaShapes[i % megaShapes.length]); +console.log("mega", megaSum); + +// A site that can never prime: string receivers and a Proxy. Its slot stays +// null forever and every read must keep taking the miss path. +function readLength(o: any): number { + return o.length; +} +let lenSum = 0; +for (let i = 0; i < 100; i++) lenSum += readLength("abc" + i); +const proxied: any = new Proxy({ length: 7 }, { get: (t, k) => (k === "length" ? 42 : undefined) }); +for (let i = 0; i < 10; i++) lenSum += readLength(proxied); +console.log("never-primes", lenSum); + +// A first read on a fresh site through a receiver that throws: the slot is +// still null when the nullish check fires, and the message must match node. +function readY(o: any): number { + return o.y; +} +try { + readY(null); +} catch (e: any) { + console.log("nullish", e instanceof TypeError, e.message); +} +console.log("after-throw", readY({ y: 5 })); + +// Inherited data property: the miss resolves through the prototype and the +// site primes only for own slots, so a mix keeps working. +const proto = { x: 99 }; +const inheriting: any = Object.create(proto); +let inhSum = 0; +for (let i = 0; i < 100; i++) inhSum += readX(i % 2 === 0 ? inheriting : mono); +console.log("inherited", inhSum); + +// --------------------------------------------------------------------------- +// Static-key writes: the four inline ways, the outlined poly tail (shapes +// 5..8) and beyond it. +// --------------------------------------------------------------------------- +function writeX(o: any, v: number): void { + o.x = v; +} +const writeShapes: any[] = []; +for (let s = 0; s < 12; s++) { + const o: any = {}; + for (let k = 0; k < s; k++) o["w" + k] = k; + o.x = -1; + writeShapes.push(o); +} +for (let round = 0; round < 50; round++) { + for (let s = 0; s < writeShapes.length; s++) writeX(writeShapes[s], round * 100 + s); +} +console.log( + "write-static", + writeShapes.map((o: any) => o.x).join(","), +); + +// A write site whose receivers are frozen never primes (the slot stays null) +// and must keep throwing under strict mode on every write, not just the first. +const frozen: any = Object.freeze({ x: 1 }); +let frozenThrows = 0; +for (let i = 0; i < 10; i++) { + try { + writeX(frozen, i); + } catch (e: any) { + if (e instanceof TypeError) frozenThrows++; + } +} +console.log("write-frozen", frozen.x, frozenThrows); + +// --------------------------------------------------------------------------- +// Dynamic-key writes: rotating keys on one shape, then several shapes. +// --------------------------------------------------------------------------- +function writeKey(o: any, k: string, v: number): void { + o[k] = v; +} +const dyn: any = { k0: 0, k1: 0, k2: 0, k3: 0, k4: 0 }; +for (let i = 0; i < 500; i++) writeKey(dyn, "k" + (i % 5), i); +console.log("write-dyn", dyn.k0, dyn.k1, dyn.k2, dyn.k3, dyn.k4); +const dynShapes: any[] = [{ a: 1 }, { a: 1, b: 2 }, { a: 1, b: 2, c: 3 }]; +for (let i = 0; i < 300; i++) writeKey(dynShapes[i % 3], "a", i); +console.log("write-dyn-shapes", dynShapes.map((o: any) => o.a).join(",")); + +// --------------------------------------------------------------------------- +// Symbol-keyed reads and the composed `o[sym].field` read. +// --------------------------------------------------------------------------- +const tag = Symbol("tag"); +function readSym(o: any): any { + return o[tag]; +} +function readSymField(o: any): number { + return o[tag].n; +} +const symHolder: any = { plain: 1 }; +symHolder[tag] = { n: 3 }; +let symSum = 0; +for (let i = 0; i < 200; i++) symSum += readSymField(symHolder); +symHolder[tag] = { n: 4 }; // mutation must invalidate the composed cache +for (let i = 0; i < 200; i++) symSum += readSymField(symHolder); +console.log("symbol", symSum, readSym(symHolder).n, readSym({}) === undefined); + +// --------------------------------------------------------------------------- +// Array subclasses: `.length` and `[i]` on object-backed instances go through +// their own per-site caches. +// --------------------------------------------------------------------------- +class Stack extends Array { + peek(): number { + return this[this.length - 1]; + } +} +function readLen(a: any): number { + return a.length; +} +function readAt(a: any, i: number): number { + return a[i]; +} +const stack: any = new Stack(); +for (let i = 0; i < 10; i++) stack.push(i * 3); +let subSum = 0; +for (let i = 0; i < 100; i++) subSum += readLen(stack) + readAt(stack, i % 10); +console.log("subclass", subSum, stack.peek(), readLen([1, 2, 3]), readAt([7, 8, 9], 1)); + +// --------------------------------------------------------------------------- +// Fusion: `if (base.field[i]) return base.field[i];` shares one cache +// between the fused guard and the generic read. +// --------------------------------------------------------------------------- +function firstTruthy(base: any, n: number): any { + for (let i = 0; i < n; i++) { + if (base.items[i]) return base.items[i]; + } + return "none"; +} +console.log( + "fused", + firstTruthy({ items: [0, "", 0, "hit", 5] }, 5), + firstTruthy({ items: [0, 0] }, 2), + firstTruthy({ items: [1] }, 1), +); + +// --------------------------------------------------------------------------- +// Sites inside functions that never run cost nothing and must not disturb +// their neighbours: the module has hundreds of them. +// --------------------------------------------------------------------------- +const dead: Array<(o: any) => number> = []; +for (let i = 0; i < 4; i++) { + dead.push((o: any) => o.a0 + o.a1 + o.a2 + o.a3 + o.a4 + o.a5 + o.a6 + o.a7 + o.a8 + o.a9); +} +console.log("dead-sites", dead.length, typeof dead[0]); +console.log("done"); From 60c5c395c54cf947e33ca0919554b4a174639a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 14:53:12 +0200 Subject: [PATCH 3/9] changelog: fragment for #9729 Claude-Session: https://claude.ai/code/session_014RVEmbrpNKHwaQdrcMgMHc --- changelog.d/9729-lazy-inline-cache-slots.md | 70 +++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 changelog.d/9729-lazy-inline-cache-slots.md diff --git a/changelog.d/9729-lazy-inline-cache-slots.md b/changelog.d/9729-lazy-inline-cache-slots.md new file mode 100644 index 0000000000..b6e9c39803 --- /dev/null +++ b/changelog.d/9729-lazy-inline-cache-slots.md @@ -0,0 +1,70 @@ +### Inline caches are allocated per *used* site, not per emitted site (#9708) + +Every inline-cache site codegen emitted — the generic property read, the +static- and dynamic-key write ICs and their poly tail, the Symbol-keyed and +composed `o[sym].field` reads, the Array-subclass `length` / `[i]` caches, +the fused `if (a.f[i]) return a.f[i]` cache and the imported-object method +guard — owned a `[12 x i64] zeroinitializer` global: 96 B of `__bss` per +site whether or not the program ever executed it. On the Claude Code bundle +that was 262k caches, 25 MB of zero-fill, and 18.7 MB of it **dirty resident +memory at idle**, because a page is dirtied by the first cache touched on it +and the few thousand hot sites are scattered across all of them. + +A site now owns an 8-byte pointer **slot**, `@perry_ic_N = private global +ptr null`. The cache words live in a runtime arena +(`perry-runtime/src/object/field_get_set/ic_slot.rs`): the miss handler +resolves the slot with `pic_slot_resolve` the first time it actually +*primes* the site, bump-allocates the words from a 64 KiB zeroed chunk and +publishes them with a compare-and-swap (two `perry/thread` agents racing on +one site agree on one cache). A miss that cannot prime — proxy, string or +small-handle receiver, a missing key, an accessor, a frozen target — never +touches the slot, so such a site costs its 8 bytes and nothing else; the +write IC's poly tail is not allocated until a fifth shape arrives. Cache +layout and every prime/evict policy are unchanged: the runtime writes the +same words through the same `PicCache` type, and `pic_slot_resolve` sizes +the allocation from that type, so the width pairing test keeps its meaning. + +**Hot path.** Each inline hit path loads the slot (a load with no dependency +on the receiver, so it issues alongside the header loads) and folds `!= null` +into the receiver guard it already evaluates — one fused compare, no new +block — then reads the cache words through the loaded pointer; the runtime +entries take the slot's address. Where a site reads word 0 inside a flat +predicate (the dynamic-key write IC, the array-like index cache, the method +guard) it reads through `select(present, cache, slot)`: the slot's own 8 +bytes of null are exactly the zero token an empty global used to read as, so +the branch structure and the transition-IC reachability are untouched. +Measured on x86-64 (`perf stat -e instructions:u`, perry-dev builds): + +| program | base | lazy slots | delta | +|---|---:|---:|---:| +| all-generic-IC microbenchmark (95M IC ops) | 13.662 G | 14.196 G | +3.9 % (3 instr per hit: slot load, `test`, never-taken `je`) | +| `bench_object_property` | 250.9 M | 248.6 M | −0.9 % | +| `bench_json_readonly` | 2 263.2 M | 2 258.6 M | −0.2 % | +| `bench_dynamic_property_keys` | 1 129.1 M | 1 124.5 M | −0.4 % | +| `07_object_create`, `09_method_calls`, `12_binary_trees`, `14_closure` | | | ±0.00 % | + +The typed-feedback IC counters (`PERRY_TYPED_FEEDBACK_TRACE`) are identical +on both arms for the microbenchmark — 81 666 674 guard passes, 18 333 339 +guard failures, 18 333 339 fallback calls over 18 sites — so hit rates are +unchanged, not merely output. On the issue's target (macOS arm64) the fused +compare is a `ccmp`, so the hit-path cost there is the slot load plus one +instruction. + +**Footprint.** A generated probe with 16 000 read sites of which 1 604 prime +(every 10th function runs — the scattered-hot-sites shape from the issue), +Linux x86-64, 4 KiB pages: `.bss` 2 408 752 → 997 144 B, whole-process +anonymous `Private_Dirty` 1 660 → 632 kB. The `PERRY_GC_CENSUS` side table +gains an `ic.lazy_caches` row (resolved sites, arena bytes) so a run can +assert the subject was live; the issue's macOS `vmmap` numbers are the ones +to re-measure on a bundle build. + +Gap coverage: `test_gap_9708_lazy_inline_cache_slots.ts` exercises every +IC shape across the null → allocated transition — mono/poly/megamorphic +reads, a site that can never prime, a nullish first read, inherited +properties, static writes through the four ways and the poly tail, a frozen +target, rotating dynamic keys, Symbol and composed Symbol-then-field reads +with invalidation, Array-subclass `length`/index, the fused field-index +return, and hundreds of never-executed sites — and matches node byte for +byte. `array/subclass.rs` was at the 2 000-line cap, so +`js_packed_arraylike_index_get` and its cache types moved to the child module +`array/subclass_packed_index.rs`. From ab01392b1e8c3ee5a7730320ef3fdb7f83658c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 15:13:28 +0200 Subject: [PATCH 4/9] fix(transform): unwind a labeled escape out of nested loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `break label` / `continue label` that targets an outer loop from inside a nested loop threw `TypeError: Cannot read properties of undefined (reading 'done')` (break) or silently produced nothing (continue), in sync generators, async generators and async functions alike. Generator linearization gives each loop one break sentinel and one continue sentinel, so a completion can only name the loop it sits in. `rewrite_labeled_bc_in_stmts` converts labeled completions to plain ones at the labeled loop's own body level and stops at nested loops — a plain completion there would bind to the nested loop. The escape that was left survived into a state body, where the dispatch lowering has no sentinel for it and dropped it. The code noted the gap ("the single-sentinel scheme can't yet distinguish targets"). Unwind it through a carrier local instead, so every completion the linearizer sees is plain: the escape sets the carrier and plain-breaks out of its loop, each intermediate loop propagates with `if (carrier != 0) break`, and the labeled loop turns the carrier back into the real `break`/`continue`. A switch carrying an escape is desugared to `if`s first, since a plain `break` inside a switch binds to the switch. The issue's own repro was already fixed by #9189; the surviving hole is the cross-loop target, where the switch turns out to be incidental. Closes #9199 --- .../9730-labeled-escape-nested-loops.md | 51 ++++ .../src/generator/break_continue.rs | 285 ++++++++++++++++++ .../src/generator/linearize.rs | 25 +- ...st_gap_9199_labeled_escape_nested_loops.ts | 106 +++++++ 4 files changed, 463 insertions(+), 4 deletions(-) create mode 100644 changelog.d/9730-labeled-escape-nested-loops.md create mode 100644 test-files/test_gap_9199_labeled_escape_nested_loops.ts diff --git a/changelog.d/9730-labeled-escape-nested-loops.md b/changelog.d/9730-labeled-escape-nested-loops.md new file mode 100644 index 0000000000..80f244a6e2 --- /dev/null +++ b/changelog.d/9730-labeled-escape-nested-loops.md @@ -0,0 +1,51 @@ +**A labeled `break`/`continue` that targets an outer loop from inside a nested +loop now works in generators, async generators and async functions** (#9199). +It previously threw `TypeError: Cannot read properties of undefined (reading +'done')` for `break`, and silently produced nothing for `continue`. + +```ts +async function* g() { + O: for (const x of [1, 2]) { I: for (const y of [0, 1]) { yield "b" + x + y; break O; } } +} +// node: b10 before: TypeError … reading 'done' +``` + +Generator linearization gives each loop a single `break` sentinel and a single +`continue` sentinel, so a completion can only name the loop it sits in. +`rewrite_labeled_bc_in_stmts` therefore converted `break label` / `continue +label` to plain completions **only at the labeled loop's own body level** and +stopped at nested loops — correctly, since a plain completion inside a nested +loop would bind to that loop. What was missing is what happens to the escape +that is left: it survived verbatim into a state body, where the dispatch +lowering has no sentinel for it and dropped it. The limitation was noted in the +code ("the single-sentinel scheme can't yet distinguish targets"). + +Rather than teach the state machine to name a distant target, the escape is now +unwound one loop at a time through a carrier local, so every completion the +linearizer sees is plain and binds to the loop it is in: + +``` +__esc = 0; +inner: while (…) { … __esc = 1; break; … } // was `break label` +if (__esc == 1) break; // in the labeled loop +if (__esc == 2) continue; +``` + +Deeper nesting reuses the same carrier and propagates outward with a bare +`if (__esc != 0) break;` after each intermediate loop. A `switch` that carries +an escape is desugared to `if`s first, since a plain `break` inside a switch +would bind to the switch. + +The hole was wider than the issue's own repro, which #9189 had already closed: +it reached sync generators and async functions as well as async generators, +and the `switch` in the report was incidental — a bare `break outer` in a +nested loop failed on its own, while the switch-wrapped form worked because +#9186's routing already handled it. + +`test-files/test_gap_9199_labeled_escape_nested_loops.ts` pins 13 shapes: +`break`/`continue` of an outer label from a nested loop in all three function +kinds, three-deep nesting, a `while` outer, an `await` before the escape, a +conditional escape, `try`/`finally` around it (finalizers still run in order), +a reused label name on a sibling loop, and the switch-wrapped form that already +worked. Unpatched the fixture throws on its first row and then hangs; patched +it is byte-identical to node 26.5.1. diff --git a/crates/perry-transform/src/generator/break_continue.rs b/crates/perry-transform/src/generator/break_continue.rs index ff3963d58d..0536466a39 100644 --- a/crates/perry-transform/src/generator/break_continue.rs +++ b/crates/perry-transform/src/generator/break_continue.rs @@ -817,3 +817,288 @@ pub fn stmts_have_continue_inside_try_finally(stmts: &[Stmt]) -> bool { _ => false, }) } + +// ── #9199: labeled break/continue that escapes a NESTED loop ────────────── +// +// `rewrite_labeled_bc_in_stmts` converts `break label` / `continue label` to +// plain completions only at the labeled loop's OWN body level, and stops at +// nested loops (a plain completion there would bind to the nested loop). The +// linearizer's single break/continue sentinel per loop then has no way to name +// an outer loop's target, so a labeled completion crossing a loop boundary +// survived verbatim into a state body and the dispatch lowering dropped it: +// `break` produced a malformed iterator result ("Cannot read properties of +// undefined (reading 'done')"), `continue` silently produced nothing at all. +// +// The fix is to stop asking the state machine to name a distant target. A +// carrier local unwinds the escape one loop at a time, so every completion the +// linearizer sees is plain and binds to the loop it sits in: +// +// __esc = 0; +// inner: while (…) { … __esc = 1; break; … } // `break label` +// if (__esc == 1) break; // in the labeled loop +// if (__esc == 2) continue; +// +// Deeper nesting reuses the same carrier and propagates with a bare +// `if (__esc != 0) break;` after each intermediate loop, so an escape from any +// depth walks out to the labeled loop without the linearizer ever seeing a +// labeled completion. + +/// Carrier value for a `break