diff --git a/changelog.d/9041-store-receiver-lanes.md b/changelog.d/9041-store-receiver-lanes.md new file mode 100644 index 0000000000..88d2ffab52 --- /dev/null +++ b/changelog.d/9041-store-receiver-lanes.md @@ -0,0 +1 @@ +Gave module-global (and captured) array receivers the inline guarded index-store lane and both inline push tiers, and replaced the packed versioned loop's per-store runtime guard call with the range store's inline value check — isolated stores through globals 34.7 → 3.1 ns, `i < a.length` store loops 9.1 → 1.4 ns, global pushes to node parity. diff --git a/crates/perry-codegen/src/expr/array_push.rs b/crates/perry-codegen/src/expr/array_push.rs index b227cd1af5..53ce6b6464 100644 --- a/crates/perry-codegen/src/expr/array_push.rs +++ b/crates/perry-codegen/src/expr/array_push.rs @@ -54,6 +54,7 @@ use anyhow::{anyhow, Result}; use perry_hir::Expr; +use crate::block::LlBlock; use crate::nanbox::double_literal; use crate::native_value::{ BoundsState, BufferAccessMode, ExpectedNativeRep, LoweredValue, MaterializationReason, @@ -434,6 +435,56 @@ fn emit_push_writeback( Ok(()) } +/// Where an inline push tier's receiver binding lives: a stack slot or a +/// module-global root cell. Both inline tiers below need exactly two binding +/// operations — a head write-back after a slow/realloc arm, and a head reload +/// at the merge for the returned `length` — and both were hard-coded to +/// `ctx.locals`, which silently excluded module-global receivers from BOTH +/// tiers: a global `out.push(v)` fell to a bare `js_array_push_f64_spec` call +/// per push (~26 ns vs 2.5 on the isolated append). The write-back twin +/// (`emit_push_writeback`) has handled globals all along; this mirrors its +/// two arms for the tiers. #8617 precedent: extending an inline lane's +/// admission from slot locals to module-global bindings. +enum PushReceiverHome { + Slot(String), + Global(String), +} + +impl PushReceiverHome { + fn resolve(ctx: &FnCtx<'_>, array_id: u32) -> Option { + if ctx.boxed_vars.contains(&array_id) || ctx.closure_captures.contains_key(&array_id) { + return None; + } + if let Some(slot) = ctx.locals.get(&array_id) { + return Some(Self::Slot(slot.clone())); + } + if let Some(name) = ctx.module_globals.get(&array_id) { + return Some(Self::Global(format!("@{}", name))); + } + None + } + + fn store_head(&self, blk: &mut LlBlock, new_box: &str) { + match self { + Self::Slot(slot) => { + blk.store(DOUBLE, new_box, slot); + } + Self::Global(g_ref) => { + // GC_STORE_AUDIT(ROOT): module global array slot is a + // registered mutable GC root. + emit_root_nanbox_store_on_block(blk, new_box, g_ref); + } + } + } + + fn load_head(&self, blk: &mut LlBlock) -> String { + match self { + Self::Slot(slot) => blk.load(DOUBLE, slot), + Self::Global(g_ref) => blk.load(DOUBLE, g_ref), + } + } +} + fn lower_array_push_value( ctx: &mut FnCtx<'_>, value: &Expr, @@ -864,143 +915,139 @@ fn lower_inner(ctx: &mut FnCtx<'_>, expr: &Expr, value_discarded: bool) -> Resul crate::type_analysis::expr_produces_canonical_raw_f64(ctx, value); let keep_guarded_numeric_push = super::typed_feedback_emission_enabled() || !inline_value_shape; - if require_numeric_layout - && keep_guarded_numeric_push - && !ctx.boxed_vars.contains(array_id) - && !ctx.closure_captures.contains_key(array_id) - && ctx.locals.contains_key(array_id) - { - let slot = ctx.locals.get(array_id).cloned().unwrap(); - let feedback_site_id = emit_typed_feedback_register_site( - ctx, - TypedFeedbackKind::ArrayElement, - "array.push", - TypedFeedbackContract::numeric_array_push(), - ); - let fast_idx = ctx.new_block("apush.numeric_fast"); - let fallback_idx = ctx.new_block("apush.numeric_fallback"); - let merge_idx = ctx.new_block("apush.numeric_merge"); - let fast_label = ctx.block_label(fast_idx); - let fallback_label = ctx.block_label(fallback_idx); - let merge_label = ctx.block_label(merge_idx); - - let guard_ok = { - let blk = ctx.block(); - let guard_i32 = blk.call( - I32, - "js_typed_feedback_numeric_array_push_guard", - &[(I64, &feedback_site_id), (DOUBLE, &arr_box), (DOUBLE, &v)], - ); - blk.icmp_ne(I32, &guard_i32, "0") - }; - ctx.block().cond_br(&guard_ok, &fast_label, &fallback_label); - - ctx.current_block = fast_idx; - { - let blk = ctx.block(); - let arr_handle = unbox_to_i64(blk, &arr_box); - let new_handle = blk.call( - I64, - "js_array_numeric_push_f64_unboxed", - &[(I64, &arr_handle), (DOUBLE, &v)], + if require_numeric_layout && keep_guarded_numeric_push { + if let Some(home) = PushReceiverHome::resolve(ctx, *array_id) { + let feedback_site_id = emit_typed_feedback_register_site( + ctx, + TypedFeedbackKind::ArrayElement, + "array.push", + TypedFeedbackContract::numeric_array_push(), ); - let new_box = nanbox_pointer_inline(blk, &new_handle); - blk.store(DOUBLE, &new_box, &slot); - blk.br(&merge_label); - } - let pushed = LoweredValue { - semantic: SemanticKind::JsNumber, - rep: NativeRep::F64, - llvm_ty: DOUBLE, - value: v.clone(), - }; - ctx.record_lowered_value_with_access_mode_and_facts( - "NumericArrayPush", - Some(*array_id), - "js_array_numeric_push_f64_unboxed", - &pushed, - Some(BoundsState::Guarded { - guard_id: "numeric_array_push_guard".to_string(), - }), - None, - Some(BufferAccessMode::CheckedNative), - None, - None, - None, - vec![raw_f64_layout_fact( + let fast_idx = ctx.new_block("apush.numeric_fast"); + let fallback_idx = ctx.new_block("apush.numeric_fallback"); + let merge_idx = ctx.new_block("apush.numeric_merge"); + let fast_label = ctx.block_label(fast_idx); + let fallback_label = ctx.block_label(fallback_idx); + let merge_label = ctx.block_label(merge_idx); + + let guard_ok = { + let blk = ctx.block(); + let guard_i32 = blk.call( + I32, + "js_typed_feedback_numeric_array_push_guard", + &[(I64, &feedback_site_id), (DOUBLE, &arr_box), (DOUBLE, &v)], + ); + blk.icmp_ne(I32, &guard_i32, "0") + }; + ctx.block().cond_br(&guard_ok, &fast_label, &fallback_label); + + ctx.current_block = fast_idx; + { + let blk = ctx.block(); + let arr_handle = unbox_to_i64(blk, &arr_box); + let new_handle = blk.call( + I64, + "js_array_numeric_push_f64_unboxed", + &[(I64, &arr_handle), (DOUBLE, &v)], + ); + let new_box = nanbox_pointer_inline(blk, &new_handle); + home.store_head(blk, &new_box); + blk.br(&merge_label); + } + let pushed = LoweredValue { + semantic: SemanticKind::JsNumber, + rep: NativeRep::F64, + llvm_ty: DOUBLE, + value: v.clone(), + }; + ctx.record_lowered_value_with_access_mode_and_facts( + "NumericArrayPush", Some(*array_id), - "consumed", - "numeric_array_push_guard", + "js_array_numeric_push_f64_unboxed", + &pushed, + Some(BoundsState::Guarded { + guard_id: "numeric_array_push_guard".to_string(), + }), None, - )], - Vec::new(), - false, - false, - Vec::new(), - ); - - ctx.current_block = fallback_idx; - { - let blk = ctx.block(); - crate::expr::emit_typed_feedback_record_call( - blk, - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], + Some(BufferAccessMode::CheckedNative), + None, + None, + None, + vec![raw_f64_layout_fact( + Some(*array_id), + "consumed", + "numeric_array_push_guard", + None, + )], + Vec::new(), + false, + false, + Vec::new(), ); - let arr_handle = unbox_to_i64(blk, &arr_box); - let new_handle = blk.call( - I64, + + ctx.current_block = fallback_idx; + { + let blk = ctx.block(); + crate::expr::emit_typed_feedback_record_call( + blk, + "js_typed_feedback_record_fallback_call", + &[(I64, &feedback_site_id)], + ); + let arr_handle = unbox_to_i64(blk, &arr_box); + let new_handle = blk.call( + I64, + "js_array_push_f64_spec", + &[(I64, &arr_handle), (DOUBLE, &v)], + ); + let new_box = nanbox_pointer_inline(blk, &new_handle); + home.store_head(blk, &new_box); + blk.br(&merge_label); + } + let fallback = LoweredValue { + semantic: SemanticKind::JsValue, + rep: NativeRep::JsValue, + llvm_ty: DOUBLE, + value: v.clone(), + }; + ctx.record_lowered_value_with_access_mode_and_facts( + "NumericArrayPush", + Some(*array_id), "js_array_push_f64_spec", - &[(I64, &arr_handle), (DOUBLE, &v)], + &fallback, + Some(BoundsState::Unknown), + None, + Some(BufferAccessMode::DynamicFallback), + Some(MaterializationReason::RuntimeApi), + None, + None, + Vec::new(), + vec![ + raw_f64_layout_fact( + Some(*array_id), + "rejected", + "numeric_array_push_guard", + Some(MaterializationReason::RuntimeApi), + ), + raw_f64_layout_fact( + Some(*array_id), + "invalidated", + "runtime_api", + Some(MaterializationReason::RuntimeApi), + ), + ], + false, + false, + Vec::new(), ); - let new_box = nanbox_pointer_inline(blk, &new_handle); - blk.store(DOUBLE, &new_box, &slot); - blk.br(&merge_label); - } - let fallback = LoweredValue { - semantic: SemanticKind::JsValue, - rep: NativeRep::JsValue, - llvm_ty: DOUBLE, - value: v.clone(), - }; - ctx.record_lowered_value_with_access_mode_and_facts( - "NumericArrayPush", - Some(*array_id), - "js_array_push_f64_spec", - &fallback, - Some(BoundsState::Unknown), - None, - Some(BufferAccessMode::DynamicFallback), - Some(MaterializationReason::RuntimeApi), - None, - None, - Vec::new(), - vec![ - raw_f64_layout_fact( - Some(*array_id), - "rejected", - "numeric_array_push_guard", - Some(MaterializationReason::RuntimeApi), - ), - raw_f64_layout_fact( - Some(*array_id), - "invalidated", - "runtime_api", - Some(MaterializationReason::RuntimeApi), - ), - ], - false, - false, - Vec::new(), - ); - ctx.current_block = merge_idx; - if value_discarded { - // Skip the slot reload too — it only feeds the length. - return Ok(double_literal(0.0)); + ctx.current_block = merge_idx; + if value_discarded { + // Skip the slot reload too — it only feeds the length. + return Ok(double_literal(0.0)); + } + let current_box = home.load_head(ctx.block()); + return Ok(emit_array_box_length(ctx, ¤t_box, false)); } - let current_box = ctx.block().load(DOUBLE, &slot); - return Ok(emit_array_box_length(ctx, ¤t_box, false)); } // Fast path: local-bound, non-captured, non-boxed array. @@ -1029,11 +1076,7 @@ fn lower_inner(ctx: &mut FnCtx<'_>, expr: &Expr, value_discarded: bool) -> Resul // array pointer doesn't change unless we grow. The slow // branches both update the slot via the existing // boxed/captured/local fall-through below. - if !ctx.boxed_vars.contains(array_id) - && !ctx.closure_captures.contains_key(array_id) - && ctx.locals.contains_key(array_id) - { - let slot = ctx.locals.get(array_id).cloned().unwrap(); + if let Some(home) = PushReceiverHome::resolve(ctx, *array_id) { let apush_meta_offset = crate::target_layout::object_meta_slot_offset_bytes(ctx.target_triple) .to_string(); @@ -1164,7 +1207,7 @@ fn lower_inner(ctx: &mut FnCtx<'_>, expr: &Expr, value_discarded: bool) -> Resul &[(I64, &arr_handle), (DOUBLE, &v)], ); let new_box = nanbox_pointer_inline(blk, &new_handle); - blk.store(DOUBLE, &new_box, &slot); + home.store_head(blk, &new_box); blk.br(&merge_label); } @@ -1420,7 +1463,7 @@ fn lower_inner(ctx: &mut FnCtx<'_>, expr: &Expr, value_discarded: bool) -> Resul &[(I64, &arr_handle), (DOUBLE, &v)], ); let new_box = nanbox_pointer_inline(blk, &new_handle); - blk.store(DOUBLE, &new_box, &slot); + home.store_head(blk, &new_box); blk.br(&merge_label); } @@ -1429,7 +1472,7 @@ fn lower_inner(ctx: &mut FnCtx<'_>, expr: &Expr, value_discarded: bool) -> Resul // Skip the slot reload too — it only feeds the length. return Ok(double_literal(0.0)); } - let current_box = ctx.block().load(DOUBLE, &slot); + let current_box = home.load_head(ctx.block()); return Ok(emit_array_box_length(ctx, ¤t_box, false)); } diff --git a/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs b/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs index 2ba50f34ff..90ed6e59cc 100644 --- a/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs +++ b/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs @@ -90,6 +90,8 @@ pub(super) const VERIFIED_BARRIER_STEMS: &[(&str, StemKind)] = &[ ("class_field_set", StemKind::PointerTestedStore), ("ctor_prologue", StemKind::ValueAndGenerationTested), ("idxset.inbounds", StemKind::ValueAndGenerationTested), + ("idxset.recv_captured", StemKind::ValueAndGenerationTested), + ("idxset.recv_global", StemKind::ValueAndGenerationTested), ("idxset.recv_prop", StemKind::ValueAndGenerationTested), ("put.pic", StemKind::PointerTestedStore), ]; @@ -452,6 +454,177 @@ fn idxset_inbounds_ir() -> String { .expect("LLVM IR should be UTF-8") } +/// `let g = []` at module level, `probe(v) { for (let i = 0; i < 8; i++) +/// g[i] = v }` — a receiver that resolves through `ctx.module_globals` +/// instead of `ctx.locals` is what routes the store into the +/// `idxset.recv_global` arm of `emit_guarded_inbounds_array_store` (the +/// module-global inline lane this census entry witnesses). Counter and +/// `v: Any` reasoning as in `idxset_inbounds_ir`. +fn idxset_recv_global_ir() -> String { + const G_ID: u32 = 30; + let mut m = Module::new("idxset_recv_global_census.ts"); + m.init = vec![Stmt::Let { + id: G_ID, + name: "g".to_string(), + // An ARRAY-typed global is what keeps the store on the typed lane at + // all (`Type::Any` here sent it to `js_dyn_index_set_strict`, never + // reaching the receiver ladder this stem lives in). + ty: Type::Array(Box::new(Type::Any)), + mutable: true, + init: Some(Expr::Array(Vec::new())), + }]; + m.functions = vec![Function { + id: 1, + name: "probe".to_string(), + type_params: Vec::new(), + params: vec![Param { + id: VAL_ID, + name: "v".to_string(), + ty: Type::Any, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }], + return_type: Type::Any, + body: vec![ + Stmt::For { + init: Some(Box::new(Stmt::Let { + id: IDX_ID, + name: "i".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + })), + condition: Some(Expr::Compare { + op: CompareOp::Lt, + left: Box::new(Expr::LocalGet(IDX_ID)), + right: Box::new(Expr::Integer(8)), + }), + update: Some(Expr::Update { + id: IDX_ID, + op: UpdateOp::Increment, + prefix: false, + }), + body: vec![Stmt::Expr(Expr::IndexSet { + object: Box::new(Expr::LocalGet(G_ID)), + index: Box::new(Expr::LocalGet(IDX_ID)), + value: Box::new(Expr::LocalGet(VAL_ID)), + })], + }, + Stmt::Return(Some(Expr::LocalGet(G_ID))), + ], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }]; + m.init_kind = ModuleInitKind::Eager; + String::from_utf8(compile_module(&m, ir_opts()).expect("module compiles")) + .expect("LLVM IR should be UTF-8") +} + +/// `probe(v) { let a = []; const fill = () => { for (...) a[i] = v }; +/// fill(); return a }` — inside the closure the receiver id is neither a +/// stack local nor a module global, which is the `idxset.recv_captured` +/// arm. The capture is NON-mutable (`a` is never reassigned; element stores +/// mutate the array, not the binding), so the closure reads the captured +/// pointer directly rather than through a box. +fn idxset_recv_captured_ir() -> String { + const FILL_ID: u32 = 31; + let mut m = Module::new("idxset_recv_captured_census.ts"); + m.functions = vec![Function { + id: 1, + name: "probe".to_string(), + type_params: Vec::new(), + params: vec![Param { + id: VAL_ID, + name: "v".to_string(), + ty: Type::Any, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }], + return_type: Type::Any, + body: vec![ + Stmt::Let { + id: ARR_ID, + name: "a".to_string(), + ty: Type::Any, + mutable: true, + init: Some(Expr::Array(Vec::new())), + }, + Stmt::Let { + id: FILL_ID, + name: "fill".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::Closure { + func_id: 2, + params: Vec::new(), + return_type: Type::Any, + body: vec![Stmt::For { + init: Some(Box::new(Stmt::Let { + id: IDX_ID, + name: "i".to_string(), + ty: Type::Number, + mutable: true, + init: Some(Expr::Integer(0)), + })), + condition: Some(Expr::Compare { + op: CompareOp::Lt, + left: Box::new(Expr::LocalGet(IDX_ID)), + right: Box::new(Expr::Integer(8)), + }), + update: Some(Expr::Update { + id: IDX_ID, + op: UpdateOp::Increment, + prefix: false, + }), + body: vec![Stmt::Expr(Expr::IndexSet { + object: Box::new(Expr::LocalGet(ARR_ID)), + index: Box::new(Expr::LocalGet(IDX_ID)), + value: Box::new(Expr::LocalGet(VAL_ID)), + })], + }], + captures: vec![ARR_ID, VAL_ID], + mutable_captures: Vec::new(), + captures_this: false, + captures_new_target: false, + enclosing_class: None, + is_arrow: true, + is_async: false, + is_generator: false, + is_strict: false, + }), + }, + Stmt::Expr(Expr::Call { + callee: Box::new(Expr::LocalGet(FILL_ID)), + args: Vec::new(), + type_args: Vec::new(), + byte_offset: 0, + }), + Stmt::Return(Some(Expr::LocalGet(ARR_ID))), + ], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }]; + m.init_kind = ModuleInitKind::Eager; + String::from_utf8(compile_module(&m, ir_opts()).expect("module compiles")) + .expect("LLVM IR should be UTF-8") +} + /// `const a = []; a.push({v: 1})` — the pointer-valued push whose barrier /// #7511 gates (same fixture as `array_push.rs::parent_gate_tests`). fn apush_ir() -> String { @@ -507,6 +680,8 @@ fn probe_ir(stem: &str) -> String { "class_field_set" => super::class_field_barrier_tests::ir(), "ctor_prologue" => ctor_prologue_ir(), "idxset.inbounds" => idxset_inbounds_ir(), + "idxset.recv_captured" => idxset_recv_captured_ir(), + "idxset.recv_global" => idxset_recv_global_ir(), "idxset.recv_prop" => super::index_set_barrier_tests::ir(), "put.pic" => super::write_pic_barrier_tests::census_put_pic_ir(), other => panic!( diff --git a/crates/perry-codegen/src/expr/index_set.rs b/crates/perry-codegen/src/expr/index_set.rs index 47a27e14c9..ccbbcec6b0 100644 --- a/crates/perry-codegen/src/expr/index_set.rs +++ b/crates/perry-codegen/src/expr/index_set.rs @@ -1147,26 +1147,63 @@ pub(crate) fn lower( &feedback_site_id, )?; } else if let Some(global_name) = ctx.module_globals.get(&id).cloned() { - let blk = ctx.block(); - let arr_bits = blk.bitcast_double_to_i64(&arr_box); - let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); - let idx_i32 = blk.fptosi(DOUBLE, &idx_double, I32); - let new_handle = blk.call( - I64, - "js_typed_feedback_array_set_f64_extend", - &[ - (I64, &feedback_site_id), - (I64, &arr_handle), - (I32, &idx_i32), - (DOUBLE, &val_double), - ], - ); - let new_box = nanbox_pointer_inline(blk, &new_handle); - let g_ref = format!("@{}", global_name); - // GC_STORE_AUDIT(ROOT): module global array slot is a registered mutable GC root. - emit_root_nanbox_store_on_block(ctx.block(), &new_box, &g_ref); - // The extending runtime setter barriers the actual - // destination slot on every pointer-bearing store. + // A module-global receiver took a bare extend call + // on EVERY store — the only receiver shape with no + // inline arm at all (params and slot locals get + // `lower_index_set_fast`, property receivers get + // the guarded diamond below): 9.1 vs 3.4 ns per + // in-bounds store. A STRICTLY in-bounds store + // changes no head and no length, so the global + // root needs no re-store on the fast arm — the + // head write-back below is slow-arm-only, exactly + // like `lower_index_set_fast`'s slot write-back. + let idx_i32 = { + let blk = ctx.block(); + blk.fptosi(DOUBLE, &idx_double, I32) + }; + let arr_box_c = arr_box.clone(); + let val_double_c = val_double.clone(); + let idx_i32_c = idx_i32.clone(); + let feedback_site_id_c = feedback_site_id.clone(); + let slow_store = move |ctx: &mut FnCtx<'_>| -> Result<()> { + let blk = ctx.block(); + let arr_bits = blk.bitcast_double_to_i64(&arr_box_c); + let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); + let new_handle = blk.call( + I64, + "js_typed_feedback_array_set_f64_extend", + &[ + (I64, &feedback_site_id_c), + (I64, &arr_handle), + (I32, &idx_i32_c), + (DOUBLE, &val_double_c), + ], + ); + let new_box = nanbox_pointer_inline(blk, &new_handle); + let g_ref = format!("@{}", global_name); + // GC_STORE_AUDIT(ROOT): module global array slot is a registered mutable GC root. + emit_root_nanbox_store_on_block(ctx.block(), &new_box, &g_ref); + // The extending runtime setter barriers the actual + // destination slot on every pointer-bearing store. + Ok(()) + }; + if !super::typed_feedback_emission_enabled() { + super::index_set_guarded::emit_guarded_inbounds_array_store( + ctx, + &arr_box, + &idx_i32, + &val_double, + "idxset.recv_global", + layout_note_needed, + write_barrier_needed, + value_is_numeric, + slow_store, + )?; + } else { + // Feedback-emission builds keep the out-of-line + // call so observation stays complete. + slow_store(ctx)?; + } } else { // Closure-captured array, or local without a // stack slot (rare). Issue #637 followup / hono r2: @@ -1183,22 +1220,51 @@ pub(crate) fn lower( // writeback target here. Discard the returned // pointer; downstream reads via clean_arr_ptr // follow the forwarding chain to the new head. - let blk = ctx.block(); - let arr_bits = blk.bitcast_double_to_i64(&arr_box); - let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); - let idx_i32 = blk.fptosi(DOUBLE, &idx_double, I32); - blk.call( - I64, - "js_typed_feedback_array_set_f64_extend", - &[ - (I64, &feedback_site_id), - (I64, &arr_handle), - (I32, &idx_i32), - (DOUBLE, &val_double), - ], - ); - // The extending runtime setter barriers the actual - // destination slot on every pointer-bearing store. + // Same inline-arm treatment as the global and + // property receivers: strictly in-bounds needs no + // writeback (forwarding covers the realloc case on + // the slow arm, per the note above). + let idx_i32 = { + let blk = ctx.block(); + blk.fptosi(DOUBLE, &idx_double, I32) + }; + let arr_box_c = arr_box.clone(); + let val_double_c = val_double.clone(); + let idx_i32_c = idx_i32.clone(); + let feedback_site_id_c = feedback_site_id.clone(); + let slow_store = move |ctx: &mut FnCtx<'_>| -> Result<()> { + let blk = ctx.block(); + let arr_bits = blk.bitcast_double_to_i64(&arr_box_c); + let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); + blk.call( + I64, + "js_typed_feedback_array_set_f64_extend", + &[ + (I64, &feedback_site_id_c), + (I64, &arr_handle), + (I32, &idx_i32_c), + (DOUBLE, &val_double_c), + ], + ); + // The extending runtime setter barriers the actual + // destination slot on every pointer-bearing store. + Ok(()) + }; + if !super::typed_feedback_emission_enabled() { + super::index_set_guarded::emit_guarded_inbounds_array_store( + ctx, + &arr_box, + &idx_i32, + &val_double, + "idxset.recv_captured", + layout_note_needed, + write_barrier_needed, + value_is_numeric, + slow_store, + )?; + } else { + slow_store(ctx)?; + } } } else { let idx_i32 = { diff --git a/crates/perry-codegen/src/expr/index_set_packed_loop.rs b/crates/perry-codegen/src/expr/index_set_packed_loop.rs index f9d640e4a3..584a52f234 100644 --- a/crates/perry-codegen/src/expr/index_set_packed_loop.rs +++ b/crates/perry-codegen/src/expr/index_set_packed_loop.rs @@ -228,7 +228,24 @@ pub(super) fn lower_packed_numeric_loop_index_set( array_kind: PackedNumericLoopKind, allow_holes: bool, ) -> Result { - if allow_holes && matches!(array_kind, PackedNumericLoopKind::F64) { + if matches!(array_kind, PackedNumericLoopKind::F64) { + // Both F64 fact kinds route to the inline-check store. The + // hole-tolerant range fact always did; the versioned-loop fact + // (`allow_holes=false`, bound = `arr.length`) used to keep a + // per-iteration `js_typed_feedback_numeric_array_index_set_guard` + // CALL in its fast body — plus the store's own + // `js_array_numeric_value_to_raw_f64` call — costing 9.1 vs 3.3 + // ns/store against the same loop with a constant bound, i.e. the + // "fast" version ran slower than the plain per-store diamond. Every + // check that call performed is already proven here: the loop-entry + // guard proved the dense RawF64 layout and the body walk proved + // nothing in the loop can invalidate it; the fact only ever matches + // offset-0 indices (`packed_f64_loop_fact_for_index` rejects offsets + // on non-holes facts), so the loop condition `i < arr.length` + // (re-read each iteration) proves the store in bounds; and the RHS + // value check is the range store's inline nanbox tag test — a boxed + // value side-exits to the slow loop exactly as the guard's failure + // arm did, before the store, so nothing double-applies. return lower_packed_f64_range_loop_index_set( ctx, arr_id, @@ -238,6 +255,7 @@ pub(super) fn lower_packed_numeric_loop_index_set( side_exit_label, ); } + let _ = allow_holes; let (val_double, native_value, rhs_notes) = lower_packed_numeric_loop_store_value(ctx, arr_id, value, array_kind)?; let arr_expr = Expr::LocalGet(arr_id); diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index 73d4c8761d..eaa0c68587 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -2445,19 +2445,37 @@ fn packed_f64_loop_store_update_versions_with_side_exit() { ir.contains("for.packed_f64_fast") && ir.contains("for.packed_f64_slow"), "safe store-update loop should emit fast and slow clones:\n{ir}" ); + // The fast clone's store is the inline range store: a nanbox tag check on + // the RHS value plus a raw `store double` — NO per-store runtime call. The + // per-iteration `js_typed_feedback_numeric_array_index_set_guard` (and the + // `js_array_numeric_value_to_raw_f64` canonicalization call) it used to + // keep made the "fast" clone slower than the plain per-store diamond + // (9.1 vs 3.3 ns/store); every check the guard performed is proven by the + // loop-entry guard + the loop bound (`i < arr.length`, offset-0 index). + let fast_start = ir + .find("for.packed_f64_fast") + .expect("expected packed-f64 fast clone"); + let fast_end = ir + .find("for.packed_f64_slow") + .expect("expected packed-f64 slow clone"); + let fast_clone = &ir[fast_start..fast_end]; assert!( - ir.contains("call i32 @js_typed_feedback_numeric_array_index_set_guard"), - "fast store should keep a runtime numeric/layout store guard:\n{ir}" + !fast_clone.contains("call i32 @js_typed_feedback_numeric_array_index_set_guard"), + "packed fast clone must not keep a per-store runtime guard call:\n{fast_clone}\n\n{ir}" + ); + assert!( + !fast_clone.contains("call double @js_array_numeric_value_to_raw_f64"), + "packed fast clone must not canonicalize per store — boxed values side-exit:\n{fast_clone}\n\n{ir}" ); assert!( - ir.contains("call double @js_array_numeric_value_to_raw_f64"), - "fast store should canonicalize numeric values before raw f64 storage:\n{ir}" + fast_clone.contains("packed_f64_range_store.fast"), + "packed fast clone should store through the inline range-store block:\n{fast_clone}\n\n{ir}" ); let fallback_start = ir - .find("\npacked_f64_loop_store.fallback.") + .find("\npacked_f64_range_store.side_exit.") .map(|pos| pos + 1) - .expect("expected packed-f64 store fallback block"); + .expect("expected packed-f64 store side-exit block"); let fallback_tail = &ir[fallback_start..]; let fallback_end = fallback_tail .find("\n\n") @@ -2466,7 +2484,7 @@ fn packed_f64_loop_store_update_versions_with_side_exit() { let fallback_block = &ir[fallback_start..fallback_end]; assert!( fallback_block.contains("br label %packed_f64.loop.slow.preheader."), - "packed store guard failure must side-exit to the slow clone preheader:\n{fallback_block}\n\n{ir}" + "packed store value-check failure must side-exit to the slow clone preheader:\n{fallback_block}\n\n{ir}" ); assert!( !fallback_block.contains("js_typed_feedback_array_index_set_fallback_boxed"), @@ -2512,8 +2530,8 @@ fn packed_f64_loop_store_update_versions_with_side_exit() { ); assert!( records.iter().any(|record| { - record["expr_kind"] == "PackedF64LoopStore" - && record["consumer"] == "packed_f64_loop_store" + record["expr_kind"] == "PackedF64RangeLoopStore" + && record["consumer"] == "packed_f64_range_loop_store" && record["access_mode"] == "checked_native" && record["notes"].as_array().is_some_and(|notes| { notes @@ -2522,12 +2540,12 @@ fn packed_f64_loop_store_update_versions_with_side_exit() { }) && record_has_raw_f64_layout_fact(record, "consumed_facts", "consumed") }), - "expected checked packed raw-f64 loop store record:\n{artifact:#}" + "expected checked packed raw-f64 range-store record:\n{artifact:#}" ); assert!( records.iter().any(|record| { - record["expr_kind"] == "PackedF64LoopStore" - && record["consumer"] == "packed_f64_loop_store_side_exit" + record["expr_kind"] == "PackedF64RangeLoopStore" + && record["consumer"] == "packed_f64_range_loop_store_side_exit" && record["access_mode"] == "dynamic_fallback" && record["materialization_reason"] == "runtime_api" && record["fallback_reason"] == "runtime_api" @@ -2537,7 +2555,6 @@ fn packed_f64_loop_store_update_versions_with_side_exit() { .any(|note| note == "store_guard_failure=side_exit_slow_restart") }) && record_has_raw_f64_layout_fact(record, "rejected_facts", "rejected") - && record_has_raw_f64_layout_fact(record, "rejected_facts", "invalidated") }), "expected packed store side-exit fallback evidence:\n{artifact:#}" ); @@ -2965,15 +2982,28 @@ fn packed_f64_loop_unary_math_store_versions_with_side_exit() { !fast_body.contains("js_math_to_number"), "packed fast body must not route Math.abs(arr[i]) through JSValue ToNumber:\n{fast_body}\n\n{ir}" ); + // The fast clone stores through the inline range store (see the + // store-update twin above): no per-store guard call remains in it. + let fast_start = ir + .find("for.packed_f64_fast") + .expect("expected packed-f64 fast clone"); + let fast_end = ir + .find("for.packed_f64_slow") + .expect("expected packed-f64 slow clone"); + let fast_clone = &ir[fast_start..fast_end]; assert!( - ir.contains("call i32 @js_typed_feedback_numeric_array_index_set_guard"), - "fast unary math store should keep a runtime numeric/layout store guard:\n{ir}" + !fast_clone.contains("call i32 @js_typed_feedback_numeric_array_index_set_guard"), + "unary math packed fast clone must not keep a per-store runtime guard call:\n{fast_clone}\n\n{ir}" + ); + assert!( + fast_clone.contains("packed_f64_range_store.fast"), + "unary math packed fast clone should store through the inline range-store block:\n{fast_clone}\n\n{ir}" ); let fallback_start = ir - .find("\npacked_f64_loop_store.fallback.") + .find("\npacked_f64_range_store.side_exit.") .map(|pos| pos + 1) - .expect("expected packed-f64 store fallback block"); + .expect("expected packed-f64 store side-exit block"); let fallback_tail = &ir[fallback_start..]; let fallback_end = fallback_tail .find("\n\n") @@ -3009,8 +3039,8 @@ fn packed_f64_loop_unary_math_store_versions_with_side_exit() { ); assert!( records.iter().any(|record| { - record["expr_kind"] == "PackedF64LoopStore" - && record["consumer"] == "packed_f64_loop_store" + record["expr_kind"] == "PackedF64RangeLoopStore" + && record["consumer"] == "packed_f64_range_loop_store" && record["access_mode"] == "checked_native" && record["notes"].as_array().is_some_and(|notes| { notes @@ -3022,17 +3052,16 @@ fn packed_f64_loop_unary_math_store_versions_with_side_exit() { }) && record_has_raw_f64_layout_fact(record, "consumed_facts", "consumed") }), - "expected checked packed raw-f64 loop store record for unary math RHS:\n{artifact:#}" + "expected checked packed raw-f64 range-store record for unary math RHS:\n{artifact:#}" ); assert!( records.iter().any(|record| { - record["expr_kind"] == "PackedF64LoopStore" - && record["consumer"] == "packed_f64_loop_store_side_exit" + record["expr_kind"] == "PackedF64RangeLoopStore" + && record["consumer"] == "packed_f64_range_loop_store_side_exit" && record["access_mode"] == "dynamic_fallback" && record["materialization_reason"] == "runtime_api" && record["fallback_reason"] == "runtime_api" && record_has_raw_f64_layout_fact(record, "rejected_facts", "rejected") - && record_has_raw_f64_layout_fact(record, "rejected_facts", "invalidated") }), "expected unary math packed store side-exit fallback evidence:\n{artifact:#}" ); @@ -3080,7 +3109,12 @@ fn packed_f64_loop_rejects_coercive_unary_math_store_rhs() { !records.iter().any(|record| { matches!( record["expr_kind"].as_str(), - Some("PackedF64LoopGuard" | "PackedF64LoopStore" | "PackedF64LoopLoad") + Some( + "PackedF64LoopGuard" + | "PackedF64LoopStore" + | "PackedF64RangeLoopStore" + | "PackedF64LoopLoad" + ) ) }), "coercive unary math store loop should not record packed-f64 loop facts:\n{artifact:#}" @@ -6514,7 +6548,12 @@ fn packed_f64_loop_rejects_nonnumeric_store_then_later_read() { !records.iter().any(|record| { matches!( record["expr_kind"].as_str(), - Some("PackedF64LoopGuard" | "PackedF64LoopStore" | "PackedF64LoopLoad") + Some( + "PackedF64LoopGuard" + | "PackedF64LoopStore" + | "PackedF64RangeLoopStore" + | "PackedF64LoopLoad" + ) ) }), "nonnumeric store/read loop should not record packed-f64 loop facts:\n{artifact:#}" @@ -6582,7 +6621,12 @@ fn packed_f64_loop_rejects_store_then_read_invalidation_shape() { !records.iter().any(|record| { matches!( record["expr_kind"].as_str(), - Some("PackedF64LoopGuard" | "PackedF64LoopStore" | "PackedF64LoopLoad") + Some( + "PackedF64LoopGuard" + | "PackedF64LoopStore" + | "PackedF64RangeLoopStore" + | "PackedF64LoopLoad" + ) ) }), "store-bearing loop should not record packed-f64 loop facts:\n{artifact:#}"