diff --git a/Cargo.lock b/Cargo.lock index afa558e162..122c0870fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5765,6 +5765,8 @@ dependencies = [ "sha2 0.11.0", "swc_common", "swc_ecma_ast", + "swc_ecma_transforms_base", + "swc_ecma_visit", "tar", "tempfile", "tokio", diff --git a/changelog.d/9630-template-esm-detection.md b/changelog.d/9630-template-esm-detection.md new file mode 100644 index 0000000000..6dc5d9a32d --- /dev/null +++ b/changelog.d/9630-template-esm-detection.md @@ -0,0 +1,8 @@ +### Fixed + +- **Ambiguous `.js` files now recognize module exports after complex template + interpolations (#9608).** Perry now uses SWC's program parser to find real + top-level module items instead of maintaining a partial byte scanner, so + nested templates, escaped backticks, regex character classes, comments, + strings, and division no longer hide a trailing export. Genuine CommonJS + inputs retain sloppy Script semantics. diff --git a/changelog.d/9632-bun-project-node-addons.md b/changelog.d/9632-bun-project-node-addons.md new file mode 100644 index 0000000000..209bec29ec --- /dev/null +++ b/changelog.d/9632-bun-project-node-addons.md @@ -0,0 +1,12 @@ +### Bun compatibility + +- **Root/project Node-API addons loaded through `import.meta.require` now ship + and run after a Bun extraction tree is removed.** Declare each file by its + exact project-relative path, for example + `"perry": { "nativeAddonPaths": ["native/addon.node"] }`. Perry follows + immutable aliases and simple path constants, including + `new URL("./native/addon.node", import.meta.url).pathname`, and maps relative, + absolute, and `/$bunfs/root/` spellings to the same authenticated sidecar + entry. Dynamic or otherwise unprovable paths fail compilation with guidance + instead of producing a binary that depends on the build machine's source + tree. diff --git a/crates/perry-api-manifest/src/entries/part_4.rs b/crates/perry-api-manifest/src/entries/part_4.rs index 87c1c205f0..1cc3f4ef60 100644 --- a/crates/perry-api-manifest/src/entries/part_4.rs +++ b/crates/perry-api-manifest/src/entries/part_4.rs @@ -1113,7 +1113,7 @@ pub(crate) const API_MANIFEST_PART_4: &[ApiEntry] = &[ method("bun", "stripANSI", false, None), method("bun", "wrapAnsi", false, None), method("bun", "which", false, None), - method("bun", "zstdDecompress", true, None), + method("bun", "zstdDecompress", false, None), method("bun", "zstdDecompressSync", false, None), method("bun", "gc", false, None), method("bun", "generateHeapSnapshot", false, None), diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index 467e7b2636..8ed7f9c547 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -20,305 +20,8 @@ use crate::function::LlFunction; use crate::native_value::NativeRepRecord; use crate::types::LlvmType; -/// Strip a leading LLVM linkage keyword from a global's post-`=` text, if -/// present. Linkage comes before `unnamed_addr`/`constant`/`global` in the -/// grammar, so this leaves the rest of the definition intact. -fn strip_leading_linkage(s: &str) -> &str { - for kw in [ - "private ", - "internal ", - "linkonce_odr ", - "linkonce ", - "weak_odr ", - "weak ", - "common ", - "available_externally ", - ] { - if let Some(rest) = s.strip_prefix(kw) { - return rest; - } - } - s -} - -/// Rewrite a module-global definition so it is safe to duplicate across -/// codegen units (#5391). Local-linkage (`private`/`internal`) and bare -/// external definitions are promoted to `linkonce_odr`, so the linker keeps a -/// single copy when the same global is emitted into multiple units. `external` -/// declarations (no initializer) are returned unchanged — duplicating a -/// declaration is harmless. - -/// Symbol name of a global/string definition line (`@name = ...`). -fn global_symbol_name(line: &str) -> Option<&str> { - let line = line.trim_start(); - if !line.starts_with('@') { - return None; - } - let end = line.find(" = ")?; - Some(&line[..end]) -} - -/// Collect every `@symbol` referenced in a chunk of IR text. -fn collect_symbol_refs(text: &str, out: &mut HashSet) { - let b = text.as_bytes(); - let mut i = 0usize; - while i < b.len() { - if b[i] == b'@' { - let start = i; - i += 1; - while i < b.len() - && (b[i].is_ascii_alphanumeric() || matches!(b[i], b'_' | b'.' | b'$' | b'-')) - { - i += 1; - } - if i > start + 1 { - out.insert(text[start..i].to_string()); - } - } else { - i += 1; - } - } -} - -fn metadata_definition_id(line: &str) -> Option { - let rest = line.trim_start().strip_prefix('!')?; - let (digits, _) = rest.split_once(" =")?; - digits.parse().ok() -} - -/// Collect numeric LLVM metadata references (`!123`) from instructions or -/// metadata definitions. Named metadata does not occur in Perry's alias tail. -fn collect_metadata_refs(text: &str, out: &mut HashSet) { - let bytes = text.as_bytes(); - let mut i = 0; - while i < bytes.len() { - if bytes[i] != b'!' || i + 1 >= bytes.len() || !bytes[i + 1].is_ascii_digit() { - i += 1; - continue; - } - let start = i + 1; - i = start; - while i < bytes.len() && bytes[i].is_ascii_digit() { - i += 1; - } - if let Ok(id) = text[start..i].parse() { - out.insert(id); - } - } -} - -fn promote_global_for_units(line: &str) -> String { - if line.contains(" = external ") { - return line.to_string(); - } - match line.split_once(" = ") { - Some((lhs, rhs)) => format!( - "{} = linkonce_odr {}", - lhs, - strip_leading_linkage(rhs.trim_start()) - ), - None => line.to_string(), - } -} - -/// Give a generated global one non-discardable definition. On every -/// non-Mach-O target (ELF and COFF — see `replicate_globals`) each global has -/// a unique owning codegen unit; leaving that sole definition as -/// `linkonce_odr` lets LLVM discard it when all references in the owner happen -/// to optimize away, even though other object files still reference it. -/// -/// The result is a plain STRONG definition with default visibility, so the -/// symbol's NAME must be unique across the whole program, not just the -/// module: `.str.N` constants only satisfy that through -/// [`LlModule::set_symbol_prefix`]. -fn make_unique_owner_global(line: &str) -> String { - if line.contains(" = external ") { - return line.to_string(); - } - match line.split_once(" = ") { - Some((lhs, rhs)) => format!("{} = {}", lhs, strip_leading_linkage(rhs.trim_start())), - None => line.to_string(), - } -} - -fn external_decl_for_global(line: &str) -> Option { - if line.contains(" = external ") { - return Some(line.to_string()); - } - let (name, rhs) = line.split_once(" = ")?; - let rhs = strip_leading_linkage(rhs.trim_start()); - let (kind, rest) = if let Some(rest) = rhs.strip_prefix("unnamed_addr constant ") { - ("constant", rest) - } else if let Some(rest) = rhs.strip_prefix("constant ") { - ("constant", rest) - } else if let Some(rest) = rhs.strip_prefix("global ") { - ("global", rest) - } else { - return None; - }; - let rest = rest.trim_start(); - let ty_end = match rest.as_bytes().first().copied() { - Some(b'[') | Some(b'{') | Some(b'<') => { - let (mut square, mut curly, mut angle) = (0i32, 0i32, 0i32); - let mut end = None; - for (i, b) in rest.bytes().enumerate() { - match b { - b'[' => square += 1, - b']' => square -= 1, - b'{' => curly += 1, - b'}' => curly -= 1, - b'<' => angle += 1, - b'>' => angle -= 1, - _ => {} - } - if square == 0 && curly == 0 && angle == 0 { - end = Some(i + 1); - break; - } - } - end? - } - _ => rest.find(char::is_whitespace).unwrap_or(rest.len()), - }; - Some(format!("{name} = external {kind} {}", &rest[..ty_end])) -} - -/// Attribute-group suffix for a runtime-helper `declare` line, keyed by -/// helper name (#6082 tranche 1). -/// -/// Without attributes, -O3 must treat every `js_*` call as "may read and -/// write all memory, may not return" — no CSE/LICM/DCE across any helper -/// call. The two groups below re-enable those optimizations for a small, -/// individually audited allowlist: -/// -/// * `#2` (PURE) = `nounwind willreturn readnone`. Invariant: the -/// helper's Rust body (transitively) performs NaN-box BIT manipulation -/// only — no loads, no stores, no allocation, no GC trigger, no -/// `js_throw`/longjmp, and it is total over arbitrary input bits (no -/// panic, no UB), so LLVM may CSE/hoist/sink/delete it freely. -/// * `#3` (READONLY) = `nounwind willreturn readonly`. Invariant: the -/// helper may READ heap memory (string headers, BigInt limbs) but never -/// writes, never allocates, never triggers GC, never takes a lock, and -/// never throws. LLVM may CSE/LICM it across write-free regions and -/// delete unused calls, but must still order it against any -/// possibly-writing call — which keeps it correct w.r.t. the moving GC, -/// because every GC-capable helper stays maximally clobbering. -/// -/// SYNTAX NOTE: the groups are spelled with the LEGACY `readnone` / -/// `readonly` function attributes, NOT the modern `memory(none)` / -/// `memory(read)` — old LLVM asm parsers (e.g. the Apple clang 15 shipped -/// on macos-14 CI runners, and any user clang predating LLVM's `memory` -/// attribute) reject the modern spelling with "unterminated attribute -/// group", which killed every `--backend llvm` compile through that clang -/// (caught by the simctl iOS smoke gating the v0.5.1265 release). New -/// parsers still accept the legacy spelling and auto-upgrade it to the -/// equivalent `memory(...)` form, so semantics are identical everywhere. -/// -/// SOUNDNESS NOTES (read before adding an entry): -/// * Deliberately reads-any (`readonly`, i.e. `memory(read)`), NOT an -/// argmem-scoped form: helper args are f64 NaN-boxes, not LLVM pointer -/// arguments, so `argmem` would mean "reads no memory at all" and -/// license CSE/DSE across real heap reads. -/// * Anything that can allocate or trigger GC gets NO group — the moving -/// GC's shadow-stack reload discipline depends on those calls staying -/// maximally clobbering. -/// * Anything that can reach `js_throw` (raises through the unwinder) gets NO group — -/// `willreturn` would let DCE delete a throwing call whose result is -/// unused, silently dropping the exception. -/// -/// Audited and rejected (do not re-add without a new audit): -/// `js_nanbox_string` (allocates an empty string for null input), -/// `js_get_string_pointer_unified` / `js_typed_string_arg_to_raw` -/// (materialize SSO strings onto the heap = allocation), -/// `js_typed_f64_arg_to_raw` (routes through `js_number_coerce`, whose -/// string/object paths read+parse and reach ToPrimitive), -/// `js_value_length_f64` (Buffer/TypedArray registry lookups take locks — -/// a lock acquisition writes memory). -pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { - match name { - // PURE — each verified: pure bit tests/masking on the f64/i64 args, - // total over arbitrary bits, no memory access anywhere in the body. - // js_nanbox_pointer value/nanbox.rs — tag ladder, 0 → TAG_NULL - // js_nanbox_get_pointer value/nanbox.rs — mask ladder, no deref - // js_typed_f64_arg_guard native_abi.rs — tag-band check - // js_typed_i32_arg_guard native_abi.rs — tag check + finite/fract/range - // js_typed_i1_arg_guard native_abi.rs — bits == TAG_TRUE|TAG_FALSE - // js_typed_i1_arg_to_raw native_abi.rs — bits == TAG_TRUE - // js_typed_i32_arg_to_raw native_abi.rs — bit extract / saturating cast - // js_typed_string_arg_guard native_abi.rs — STRING/SHORT_STRING tag check - "js_nanbox_pointer" - | "js_nanbox_get_pointer" - | "js_typed_f64_arg_guard" - | "js_typed_i32_arg_guard" - | "js_typed_i1_arg_guard" - | "js_typed_i1_arg_to_raw" - | "js_typed_i32_arg_to_raw" - | "js_typed_string_arg_guard" => " #2", - // READONLY — verified: tag ladder plus reads of StringHeader.utf16_len - // (via is_valid_string_ptr, a pure magnitude check) and BigInt limbs - // (js_bigint_is_zero via clean_bigint_ptr, pure bit cleanup). No - // registry/lock access, no allocation, no throw, no writes. - "js_is_truthy" => " #3", - // NOUNWIND+WILLRETURN only (#4, repsel Phase 4a.0) — each verified - // (`typed_feedback.rs` / `array/header.rs`): no `js_throw` (longjmp) - // anywhere in the body, every loop bounded by the 16M length/capacity - // sanity caps, no allocation, no GC trigger. They are NOT readonly: - // the numeric guards' first-touch path REBUILDS unmarked arrays into - // raw-f64 layout (slot writes + flag store), feedback mode - // (`PERRY_TYPED_FEEDBACK`, a runtime env check) records observations, - // and `js_array_numeric_value_to_raw_f64`'s ClassRef probe takes - // registry RwLock reads (a lock word write). #6082 trap notes apply: - // argmem is unsound for NaN-box args, and `willreturn` is only - // admissible because these helpers cannot reach `js_throw` — any - // divergence is a Rust panic-abort, which never resumes the program. - "js_typed_feedback_plain_array_index_get_guard" - | "js_typed_feedback_numeric_array_index_get_guard" - | "js_typed_feedback_plain_array_index_set_guard" - | "js_typed_feedback_numeric_array_index_set_guard" - | "js_typed_feedback_numeric_array_push_guard" - | "js_array_numeric_value_to_raw_f64" => " #4", - _ => "", - } -} - -/// Synthesize an external `declare` line matching a locally-defined function's -/// signature, so a codegen unit that calls it (but does not define it) resolves -/// the call at link time. -pub(crate) fn declare_line_for(f: &LlFunction) -> String { - let params = f - .params - .iter() - .map(|(t, _)| t.to_string()) - .collect::>() - .join(", "); - // #8175: a codegen unit that calls a promoted `preserve_nonecc` clone - // binds through this declare; the convention must ride along or the - // cross-unit ABI silently splits from the defining unit's. - let cconv: String = if f.is_preserve_none() { - format!("{} ", crate::inst::PRESERVE_NONE_CC) - } else { - String::new() - }; - format!("declare {}{} @{}({})", cconv, f.return_type, f.name, params) -} - -/// Render a function with external linkage forced, promoting an `internal` / -/// `private` definition so cross-unit calls can bind to it. Names are -/// module-prefixed and unique, so promotion never collides. -pub(crate) fn render_fn_external(f: &LlFunction) -> String { - render_fn_external_with_gc_leaf_callees(f, &HashSet::new()) -} - -pub(crate) fn render_fn_external_with_gc_leaf_callees( - f: &LlFunction, - gc_leaf_callees: &HashSet, -) -> String { - let ir = f.to_ir_with_gc_leaf_callees(gc_leaf_callees); - if f.linkage == "internal" || f.linkage == "private" { - return ir.replacen(&format!("define {} ", f.linkage), "define ", 1); - } - ir -} +mod linkage; +pub(crate) use linkage::*; fn push_statepoint_declarations(ir: &mut String) { ir.push_str( @@ -991,10 +694,12 @@ impl LlModule { /// the single giant translation unit that makes clang OOM on large bundles. /// /// The functions are split into `n` contiguous buckets. Every unit carries: - /// * the full string-constant + global set, with local-linkage and bare - /// external DEFINITIONS promoted to `linkonce_odr` (the linker keeps one - /// copy). Globals are a tiny fraction of a large module's IR, so the - /// duplication is cheap; `external` *declarations* are replicated as-is; + /// * the string constants + globals it references, with local-linkage + /// and bare external DEFINITIONS promoted to `linkonce_odr` when more + /// than one unit defines them (the linker keeps one copy) and left in + /// their original linkage otherwise (#9610). Globals are a tiny + /// fraction of a large module's IR, so the duplication is cheap; + /// `external` *declarations* are replicated as-is; /// * the module's external `declare`s plus a synthesized `declare` for /// every locally-defined function the unit does NOT itself define, so /// cross-unit calls resolve at link time (deduped by name, existing @@ -1048,16 +753,11 @@ impl LlModule { bucket_bytes[target] += sizes[i]; } - let shared_strings: Vec = self - .string_constants - .iter() - .map(|s| promote_global_for_units(s)) - .collect(); - let shared_globals: Vec = self - .globals - .iter() - .map(|g| promote_global_for_units(g)) - .collect(); + // Definitions are carried in their ORIGINAL linkage here; the + // duplicate-safe promotion below is applied per unit, and only to the + // globals that more than one unit actually defines (#9610). + let shared_strings: Vec = self.string_constants.clone(); + let shared_globals: Vec = self.globals.clone(); // name -> declare line. Existing module declarations (runtime, FFI, // cross-module) take precedence; every locally-defined function without @@ -1098,10 +798,11 @@ impl LlModule { // A global is emitted into every unit that REFERENCES it — normally // exactly one, and `linkonce_odr` lets the linker fold the rare - // multi-unit case. Definition-in-one-unit + `external` elsewhere was - // tried first and is subtly wrong under `-dead_strip`: the sole - // definition can be discarded with its unit's atoms while a live - // reference survives in another object. + // multi-unit case (only that case: see `defining_unit_count` below). + // Definition-in-one-unit + `external` elsewhere was tried first and is + // subtly wrong under `-dead_strip`: the sole definition can be + // discarded with its unit's atoms while a live reference survives in + // another object. let all_globals: Vec<&String> = shared_strings.iter().chain(shared_globals.iter()).collect(); // Globals reference OTHER globals in their initializers (a string @@ -1157,6 +858,32 @@ impl LlModule { }) .collect(); let replicate_globals = self.target_triple.contains("apple"); + // #9610: how many units end up DEFINING each global. Under the + // replicated (Mach-O) policy that is one unit per referencing bucket; + // the owner fallback keeps unreferenced globals at one. Only the + // globals a link would see twice need `linkonce_odr` to fold, and + // linkage is not free: LLVM's Mach-O section picker sends every + // weak-for-linker global to the coalesced *data* section, so a + // `zeroinitializer` global promoted for no reason leaves + // `__DATA,__bss` (zerofill, no file bytes) for file-backed + // `__DATA,__data`. Per-site inline caches are `[12 x i64] + // zeroinitializer` referenced by exactly one function each — 25.16 MB + // of literal zeros in the Claude Code binary's `__data`, 8.2% of the + // file, purely from the promotion. Only LOCAL-linkage definitions skip + // it (`has_local_linkage`) — that covers every generated cache and + // table, and keeps a strong external definition's cross-module + // coalescing exactly as it was. + let mut defining_unit_count: Vec = vec![0; all_globals.len()]; + if replicate_globals { + for need in &bucket_needs { + for &gi in need { + defining_unit_count[gi] += 1; + } + } + } + for count in &mut defining_unit_count { + *count = (*count).max(1); + } let unit_posts: Vec = bucket_metadata_refs .into_iter() @@ -1187,7 +914,11 @@ impl LlModule { let owns = global_owners[gi] == bi; if (replicate_globals && referenced) || owns { if replicate_globals { - pre.push_str(def); + if defining_unit_count[gi] > 1 || !has_local_linkage(def) { + pre.push_str(&promote_global_for_units(def)); + } else { + pre.push_str(def); + } } else { pre.push_str(&make_unique_owner_global(def)); } @@ -1707,6 +1438,93 @@ mod tests { assert!(global_unit.contains("declare double @__perry_wrap_extern_dep__value(i64)")); } + #[test] + fn mach_o_split_promotes_only_globals_two_units_define() { + // #9610: `linkonce_odr` is weak-for-linker, and + // `TargetLoweringObjectFileMachO::SelectSectionForGlobal` routes every + // weak-for-linker global to the coalesced DATA section before it ever + // asks whether the initializer is zero. So promoting a + // `zeroinitializer` global that only ONE unit defines moves it out of + // zerofill `__DATA,__bss` and writes its zeros into the file — 25.16 MB + // (8.2%) of the Claude Code binary, all of it per-site inline caches + // (`[12 x i64] zeroinitializer`, one per property-access site, each + // referenced by exactly one function and so by exactly one unit). + // Promote only what a link would otherwise see defined twice; ELF/COFF + // are unaffected either way (their BSS choice ignores linkage). + let mut m = LlModule::new("arm64-apple-macosx15.0.0"); + m.declare_function("js_ic_touch", VOID, &[PTR]); + m.add_raw_global("@perry_ic_m__0 = private global [12 x i64] zeroinitializer".to_string()); + m.add_raw_global("@perry_ic_m__1 = private global [12 x i64] zeroinitializer".to_string()); + m.add_internal_global("perry_class_keys_m__C", I64, "0"); + m.add_global("perry_class_shape_id_m__C", I32, "0"); + + // Two functions, one per unit under a 2-way split. Each touches its + // own cache; both touch the class-keys global, which therefore needs + // the linker to fold the two copies onto one storage. + for (name, ic) in [ + ("perry_fn_m__f", "@perry_ic_m__0"), + ("perry_fn_m__g", "@perry_ic_m__1"), + ] { + let f = m.define_function(name, DOUBLE, vec![]); + let e = f.create_block("entry"); + e.call_void("js_ic_touch", &[(PTR, ic)]); + e.call_void("js_ic_touch", &[(PTR, "@perry_class_keys_m__C")]); + if name == "perry_fn_m__f" { + e.call_void("js_ic_touch", &[(PTR, "@perry_class_shape_id_m__C")]); + } + e.ret(DOUBLE, "0.0"); + } + + let units = m.render_codegen_units(2); + assert_eq!(units.len(), 2, "two functions → two units"); + + for ic in ["@perry_ic_m__0", "@perry_ic_m__1"] { + let defs: Vec<&String> = units + .iter() + .filter(|u| { + u.contains(&format!("{ic} = private global [12 x i64] zeroinitializer")) + }) + .collect(); + assert_eq!( + defs.len(), + 1, + "{ic} is referenced by one function, so exactly one unit defines \ + it — in its original local linkage, which is what keeps it in __bss" + ); + for u in &units { + assert!( + !u.contains(&format!("{ic} = linkonce_odr")), + "{ic} must not be promoted: no second definition exists to fold" + ); + } + } + + // The genuinely shared global still gets the promotion — two strong + // copies of it in one link is a duplicate-symbol error, and two + // *local* copies would be two distinct storages for one runtime slot. + let shared_defs = units + .iter() + .filter(|u| u.contains("@perry_class_keys_m__C = linkonce_odr global i64 0")) + .count(); + assert_eq!( + shared_defs, 2, + "a global both units reference is defined in both, folded by linkage" + ); + + // A strong EXTERNAL definition keeps the promotion even at one unit: + // `linkonce_odr` is what lets ld64 coalesce two modules' same-named + // globals rather than report a duplicate symbol, and this change is + // about section placement, not about that. + let external_defs = units + .iter() + .filter(|u| u.contains("@perry_class_shape_id_m__C = linkonce_odr global i32 0")) + .count(); + assert_eq!( + external_defs, 1, + "a link-visible definition stays `linkonce_odr` however few units define it" + ); + } + #[test] fn duplicate_function_symbol_emitted_once() { // Two classes that sanitize to the same name produce a colliding diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs new file mode 100644 index 0000000000..e1593b5ac8 --- /dev/null +++ b/crates/perry-codegen/src/module/linkage.rs @@ -0,0 +1,332 @@ +//! LLVM linkage / symbol-reference helpers for `LlModule`, split from +//! `module.rs` for the 2000-line file cap. #9610 grew this block when +//! single-unit globals gained their own linkage so zero-init caches stay +//! in `__bss`. + +use std::collections::HashSet; + +use crate::function::LlFunction; + +/// Strip a leading LLVM linkage keyword from a global's post-`=` text, if +/// present. Linkage comes before `unnamed_addr`/`constant`/`global` in the +/// grammar, so this leaves the rest of the definition intact. +pub(crate) fn strip_leading_linkage(s: &str) -> &str { + for kw in [ + "private ", + "internal ", + "linkonce_odr ", + "linkonce ", + "weak_odr ", + "weak ", + "common ", + "available_externally ", + ] { + if let Some(rest) = s.strip_prefix(kw) { + return rest; + } + } + s +} + +/// Symbol name of a global/string definition line (`@name = ...`). +pub(crate) fn global_symbol_name(line: &str) -> Option<&str> { + let line = line.trim_start(); + if !line.starts_with('@') { + return None; + } + let end = line.find(" = ")?; + Some(&line[..end]) +} + +/// Collect every `@symbol` referenced in a chunk of IR text. +pub(crate) fn collect_symbol_refs(text: &str, out: &mut HashSet) { + let b = text.as_bytes(); + let mut i = 0usize; + while i < b.len() { + if b[i] == b'@' { + let start = i; + i += 1; + while i < b.len() + && (b[i].is_ascii_alphanumeric() || matches!(b[i], b'_' | b'.' | b'$' | b'-')) + { + i += 1; + } + if i > start + 1 { + out.insert(text[start..i].to_string()); + } + } else { + i += 1; + } + } +} + +pub(crate) fn metadata_definition_id(line: &str) -> Option { + let rest = line.trim_start().strip_prefix('!')?; + let (digits, _) = rest.split_once(" =")?; + digits.parse().ok() +} + +/// Collect numeric LLVM metadata references (`!123`) from instructions or +/// metadata definitions. Named metadata does not occur in Perry's alias tail. +pub(crate) fn collect_metadata_refs(text: &str, out: &mut HashSet) { + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] != b'!' || i + 1 >= bytes.len() || !bytes[i + 1].is_ascii_digit() { + i += 1; + continue; + } + let start = i + 1; + i = start; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + if let Ok(id) = text[start..i].parse() { + out.insert(id); + } + } +} + +/// True when a global definition already carries LOCAL linkage +/// (`private`/`internal`). Those are the only definitions whose promotion +/// `codegen_unit_parts` may skip: a local symbol cannot collide with anything +/// — not another unit of this module, not another module's copy of a +/// same-named global — so leaving it alone is inert at link time. Dropping the +/// promotion on a strong external definition would NOT be: `linkonce_odr` is +/// what lets ld64 coalesce two modules' same-named globals instead of +/// reporting a duplicate symbol. +pub(crate) fn has_local_linkage(line: &str) -> bool { + match line.split_once(" = ") { + Some((_, rhs)) => { + let rhs = rhs.trim_start(); + rhs.starts_with("private ") || rhs.starts_with("internal ") + } + None => false, + } +} + +/// Rewrite a module-global definition so it is safe to duplicate across +/// codegen units (#5391). Local-linkage (`private`/`internal`) and bare +/// external definitions are promoted to `linkonce_odr`, so the linker keeps a +/// single copy when the same global is emitted into multiple units. `external` +/// declarations (no initializer) are returned unchanged — duplicating a +/// declaration is harmless. +/// +/// Apply this ONLY to a global that really is emitted more than once (#9610). +/// `linkonce_odr` is weak-for-linker, and `TargetLoweringObjectFileMachO` +/// routes every weak-for-linker global to the coalesced data section before it +/// ever consults `SectionKind::isBSS()` — so promoting a `zeroinitializer` +/// global that only one unit defines moves it out of zerofill `__DATA,__bss` +/// and writes its zeros into the file. +pub(crate) fn promote_global_for_units(line: &str) -> String { + if line.contains(" = external ") { + return line.to_string(); + } + match line.split_once(" = ") { + Some((lhs, rhs)) => format!( + "{} = linkonce_odr {}", + lhs, + strip_leading_linkage(rhs.trim_start()) + ), + None => line.to_string(), + } +} + +/// Give a generated global one non-discardable definition. On every +/// non-Mach-O target (ELF and COFF — see `replicate_globals`) each global has +/// a unique owning codegen unit; leaving that sole definition as +/// `linkonce_odr` lets LLVM discard it when all references in the owner happen +/// to optimize away, even though other object files still reference it. +/// +/// The result is a plain STRONG definition with default visibility, so the +/// symbol's NAME must be unique across the whole program, not just the +/// module: `.str.N` constants only satisfy that through +/// [`LlModule::set_symbol_prefix`]. +pub(crate) fn make_unique_owner_global(line: &str) -> String { + if line.contains(" = external ") { + return line.to_string(); + } + match line.split_once(" = ") { + Some((lhs, rhs)) => format!("{} = {}", lhs, strip_leading_linkage(rhs.trim_start())), + None => line.to_string(), + } +} + +pub(crate) fn external_decl_for_global(line: &str) -> Option { + if line.contains(" = external ") { + return Some(line.to_string()); + } + let (name, rhs) = line.split_once(" = ")?; + let rhs = strip_leading_linkage(rhs.trim_start()); + let (kind, rest) = if let Some(rest) = rhs.strip_prefix("unnamed_addr constant ") { + ("constant", rest) + } else if let Some(rest) = rhs.strip_prefix("constant ") { + ("constant", rest) + } else if let Some(rest) = rhs.strip_prefix("global ") { + ("global", rest) + } else { + return None; + }; + let rest = rest.trim_start(); + let ty_end = match rest.as_bytes().first().copied() { + Some(b'[') | Some(b'{') | Some(b'<') => { + let (mut square, mut curly, mut angle) = (0i32, 0i32, 0i32); + let mut end = None; + for (i, b) in rest.bytes().enumerate() { + match b { + b'[' => square += 1, + b']' => square -= 1, + b'{' => curly += 1, + b'}' => curly -= 1, + b'<' => angle += 1, + b'>' => angle -= 1, + _ => {} + } + if square == 0 && curly == 0 && angle == 0 { + end = Some(i + 1); + break; + } + } + end? + } + _ => rest.find(char::is_whitespace).unwrap_or(rest.len()), + }; + Some(format!("{name} = external {kind} {}", &rest[..ty_end])) +} + +/// Attribute-group suffix for a runtime-helper `declare` line, keyed by +/// helper name (#6082 tranche 1). +/// +/// Without attributes, -O3 must treat every `js_*` call as "may read and +/// write all memory, may not return" — no CSE/LICM/DCE across any helper +/// call. The two groups below re-enable those optimizations for a small, +/// individually audited allowlist: +/// +/// * `#2` (PURE) = `nounwind willreturn readnone`. Invariant: the +/// helper's Rust body (transitively) performs NaN-box BIT manipulation +/// only — no loads, no stores, no allocation, no GC trigger, no +/// `js_throw`/longjmp, and it is total over arbitrary input bits (no +/// panic, no UB), so LLVM may CSE/hoist/sink/delete it freely. +/// * `#3` (READONLY) = `nounwind willreturn readonly`. Invariant: the +/// helper may READ heap memory (string headers, BigInt limbs) but never +/// writes, never allocates, never triggers GC, never takes a lock, and +/// never throws. LLVM may CSE/LICM it across write-free regions and +/// delete unused calls, but must still order it against any +/// possibly-writing call — which keeps it correct w.r.t. the moving GC, +/// because every GC-capable helper stays maximally clobbering. +/// +/// SYNTAX NOTE: the groups are spelled with the LEGACY `readnone` / +/// `readonly` function attributes, NOT the modern `memory(none)` / +/// `memory(read)` — old LLVM asm parsers (e.g. the Apple clang 15 shipped +/// on macos-14 CI runners, and any user clang predating LLVM's `memory` +/// attribute) reject the modern spelling with "unterminated attribute +/// group", which killed every `--backend llvm` compile through that clang +/// (caught by the simctl iOS smoke gating the v0.5.1265 release). New +/// parsers still accept the legacy spelling and auto-upgrade it to the +/// equivalent `memory(...)` form, so semantics are identical everywhere. +/// +/// SOUNDNESS NOTES (read before adding an entry): +/// * Deliberately reads-any (`readonly`, i.e. `memory(read)`), NOT an +/// argmem-scoped form: helper args are f64 NaN-boxes, not LLVM pointer +/// arguments, so `argmem` would mean "reads no memory at all" and +/// license CSE/DSE across real heap reads. +/// * Anything that can allocate or trigger GC gets NO group — the moving +/// GC's shadow-stack reload discipline depends on those calls staying +/// maximally clobbering. +/// * Anything that can reach `js_throw` (raises through the unwinder) gets NO group — +/// `willreturn` would let DCE delete a throwing call whose result is +/// unused, silently dropping the exception. +/// +/// Audited and rejected (do not re-add without a new audit): +/// `js_nanbox_string` (allocates an empty string for null input), +/// `js_get_string_pointer_unified` / `js_typed_string_arg_to_raw` +/// (materialize SSO strings onto the heap = allocation), +/// `js_typed_f64_arg_to_raw` (routes through `js_number_coerce`, whose +/// string/object paths read+parse and reach ToPrimitive), +/// `js_value_length_f64` (Buffer/TypedArray registry lookups take locks — +/// a lock acquisition writes memory). +pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { + match name { + // PURE — each verified: pure bit tests/masking on the f64/i64 args, + // total over arbitrary bits, no memory access anywhere in the body. + // js_nanbox_pointer value/nanbox.rs — tag ladder, 0 → TAG_NULL + // js_nanbox_get_pointer value/nanbox.rs — mask ladder, no deref + // js_typed_f64_arg_guard native_abi.rs — tag-band check + // js_typed_i32_arg_guard native_abi.rs — tag check + finite/fract/range + // js_typed_i1_arg_guard native_abi.rs — bits == TAG_TRUE|TAG_FALSE + // js_typed_i1_arg_to_raw native_abi.rs — bits == TAG_TRUE + // js_typed_i32_arg_to_raw native_abi.rs — bit extract / saturating cast + // js_typed_string_arg_guard native_abi.rs — STRING/SHORT_STRING tag check + "js_nanbox_pointer" + | "js_nanbox_get_pointer" + | "js_typed_f64_arg_guard" + | "js_typed_i32_arg_guard" + | "js_typed_i1_arg_guard" + | "js_typed_i1_arg_to_raw" + | "js_typed_i32_arg_to_raw" + | "js_typed_string_arg_guard" => " #2", + // READONLY — verified: tag ladder plus reads of StringHeader.utf16_len + // (via is_valid_string_ptr, a pure magnitude check) and BigInt limbs + // (js_bigint_is_zero via clean_bigint_ptr, pure bit cleanup). No + // registry/lock access, no allocation, no throw, no writes. + "js_is_truthy" => " #3", + // NOUNWIND+WILLRETURN only (#4, repsel Phase 4a.0) — each verified + // (`typed_feedback.rs` / `array/header.rs`): no `js_throw` (longjmp) + // anywhere in the body, every loop bounded by the 16M length/capacity + // sanity caps, no allocation, no GC trigger. They are NOT readonly: + // the numeric guards' first-touch path REBUILDS unmarked arrays into + // raw-f64 layout (slot writes + flag store), feedback mode + // (`PERRY_TYPED_FEEDBACK`, a runtime env check) records observations, + // and `js_array_numeric_value_to_raw_f64`'s ClassRef probe takes + // registry RwLock reads (a lock word write). #6082 trap notes apply: + // argmem is unsound for NaN-box args, and `willreturn` is only + // admissible because these helpers cannot reach `js_throw` — any + // divergence is a Rust panic-abort, which never resumes the program. + "js_typed_feedback_plain_array_index_get_guard" + | "js_typed_feedback_numeric_array_index_get_guard" + | "js_typed_feedback_plain_array_index_set_guard" + | "js_typed_feedback_numeric_array_index_set_guard" + | "js_typed_feedback_numeric_array_push_guard" + | "js_array_numeric_value_to_raw_f64" => " #4", + _ => "", + } +} + +/// Synthesize an external `declare` line matching a locally-defined function's +/// signature, so a codegen unit that calls it (but does not define it) resolves +/// the call at link time. +pub(crate) fn declare_line_for(f: &LlFunction) -> String { + let params = f + .params + .iter() + .map(|(t, _)| t.to_string()) + .collect::>() + .join(", "); + // #8175: a codegen unit that calls a promoted `preserve_nonecc` clone + // binds through this declare; the convention must ride along or the + // cross-unit ABI silently splits from the defining unit's. + let cconv: String = if f.is_preserve_none() { + format!("{} ", crate::inst::PRESERVE_NONE_CC) + } else { + String::new() + }; + format!("declare {}{} @{}({})", cconv, f.return_type, f.name, params) +} + +/// Render a function with external linkage forced, promoting an `internal` / +/// `private` definition so cross-unit calls can bind to it. Names are +/// module-prefixed and unique, so promotion never collides. +pub(crate) fn render_fn_external(f: &LlFunction) -> String { + render_fn_external_with_gc_leaf_callees(f, &HashSet::new()) +} + +pub(crate) fn render_fn_external_with_gc_leaf_callees( + f: &LlFunction, + gc_leaf_callees: &HashSet, +) -> String { + let ir = f.to_ir_with_gc_leaf_callees(gc_leaf_callees); + if f.linkage == "internal" || f.linkage == "private" { + return ir.replacen(&format!("define {} ", f.linkage), "define ", 1); + } + ir +} diff --git a/crates/perry-parser/src/lib.rs b/crates/perry-parser/src/lib.rs index 0cd570e7d6..39ab6dccd0 100644 --- a/crates/perry-parser/src/lib.rs +++ b/crates/perry-parser/src/lib.rs @@ -7,7 +7,7 @@ use anyhow::Result; use perry_diagnostics::{Diagnostic, DiagnosticCode, Diagnostics, FileId, SourceCache, Span}; use std::path::Path; use swc_common::{input::StringInput, sync::Lrc, BytePos, FileName, SourceMap}; -use swc_ecma_ast::{Module, ModuleItem, Script}; +use swc_ecma_ast::{Module, ModuleItem, Program, Script}; use swc_ecma_parser::{lexer::Lexer, EsSyntax, Parser, Syntax, TsSyntax}; use swc_ecma_visit::{VisitMut, VisitMutWith}; @@ -145,7 +145,7 @@ fn parse_source_file_with_typescript_fallback<'a>( let is_typescript = matches!(syntax, Syntax::Typescript(_)); let mut parser = parser_for_source_file_with_syntax(source_file, syntax); - match parse_module_or_script(&mut parser, filename, source) { + match parse_module_or_script(&mut parser, filename) { Ok(module) => Ok((module, parser)), Err(first_error) => { if !is_typescript && source_looks_like_typescript(source) { @@ -153,7 +153,7 @@ fn parse_source_file_with_typescript_fallback<'a>( source_file, typescript_syntax_for_filename(filename), ); - if let Ok(module) = parse_module_or_script(&mut retry_parser, filename, source) { + if let Ok(module) = parse_module_or_script(&mut retry_parser, filename) { return Ok((module, retry_parser)); } } @@ -316,24 +316,24 @@ fn source_looks_like_typescript(source: &str) -> bool { fn parse_module_or_script( parser: &mut Parser>, filename: &str, - source: &str, ) -> swc_ecma_parser::PResult { - if should_parse_as_script(filename, source) { - parser.parse_script().map(script_to_module) + if should_parse_unambiguous_program(filename) { + // Let SWC's lexer and parser identify real top-level module items. In + // particular, this keeps template expressions, nested templates, and + // regex literals synchronized without maintaining a second partial + // JavaScript lexer here. + parser.parse_program().map(program_to_module) } else { parser.parse_module() } } -fn should_parse_as_script(filename: &str, source: &str) -> bool { +fn should_parse_unambiguous_program(filename: &str) -> bool { let path = path_for_extension_check(filename); if !(path.ends_with(".js") || path.ends_with(".cjs") || path.ends_with(".jsx")) { return false; } - if !path.ends_with(".cjs") && file_is_in_esm_package_context(path) { - return false; - } - !looks_like_es_module(source) + path.ends_with(".cjs") || !file_is_in_esm_package_context(path) } /// Whether `filename` is an ES module purely by its module FORMAT — i.e. @@ -344,7 +344,7 @@ fn should_parse_as_script(filename: &str, source: &str) -> bool { /// `"type":"module"`); an ambiguous extension (`.js`/`.ts`/`.jsx`/`.tsx`) is a /// module when it sits in an ESM package context (`"type":"module"` / /// conditional-export map). Mirrors the format half of Node's CJS-vs-ESM -/// detection and of `should_parse_as_script`'s package-context guard. +/// detection and of `should_parse_unambiguous_program`'s package-context guard. /// /// Module code is strict-mode code, so lowering consults this to decide the /// runtime strictness of a file that carries no in-file module syntax (#6542): @@ -361,205 +361,6 @@ pub fn file_is_es_module_by_format(filename: &str) -> bool { file_is_in_esm_package_context(path) } -fn looks_like_es_module(source: &str) -> bool { - #[derive(Clone, Copy, PartialEq, Eq)] - enum State { - Code, - String(u8), - LineComment, - BlockComment, - } - - fn is_ident(b: u8) -> bool { - b == b'_' || b == b'$' || b.is_ascii_alphanumeric() - } - - // Whether a top-level `import`/`export` keyword found here can begin a - // module item, given `last_sig` — the last significant *code* byte seen so - // far (0 = start of input). A module item starts at input start or right - // after a statement boundary (`;`, `{`, `}`); anything else (an operator, an - // identifier byte, a string/regex terminator) means the keyword is part of a - // larger expression and not a real `import`/`export` statement. - // - // `last_sig` is tracked during the forward scan rather than recovered by - // walking the raw bytes backward, because a backward walk cannot tell that - // the preceding bytes were inside a comment. Bundler chunks almost always - // open with a banner comment (`// chunk-….js`) immediately followed by a - // top-level `export`/`import`; a raw backward walk would see the comment's - // last character (e.g. the `)` of "(cross-chunk re-export)") and wrongly - // conclude the keyword can't start a module item, so the `.js` chunk parsed - // as a Script and SWC raised `ImportExportInScript` (issue #5207). - fn allows_module_item(last_sig: u8) -> bool { - matches!(last_sig, 0 | b';' | b'{' | b'}') - } - - fn next_after_keyword(bytes: &[u8], i: usize, keyword: &[u8]) -> Option { - let end = i.checked_add(keyword.len())?; - if bytes.get(i..end)? != keyword { - return None; - } - if i > 0 && is_ident(bytes[i - 1]) { - return None; - } - if bytes.get(end).is_some_and(|b| is_ident(*b)) { - return None; - } - Some(end) - } - - // A `/` starts a regex literal (not division) when the preceding token - // cannot end an expression: an operator/punctuator, start of input, or a - // keyword like `return`. Regex literals may contain unescaped quote chars - // (e.g. picomatch's `/(^[*!]|[/()[\]{}"])/`), which would desync the - // string-state scan below if skipped as ordinary code. - fn regex_can_start_here(bytes: &[u8], slash_at: usize) -> bool { - let mut i = slash_at; - while i > 0 { - i -= 1; - match bytes[i] { - b' ' | b'\t' | b'\r' | b'\n' => continue, - b'=' | b'(' | b',' | b':' | b'[' | b'!' | b'&' | b'|' | b'?' | b'{' | b'}' - | b';' | b'+' | b'-' | b'*' | b'%' | b'~' | b'^' | b'<' | b'>' => return true, - c if is_ident(c) => { - let end = i + 1; - let mut start = end; - while start > 0 && is_ident(bytes[start - 1]) { - start -= 1; - } - return matches!( - &bytes[start..end], - b"return" - | b"typeof" - | b"instanceof" - | b"in" - | b"of" - | b"case" - | b"do" - | b"else" - | b"void" - | b"delete" - | b"throw" - | b"new" - | b"yield" - | b"await" - ); - } - _ => return false, - } - } - true - } - - // Returns the index just past the closing `/`, or None if no regex - // terminator is found on this line (then it was division after all). - fn skip_regex_literal(bytes: &[u8], slash_at: usize) -> Option { - let mut i = slash_at + 1; - let mut in_class = false; - while i < bytes.len() { - match bytes[i] { - b'\\' => i += 2, - b'\n' => return None, - b'[' => { - in_class = true; - i += 1; - } - b']' => { - in_class = false; - i += 1; - } - b'/' if !in_class => return Some(i + 1), - _ => i += 1, - } - } - None - } - - let bytes = source.as_bytes(); - let mut i = 0; - let mut state = State::Code; - // Last significant code byte seen (0 = start of input). Comments are - // transparent — they never update this — so a banner comment before a - // top-level `import`/`export` no longer hides the keyword. Strings and - // regex literals leave their terminator (`"`/`'`/`` ` ``/`/`) as the last - // significant byte, matching the old backward walk's behavior. - let mut last_sig: u8 = 0; - while i < bytes.len() { - match state { - State::Code => { - if bytes[i] == b'\'' || bytes[i] == b'"' || bytes[i] == b'`' { - state = State::String(bytes[i]); - i += 1; - } else if bytes[i] == b'/' && bytes.get(i + 1) == Some(&b'/') { - state = State::LineComment; - i += 2; - } else if bytes[i] == b'/' && bytes.get(i + 1) == Some(&b'*') { - state = State::BlockComment; - i += 2; - } else if bytes[i] == b'/' && regex_can_start_here(bytes, i) { - match skip_regex_literal(bytes, i) { - Some(end) => i = end, - None => i += 1, - } - // A regex literal (or a `/` division operator) is an - // expression token — a following keyword can't begin a - // module item. - last_sig = b'/'; - } else { - if allows_module_item(last_sig) { - if let Some(end) = next_after_keyword(bytes, i, b"export") { - if matches!( - bytes.get(end), - Some(b' ' | b'\t' | b'\r' | b'\n' | b'{' | b'*') - ) { - return true; - } - } - if let Some(end) = next_after_keyword(bytes, i, b"import") { - if matches!( - bytes.get(end), - Some(b' ' | b'\t' | b'\r' | b'\n' | b'{' | b'*' | b'"' | b'\'') - ) || bytes.get(end) == Some(&b'.') - { - return true; - } - } - } - if !matches!(bytes[i], b' ' | b'\t' | b'\r' | b'\n') { - last_sig = bytes[i]; - } - i += 1; - } - } - State::String(quote) => { - if bytes[i] == b'\\' { - i += 2; - } else { - if bytes[i] == quote { - state = State::Code; - last_sig = quote; - } - i += 1; - } - } - State::LineComment => { - if bytes[i] == b'\n' { - state = State::Code; - } - i += 1; - } - State::BlockComment => { - if bytes[i] == b'*' && bytes.get(i + 1) == Some(&b'/') { - i += 2; - state = State::Code; - } else { - i += 1; - } - } - } - } - false -} - fn file_is_in_esm_package_context(filename: &str) -> bool { let path = Path::new(filename); let mut current = Path::new(filename).parent(); @@ -605,6 +406,13 @@ fn script_to_module(script: Script) -> Module { } } +fn program_to_module(program: Program) -> Module { + match program { + Program::Module(module) => module, + Program::Script(script) => script_to_module(script), + } +} + #[cfg(test)] fn normalize_unicode_identifier_escapes(source: &str) -> String { normalize_unicode_identifier_escapes_with_metadata(source).source @@ -1096,24 +904,96 @@ mod tests { use super::*; #[test] - fn test_looks_like_es_module_survives_regex_with_quote() { + fn test_es_module_detection_survives_regex_with_quote() { // Regression: picomatch's bundled source contains a regex literal with - // an unescaped `"` inside a character class. The module-detection scan - // must not enter string state there, or a trailing `export` (appended - // by the CJS wrap) is missed and the file parses as a Script. + // an unescaped `"` inside a character class. Module detection must lex + // the whole regex before looking for a trailing `export` appended by + // the CJS wrap. let source = "const re = /(^[*!]|[/()[\\]{}\"])/;\nconst x = \"ok\";\nexport default x;\n"; let module = parse_typescript(source, "vendored.js").unwrap(); assert_eq!(module.body.len(), 3); + assert!(matches!( + module.body.last(), + Some(ModuleItem::ModuleDecl(_)) + )); + } + + #[test] + fn test_es_module_detection_handles_template_interpolations() { + // Regression for #9608: the old byte scanner treated a whole template + // as an opaque string. A backtick in an interpolation regex could end + // that fake string early and hide a trailing export. SWC's program + // parser must classify all of these as modules. + let cases = [ + ( + "issue repro", + r#"const value = `${"x".replace(/[`].*$/, "")}`; +export { value }; +"#, + ), + ( + "nested expression braces", + r#"const value = `${({ nested: { value: "`" } }).nested.value}`; +export { value }; +"#, + ), + ( + "nested template", + r#"const value = `outer ${`inner ${String(/[`]/)}`}`; +export { value }; +"#, + ), + ( + "escaped template backtick", + r#"const value = `escaped \` ${"`"}`; +export { value }; +"#, + ), + ( + "regex backtick and interpolation opener in character class", + r#"const value = `${/[`${}]/.test("`")}`; +export { value }; +"#, + ), + ( + "comment and string in interpolation", + r#"const value = `${(() => { + /* ` and ${ are inert here */ + return "` and ${ are inert here"; +})()}`; +export { value }; +"#, + ), + ( + "division followed by regex", + r#"const value = `${10 / 2 + /[`]/.source.length}`; +export { value }; +"#, + ), + ]; + + for (name, source) in cases { + let mut cache = SourceCache::new(); + let result = parse_typescript_with_cache(source, "ambiguous.js", &mut cache) + .unwrap_or_else(|error| panic!("{name} did not parse as a module: {error:?}")); + assert!( + result.diagnostics.is_empty(), + "{name} produced diagnostics: {:?}", + result.diagnostics + ); + assert!(matches!( + result.module.body.last(), + Some(ModuleItem::ModuleDecl(_)) + )); + } } #[test] fn test_banner_comment_before_top_level_export_is_module() { // Regression for #5207: a bundler code-split chunk almost always opens // with a banner comment immediately followed by a top-level `export` - // (or `import`). The module-detection scan must look through the comment - // — its last character (here the `)` of "(cross-chunk re-export)") must - // not be mistaken for a preceding code token that bars a module item, or - // the `.js` chunk parses as a Script and SWC raises ImportExportInScript. + // (or `import`). Module detection must look through the comment rather + // than treating its last character as code before the module item. let cases = [ "// runtime chunk (cross-chunk re-export)\nexport function rt(x) { return x; }\n", "// banner foo\nexport const V = 1;\n", @@ -1121,28 +1001,46 @@ mod tests { "// a\n// b\n// c\nexport { y } from \"./other.js\";\n", ]; for src in cases { - assert!( - looks_like_es_module(src), - "expected ESM classification for chunk:\n{src}" - ); - // And it must actually parse as a module rather than a Script. - parse_typescript(src, "chunk-abc.js") + let module = parse_typescript(src, "chunk-abc.js") .unwrap_or_else(|e| panic!("chunk failed to parse as a module {src:?}: {e:?}")); + assert!(matches!( + module.body.last(), + Some(ModuleItem::ModuleDecl(_)) + )); } } #[test] fn test_comment_does_not_create_false_module_classification() { - // The transparency fix must not flip a genuinely CommonJS chunk to ESM: - // a comment ending in `;`/`{`/`}` followed by a non-keyword leaves the - // file a Script, and `exportFoo`/`importMap`-style identifiers after a - // comment still don't match the `export`/`import` keywords. - assert!(!looks_like_es_module( - "// helper;\nconst exportFoo = 1;\nmodule.exports = exportFoo;\n" - )); - assert!(!looks_like_es_module( - "// note\nconst importMap = {};\nmodule.exports = importMap;\n" - )); + // A comment ending in `;`/`{`/`}` followed by a non-keyword must leave a + // genuine CommonJS chunk as a sloppy Script. `exportFoo`/`importMap` + // identifiers and template contents are not module items. + let cases = [ + "// helper;\nconst exportFoo = 1;\nwith ({}) {}\nmodule.exports = exportFoo;\n", + "// note\nconst importMap = {};\nwith ({}) {}\nmodule.exports = importMap;\n", + r#"const value = `${"`" + /[`${}]/.source}`; +with ({}) {} +module.exports = value; +"#, + ]; + for source in cases { + let mut cache = SourceCache::new(); + let result = parse_typescript_with_cache(source, "chunk.js", &mut cache) + .unwrap_or_else(|error| panic!("CommonJS source failed to parse: {error:?}")); + assert!( + result.diagnostics.is_empty(), + "CommonJS source was parsed as strict module code: {:?}", + result.diagnostics + ); + assert!( + result + .module + .body + .iter() + .all(|item| matches!(item, ModuleItem::Stmt(_))), + "CommonJS source unexpectedly contained a module declaration" + ); + } } #[test] diff --git a/crates/perry/Cargo.toml b/crates/perry/Cargo.toml index d5ce3e4e0f..94b99fbc6c 100644 --- a/crates/perry/Cargo.toml +++ b/crates/perry/Cargo.toml @@ -21,7 +21,10 @@ path = "src/main.rs" [dependencies] perry-parser.workspace = true +swc_common.workspace = true swc_ecma_ast.workspace = true +swc_ecma_transforms_base.workspace = true +swc_ecma_visit.workspace = true perry-hir.workspace = true perry-transform.workspace = true perry-codegen.workspace = true @@ -172,7 +175,6 @@ all-codegen-backends = [ [dev-dependencies] tempfile.workspace = true -swc_common.workspace = true [build-dependencies] winresource = "0.1.31" diff --git a/crates/perry/src/commands/compile/collect_modules.rs b/crates/perry/src/commands/compile/collect_modules.rs index 82d0dc6ace..c50faaee4b 100644 --- a/crates/perry/src/commands/compile/collect_modules.rs +++ b/crates/perry/src/commands/compile/collect_modules.rs @@ -36,6 +36,7 @@ mod dynamic_glob; mod eval_worker; mod feature_detect; mod import_helpers; +mod import_meta_require; mod json_module; mod native_addon; mod parse_error; @@ -56,6 +57,7 @@ use import_helpers::{ cached_resolve_import_with_lexical_base, collect_js_module_imports, ensure_bunfs_import_resolves, env_defines_for_lowering, }; +use import_meta_require::rewrite_import_meta_require_addons; use json_module::synthesize_json_module; pub(super) use native_addon::package_has_unsupported_node_addon; use native_addon::{collect_or_refuse_node_addon, refuse_compile_package_native_addon}; @@ -338,6 +340,13 @@ fn collect_module_one( fs::read_to_string(&canonical) .map_err(|e| anyhow!("Failed to read {}: {}", canonical.display(), e))? }; + // Bun exposes `import.meta.require`; unlike CommonJS `require`, aliases of + // that function and URL-derived addon paths are invisible to the ordinary + // static-require scan. Recover and rewrite exact Node-API loads before HIR + // lowering so runtime execution uses only the authenticated sidecar id. + // Run before the Bun virtual-literal asset scan so a `.node` call target + // ships once in the sidecar rather than also being embedded as inert data. + let raw_source = rewrite_import_meta_require_addons(&raw_source, &canonical, ctx)?; register_bunfs_literal_assets(&raw_source, ctx); // CJS wrapping consumes literal `require()` sites and replaces them with // generated loader calls. Queue native targets before that rewrite so the diff --git a/crates/perry/src/commands/compile/collect_modules/import_meta_require.rs b/crates/perry/src/commands/compile/collect_modules/import_meta_require.rs new file mode 100644 index 0000000000..7a68082bac --- /dev/null +++ b/crates/perry/src/commands/compile/collect_modules/import_meta_require.rs @@ -0,0 +1,485 @@ +//! Static recovery for Bun's `import.meta.require` Node-API addon loads. +//! +//! Bun standalone extraction commonly aliases the function and computes an +//! absolute path through `new URL("./addon.node", import.meta.url).pathname`. +//! Neither shape is visible to the CommonJS `require()` scanner. This pass +//! follows immutable local aliases/path constants, authorizes the resolved +//! binary, and replaces the load with `process.dlopen` against the sidecar's +//! portable logical id. + +use anyhow::{anyhow, Result}; +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use swc_common::{Globals, Mark, SyntaxContext, GLOBALS}; +use swc_ecma_ast as ast; +use swc_ecma_ast::Pass; +use swc_ecma_transforms_base::resolver; +use swc_ecma_visit::{Visit, VisitWith}; + +use super::native_addon::collect_node_addon_request; +use super::static_require_transform::resolve_static_require; +use super::CompilationContext; + +fn strip_transparent_expr(mut expression: &ast::Expr) -> &ast::Expr { + loop { + expression = match expression { + ast::Expr::Paren(value) => &value.expr, + ast::Expr::TsAs(value) => &value.expr, + ast::Expr::TsNonNull(value) => &value.expr, + ast::Expr::TsTypeAssertion(value) => &value.expr, + ast::Expr::TsConstAssertion(value) => &value.expr, + ast::Expr::TsSatisfies(value) => &value.expr, + ast::Expr::TsInstantiation(value) => &value.expr, + _ => return expression, + }; + } +} + +fn static_member_name(member: &ast::MemberExpr) -> Option<&str> { + match &member.prop { + ast::MemberProp::Ident(name) => Some(name.sym.as_ref()), + ast::MemberProp::Computed(name) => match strip_transparent_expr(&name.expr) { + ast::Expr::Lit(ast::Lit::Str(value)) => value.value.as_str(), + _ => None, + }, + ast::MemberProp::PrivateName(_) => None, + } +} + +fn is_import_meta_member(expression: &ast::Expr, expected: &str) -> bool { + let ast::Expr::Member(member) = strip_transparent_expr(expression) else { + return false; + }; + static_member_name(member) == Some(expected) + && matches!( + strip_transparent_expr(&member.obj), + ast::Expr::MetaProp(meta) if meta.kind == ast::MetaPropKind::ImportMeta + ) +} + +fn identifier_id(expression: &ast::Expr) -> Option { + match strip_transparent_expr(expression) { + ast::Expr::Ident(identifier) => Some(identifier.to_id()), + _ => None, + } +} + +fn url_pathname_specifier( + expression: &ast::Expr, + unresolved_ctxt: SyntaxContext, +) -> Option { + let ast::Expr::Member(pathname) = strip_transparent_expr(expression) else { + return None; + }; + if static_member_name(pathname) != Some("pathname") { + return None; + } + let ast::Expr::New(url) = strip_transparent_expr(&pathname.obj) else { + return None; + }; + let ast::Expr::Ident(constructor) = strip_transparent_expr(&url.callee) else { + return None; + }; + if constructor.sym.as_ref() != "URL" || constructor.ctxt != unresolved_ctxt { + return None; + } + let arguments = url.args.as_ref()?; + if arguments.len() != 2 || arguments.iter().any(|argument| argument.spread.is_some()) { + return None; + } + if !is_import_meta_member(&arguments[1].expr, "url") { + return None; + } + let ast::Expr::Lit(ast::Lit::Str(specifier)) = strip_transparent_expr(&arguments[0].expr) + else { + return None; + }; + specifier.value.as_str().map(str::to_string) +} + +fn static_specifier( + expression: &ast::Expr, + path_bindings: &HashMap, + unresolved_ctxt: SyntaxContext, +) -> Option { + match strip_transparent_expr(expression) { + ast::Expr::Lit(ast::Lit::Str(value)) => value.value.as_str().map(str::to_string), + ast::Expr::Ident(identifier) => path_bindings.get(&identifier.to_id()).cloned(), + expression => url_pathname_specifier(expression, unresolved_ctxt), + } +} + +#[derive(Default)] +struct BindingScan { + counts: HashMap, + const_initializers: HashMap>>, + modified: HashSet, +} + +impl Visit for BindingScan { + fn visit_binding_ident(&mut self, binding: &ast::BindingIdent) { + *self.counts.entry(binding.id.to_id()).or_default() += 1; + binding.visit_children_with(self); + } + + fn visit_var_decl(&mut self, declaration: &ast::VarDecl) { + // Only `const` bindings are forwarding facts. `let`/`var` require a + // complete mutation analysis (destructuring and loop heads included), + // while Bun's emitted aliases and path constants use `const`. + if declaration.kind == ast::VarDeclKind::Const { + for declarator in &declaration.decls { + if let (ast::Pat::Ident(binding), Some(initializer)) = + (&declarator.name, &declarator.init) + { + self.const_initializers + .entry(binding.id.to_id()) + .or_default() + .push(initializer.clone()); + } + } + } + declaration.visit_children_with(self); + } + + fn visit_assign_expr(&mut self, assignment: &ast::AssignExpr) { + if let ast::AssignTarget::Simple(ast::SimpleAssignTarget::Ident(binding)) = &assignment.left + { + self.modified.insert(binding.id.to_id()); + } + assignment.visit_children_with(self); + } + + fn visit_update_expr(&mut self, update: &ast::UpdateExpr) { + if let ast::Expr::Ident(identifier) = strip_transparent_expr(&update.arg) { + self.modified.insert(identifier.to_id()); + } + update.visit_children_with(self); + } +} + +fn immutable_initializers(scan: &BindingScan) -> HashMap { + scan.const_initializers + .iter() + .filter_map(|(name, initializers)| { + (scan.counts.get(name) == Some(&1) + && initializers.len() == 1 + && !scan.modified.contains(name)) + .then(|| (name.clone(), initializers[0].as_ref())) + }) + .collect() +} + +fn recover_require_aliases(initializers: &HashMap) -> HashSet { + let mut aliases = HashSet::new(); + loop { + let mut changed = false; + for (name, initializer) in initializers { + if aliases.contains(name) { + continue; + } + let is_alias = is_import_meta_member(initializer, "require") + || identifier_id(initializer).is_some_and(|source| aliases.contains(&source)); + if is_alias { + changed |= aliases.insert(name.clone()); + } + } + if !changed { + return aliases; + } + } +} + +fn recover_path_bindings( + initializers: &HashMap, + unresolved_ctxt: SyntaxContext, +) -> HashMap { + let mut paths = HashMap::new(); + loop { + let mut changed = false; + for (name, initializer) in initializers { + if paths.contains_key(name) { + continue; + } + if let Some(specifier) = static_specifier(initializer, &paths, unresolved_ctxt) { + paths.insert(name.clone(), specifier); + changed = true; + } + } + if !changed { + return paths; + } + } +} + +struct RequireCall { + start: usize, + end: usize, + specifier: Option, +} + +struct CallScan<'a> { + aliases: &'a HashSet, + paths: &'a HashMap, + unresolved_ctxt: SyntaxContext, + calls: Vec, +} + +impl Visit for CallScan<'_> { + fn visit_call_expr(&mut self, call: &ast::CallExpr) { + let recognized = match &call.callee { + ast::Callee::Expr(callee) => { + is_import_meta_member(callee, "require") + || identifier_id(callee).is_some_and(|id| self.aliases.contains(&id)) + } + _ => false, + }; + if recognized { + let specifier = (call.args.len() == 1 && call.args[0].spread.is_none()) + .then(|| static_specifier(&call.args[0].expr, self.paths, self.unresolved_ctxt)) + .flatten(); + self.calls.push(RequireCall { + start: call.span.lo.0.saturating_sub(1) as usize, + end: call.span.hi.0.saturating_sub(1) as usize, + specifier, + }); + } + call.visit_children_with(self); + } +} + +fn looks_like_node_addon(specifier: &str) -> bool { + specifier + .split(['?', '#']) + .next() + .is_some_and(|path| path.ends_with(".node")) +} + +fn unique_identifier(source: &str, prefix: &str, index: usize) -> String { + let mut suffix = index; + loop { + let name = format!("{prefix}_{suffix}"); + if !source.contains(&name) { + return name; + } + suffix += 1; + } +} + +pub(super) fn rewrite_import_meta_require_addons( + source: &str, + module_path: &Path, + ctx: &mut CompilationContext, +) -> Result { + if !source.contains("import.meta") || !source.contains("require") { + return Ok(source.to_string()); + } + let filename = module_path.to_string_lossy(); + // A `.js` file that only uses `import.meta` has no import/export token for + // the parser's script-vs-module heuristic. Append a zero-width-for-existing + // spans module marker for this analysis parse; the emitted source and all + // original byte offsets remain unchanged. + let analysis_source = format!("{source}\nexport {{}};\n"); + let module = perry_parser::parse_typescript(&analysis_source, &filename).map_err(|error| { + anyhow!( + "failed to analyze `import.meta.require` in {}: {error}", + module_path.display() + ) + })?; + // SWC's resolver assigns a distinct syntax context to every lexical + // binding and its references. That makes the following dataflow safe + // across nested scopes: a parameter named `load` cannot be mistaken for + // an outer `const load = import.meta.require` alias. + let mut program = ast::Program::Module(module); + let unresolved_ctxt = GLOBALS.set(&Globals::new(), || { + let unresolved_mark = Mark::new(); + let top_level_mark = Mark::new(); + resolver(unresolved_mark, top_level_mark, true).process(&mut program); + SyntaxContext::empty().apply_mark(unresolved_mark) + }); + let ast::Program::Module(module) = program else { + unreachable!("the resolver preserves a module program") + }; + let mut bindings = BindingScan::default(); + module.visit_with(&mut bindings); + let initializers = immutable_initializers(&bindings); + let aliases = recover_require_aliases(&initializers); + let paths = recover_path_bindings(&initializers, unresolved_ctxt); + let mut calls = CallScan { + aliases: &aliases, + paths: &paths, + unresolved_ctxt, + calls: Vec::new(), + }; + module.visit_with(&mut calls); + + let mut replacements = Vec::new(); + let process_alias = unique_identifier(source, "__perry_import_meta_process", 0); + for (index, call) in calls.calls.into_iter().enumerate() { + let Some(specifier) = call.specifier else { + anyhow::bail!( + "cannot statically prove the Node-API addon path passed to `import.meta.require` in {}. Declare every project-owned addon with an exact `perry.nativeAddonPaths` entry and call the unmodified binding with a string literal or `new URL(\"./addon.node\", import.meta.url).pathname`.", + module_path.display() + ); + }; + let target = resolve_static_require( + module_path.parent().unwrap_or_else(|| Path::new(".")), + &specifier, + ctx.bunfs_root.as_deref(), + ); + let Some(target) = target else { + if looks_like_node_addon(&specifier) { + anyhow::bail!( + "statically declared Node-API addon `{specifier}` from {} could not be resolved. Project-owned addons must be listed by exact path in `perry.nativeAddonPaths`.", + module_path.display() + ); + } + continue; + }; + if target.extension().and_then(|extension| extension.to_str()) != Some("node") { + continue; + } + let Some(logical_id) = collect_node_addon_request(ctx, &target)? else { + continue; + }; + if call.start > call.end + || call.end > source.len() + || !source.is_char_boundary(call.start) + || !source.is_char_boundary(call.end) + { + anyhow::bail!( + "invalid source span while rewriting `import.meta.require` in {}", + module_path.display() + ); + } + let temporary = unique_identifier(source, "__perry_import_meta_addon", index); + let request = serde_json::to_string(&logical_id)?; + let replacement = format!( + "(function() {{ const {temporary} = {{ exports: {{}} }}; {process_alias}.dlopen({temporary}, {request}); return {temporary}.exports; }})()" + ); + replacements.push((call.start, call.end, replacement)); + } + + replacements.sort_by_key(|(start, _, _)| *start); + let mut rewritten = source.to_string(); + for (start, end, replacement) in replacements.into_iter().rev() { + rewritten.replace_range(start..end, &replacement); + } + if rewritten != source { + let import = format!("import * as {process_alias} from \"node:process\";\n"); + if rewritten.starts_with("#!") { + if let Some(line_end) = rewritten.find('\n') { + rewritten.insert_str(line_end + 1, &import); + } else { + rewritten.push('\n'); + rewritten.push_str(&import); + } + } else { + rewritten.insert_str(0, &import); + } + } + Ok(rewritten) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn fixture() -> (tempfile::TempDir, PathBuf, CompilationContext) { + let dir = tempfile::tempdir().unwrap(); + let native = dir.path().join("native"); + std::fs::create_dir_all(&native).unwrap(); + let addon = native.join("addon.node"); + std::fs::copy(std::env::current_exe().unwrap(), &addon).unwrap(); + let addon = addon.canonicalize().unwrap(); + let mut ctx = CompilationContext::new(dir.path().to_path_buf()); + ctx.bunfs_root = Some(dir.path().canonicalize().unwrap()); + ctx.native_addon_paths + .insert(addon, "native/addon.node".to_string()); + (dir, PathBuf::from("main.js"), ctx) + } + + #[test] + fn follows_aliases_url_path_constants_and_bun_virtual_paths() { + let (dir, relative_entry, mut ctx) = fixture(); + let entry = dir.path().join(relative_entry); + let source = r#" +const load = import.meta.require; +const forwarded = load; +const addonPath = new URL("./native/addon.node", import.meta.url).pathname; +const a = forwarded(addonPath); +const b = load("./native/addon.node"); +const c = load("/$bunfs/root/native/addon.node"); +const d = import.meta.require(new URL("./native/addon.node", import.meta.url).pathname); +"#; + let rewritten = rewrite_import_meta_require_addons(source, &entry, &mut ctx).unwrap(); + assert_eq!(rewritten.matches(".dlopen(").count(), 4, "{rewritten}"); + assert_eq!( + rewritten.matches("\"$project/native/addon.node\"").count(), + 4 + ); + assert_eq!(ctx.native_addons.len(), 1); + assert!(!ctx.native_addons["$project/native/addon.node"].ship_package_payload); + } + + #[test] + fn rejects_a_dynamic_path_with_policy_guidance() { + let (dir, relative_entry, mut ctx) = fixture(); + let entry = dir.path().join(relative_entry); + let error = rewrite_import_meta_require_addons( + "const load = import.meta.require; load(process.env.ADDON_PATH);", + &entry, + &mut ctx, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("cannot statically prove"), "{error}"); + assert!(error.contains("perry.nativeAddonPaths"), "{error}"); + } + + #[test] + fn does_not_follow_a_modified_binding() { + let (dir, relative_entry, mut ctx) = fixture(); + let entry = dir.path().join(relative_entry); + let source = + r#"let load = import.meta.require; load = other; load("./native/addon.node");"#; + let rewritten = rewrite_import_meta_require_addons(source, &entry, &mut ctx).unwrap(); + assert_eq!(rewritten, source); + assert!(ctx.native_addons.is_empty()); + } + + #[test] + fn does_not_treat_a_shadowed_url_constructor_as_static() { + let (dir, relative_entry, mut ctx) = fixture(); + let entry = dir.path().join(relative_entry); + let error = rewrite_import_meta_require_addons( + r#"const URL = CustomURL; const load = import.meta.require; load(new URL("./native/addon.node", import.meta.url).pathname);"#, + &entry, + &mut ctx, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("cannot statically prove"), "{error}"); + } + + #[test] + fn follows_nested_aliases_without_rewriting_same_named_bindings() { + let (dir, relative_entry, mut ctx) = fixture(); + let entry = dir.path().join(relative_entry); + let source = r#" +function loadAddon() { + const load = import.meta.require; + return load("./native/addon.node"); +} +function unrelated(load) { + return load("./native/addon.node"); +} +"#; + let rewritten = rewrite_import_meta_require_addons(source, &entry, &mut ctx).unwrap(); + assert_eq!(rewritten.matches(".dlopen(").count(), 1, "{rewritten}"); + assert!( + rewritten.contains("return load(\"./native/addon.node\");\n}"), + "{rewritten}" + ); + } +} diff --git a/crates/perry/src/commands/compile/collect_modules/native_addon.rs b/crates/perry/src/commands/compile/collect_modules/native_addon.rs index 030d8ec1b9..b9f83878f6 100644 --- a/crates/perry/src/commands/compile/collect_modules/native_addon.rs +++ b/crates/perry/src/commands/compile/collect_modules/native_addon.rs @@ -151,26 +151,58 @@ fn validate_node_api_binary(path: &std::path::Path) -> Result<()> { Ok(()) } +fn path_is_inside_node_modules(path: &std::path::Path) -> bool { + path.components().any(|component| { + matches!(component, std::path::Component::Normal(part) if part == "node_modules") + }) +} + /// Record an approved `.node` graph member or emit the existing actionable -/// unsupported-addon diagnostic. Returns true exactly for `.node` inputs so -/// the caller can stop before attempting to parse the native binary. -pub(super) fn collect_or_refuse_node_addon( +/// unsupported-addon diagnostic. The returned logical id is the only path +/// that generated code may pass to the authenticated runtime loader. +pub(super) fn collect_node_addon_request( ctx: &mut CompilationContext, canonical: &std::path::Path, -) -> Result { +) -> Result> { if canonical.extension().and_then(|ext| ext.to_str()) != Some("node") { - return Ok(false); + return Ok(None); + } + if let Some(project_path) = ctx.native_addon_paths.get(canonical).cloned() { + // Keep project entries in a namespace that cannot collide with the + // existing `/` logical-id scheme. + let logical_id = format!("$project/{project_path}"); + validate_node_api_binary(canonical)?; + let package_dir = canonical + .parent() + .unwrap_or_else(|| std::path::Path::new(".")) + .to_path_buf(); + let entry_relative = canonical + .file_name() + .map(PathBuf::from) + .ok_or_else(|| anyhow::anyhow!("Node-API addon path has no filename"))?; + ctx.native_addons + .entry(logical_id.clone()) + .or_insert_with(|| NativeAddonModule { + logical_id: logical_id.clone(), + package: "$project".to_string(), + version: "0.0.0".to_string(), + source_path: canonical.to_path_buf(), + package_dir, + entry_relative, + ship_package_payload: false, + }); + return Ok(Some(logical_id)); } let package_root = nearest_package_root(canonical); if package_root .as_deref() .is_some_and(package_is_parcel_watcher_facade) { - return Ok(true); + return Ok(None); } - let Some(package_root) = package_root else { + let Some(package_root) = package_root.filter(|root| path_is_inside_node_modules(root)) else { anyhow::bail!( - "`{}` is a Node native addon outside an npm package. Addons must be selected through an exact `perry.nativeAddons` package entry.", + "`{}` is a project-owned Node native addon and is not authorized. Add its exact project-relative path to `perry.nativeAddonPaths` (for example `\"nativeAddonPaths\": [\"native/addon.node\"]`).", canonical.display() ); }; @@ -199,14 +231,31 @@ pub(super) fn collect_or_refuse_node_addon( ctx.native_addons .entry(logical_id.clone()) .or_insert_with(|| NativeAddonModule { - logical_id, + logical_id: logical_id.clone(), package: owner_package, version, source_path: canonical.to_path_buf(), package_dir: package_root, entry_relative, + ship_package_payload: true, }); - Ok(true) + Ok(Some(logical_id)) +} + +/// Returns true exactly for `.node` inputs handled as sidecar graph members so +/// the caller can stop before attempting to parse native bytes as source. +pub(super) fn collect_or_refuse_node_addon( + ctx: &mut CompilationContext, + canonical: &std::path::Path, +) -> Result { + if canonical.extension().and_then(|ext| ext.to_str()) == Some("node") + && nearest_package_root(canonical) + .as_deref() + .is_some_and(package_is_parcel_watcher_facade) + { + return Ok(true); + } + collect_node_addon_request(ctx, canonical).map(|request| request.is_some()) } fn package_is_parcel_watcher_facade(package_root: &std::path::Path) -> bool { @@ -340,6 +389,15 @@ pub(super) fn refuse_compile_package_native_addon( let Some(package_root) = package_root_for_compile_package(ctx, canonical) else { return Ok(()); }; + // The host project can legitimately contain exact path-authorized addons + // (#9606). This package-wide preflight exists for dependencies selected by + // `compilePackages`; project members are checked individually when their + // `.node` edge is collected, so scanning the host root here would reject + // an authorized addon merely because Bun-root routing selected the host + // JS. Keep checking symlinked/file: dependency roots outside node_modules. + if ctx.cache_root.starts_with(&package_root) { + return Ok(()); + } if !ctx .checked_compile_package_native_addon_roots .insert(package_root.clone()) diff --git a/crates/perry/src/commands/compile/collect_modules/static_require_transform.rs b/crates/perry/src/commands/compile/collect_modules/static_require_transform.rs index e196d647f8..e7e9fc99ba 100644 --- a/crates/perry/src/commands/compile/collect_modules/static_require_transform.rs +++ b/crates/perry/src/commands/compile/collect_modules/static_require_transform.rs @@ -186,7 +186,7 @@ pub(super) fn transform_static_literal_requires_with_bunfs( prepend_imports_preserving_shebang(&transformed, &imports) } -fn resolve_static_require( +pub(super) fn resolve_static_require( module_dir: &Path, specifier: &str, bunfs_root: Option<&Path>, diff --git a/crates/perry/src/commands/compile/collect_modules/tests.rs b/crates/perry/src/commands/compile/collect_modules/tests.rs index ce0a94c323..5af8ee041e 100644 --- a/crates/perry/src/commands/compile/collect_modules/tests.rs +++ b/crates/perry/src/commands/compile/collect_modules/tests.rs @@ -42,6 +42,7 @@ fn approved_node_addon_is_recorded_as_a_relocatable_graph_member() { assert_eq!(record.package, "demo-addon"); assert_eq!(record.version, "1.2.3"); assert_eq!(record.entry_relative, std::path::Path::new("binding.node")); + assert!(record.ship_package_payload); } #[test] @@ -903,6 +904,30 @@ fn compile_package_with_node_file_is_rejected() { ); } +#[test] +fn external_compile_package_root_still_gets_native_addon_preflight() { + let dir = tempfile::tempdir().expect("tempdir"); + let host = dir.path().join("host"); + let package = dir.path().join("linked-addon"); + std::fs::create_dir_all(&host).expect("host directory"); + std::fs::create_dir_all(&package).expect("linked package directory"); + std::fs::write(package.join("package.json"), r#"{"name":"linked-addon"}"#) + .expect("linked package manifest"); + std::fs::write(package.join("binding.gyp"), "{}\n").expect("native addon marker"); + let entry = package.join("index.js"); + std::fs::write(&entry, "module.exports = 1\n").expect("linked package entry"); + + let mut ctx = CompilationContext::new(host); + ctx.compile_package_dirs + .insert(package.canonicalize().expect("canonical linked package")); + let error = refuse_compile_package_native_addon( + &mut ctx, + &entry.canonicalize().expect("canonical linked entry"), + ) + .expect_err("an external file: dependency must retain package-wide preflight"); + assert!(error.to_string().contains("binding.gyp"), "{error}"); +} + #[test] fn compile_package_with_node_directory_is_allowed() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/perry/src/commands/compile/host_config.rs b/crates/perry/src/commands/compile/host_config.rs index 19f2da8755..01da27af50 100644 --- a/crates/perry/src/commands/compile/host_config.rs +++ b/crates/perry/src/commands/compile/host_config.rs @@ -14,7 +14,7 @@ use std::collections::{BTreeMap, HashMap}; use std::fs; -use std::path::Path; +use std::path::{Component, Path, PathBuf}; use anyhow::Result; use perry_codegen::FpContractMode; @@ -58,6 +58,29 @@ fn is_exact_npm_package_name(name: &str) -> bool { valid_segment(name) } +fn normalized_project_addon_path(value: &str) -> Option<(PathBuf, String)> { + let path = Path::new(value.trim()); + if path.as_os_str().is_empty() + || path.is_absolute() + || path.extension().and_then(|extension| extension.to_str()) != Some("node") + { + return None; + } + let mut relative = PathBuf::new(); + let mut portable = Vec::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::Normal(part) if part != "node_modules" => { + relative.push(part); + portable.push(part.to_string_lossy().into_owned()); + } + _ => return None, + } + } + (!portable.is_empty()).then(|| (relative, portable.join("/"))) +} + fn parse_boolean_switch(value: &str) -> Option { match value.trim().to_ascii_lowercase().as_str() { "1" | "true" => Some(true), @@ -257,6 +280,54 @@ pub(super) fn apply_pkg_and_toml_config( ctx.native_addon_packages.insert(name.to_string()); } } + // #9606: extracted Bun standalones can carry project-owned + // `.node` files that have no npm package identity. Keep those + // exact path grants separate from package grants: authorizing + // `native/addon.node` must not trust its directory, another + // addon, or a package under node_modules. + if let Some(native_addon_paths) = pkg + .get("perry") + .and_then(|perry| perry.get("nativeAddonPaths")) + { + let entries = native_addon_paths.as_array().ok_or_else(|| { + anyhow::anyhow!( + "`perry.nativeAddonPaths` must be an array of exact project-relative `.node` paths" + ) + })?; + let config_root = pkg_json_path + .parent() + .ok_or_else(|| { + anyhow::anyhow!("project package.json has no parent directory") + })? + .canonicalize()?; + for (index, entry) in entries.iter().enumerate() { + let value = entry.as_str().ok_or_else(|| { + anyhow::anyhow!( + "`perry.nativeAddonPaths[{index}]` must be a project-relative path string" + ) + })?; + let (relative, logical_id) = normalized_project_addon_path(value) + .ok_or_else(|| { + anyhow::anyhow!( + "`perry.nativeAddonPaths[{index}]` must name one exact project-relative `.node` file outside `node_modules`; invalid entry `{value}`" + ) + })?; + let configured = config_root.join(&relative); + let canonical = configured.canonicalize().map_err(|error| { + anyhow::anyhow!( + "configured Node-API addon `{}` is unavailable: {error}", + configured.display() + ) + })?; + if !canonical.starts_with(&config_root) || !canonical.is_file() { + anyhow::bail!( + "configured Node-API addon `{}` must resolve to a file inside the host project", + configured.display() + ); + } + ctx.native_addon_paths.insert(canonical, logical_id); + } + } // #1680 (Phase 2 of #1677): build-time codegen steps. Each // entry is a shell command (or `{ command, label }`) run // before module collection so codegen libraries with an @@ -1262,7 +1333,7 @@ pub(super) fn apply_pkg_and_toml_config( } } - if !ctx.native_addon_packages.is_empty() { + if !ctx.native_addon_packages.is_empty() || !ctx.native_addon_paths.is_empty() { let target = args.target.as_deref().unwrap_or("native"); let unsupported = matches!( target, @@ -1284,7 +1355,7 @@ pub(super) fn apply_pkg_and_toml_config( ); if unsupported { anyhow::bail!( - "`perry.nativeAddons` is unavailable for target `{target}`; prebuilt Node-API sidecars are supported only on desktop/server targets" + "`perry.nativeAddons` / `perry.nativeAddonPaths` are unavailable for target `{target}`; prebuilt Node-API sidecars are supported only on desktop/server targets" ); } } @@ -1318,6 +1389,30 @@ mod tests { } } + #[test] + fn project_addon_policy_accepts_only_exact_relative_node_paths() { + for (value, expected) in [ + ("native/addon.node", "native/addon.node"), + ("./addon.node", "addon.node"), + ("native/platform/addon.node", "native/platform/addon.node"), + ] { + let (_, portable) = normalized_project_addon_path(value).expect(value); + assert_eq!(portable, expected); + } + for value in [ + "", + ".", + "native", + "native/addon.so", + "../addon.node", + "/tmp/addon.node", + "node_modules/pkg/addon.node", + "native/../../addon.node", + ] { + assert!(normalized_project_addon_path(value).is_none(), "{value}"); + } + } + #[test] fn auto_switch_accepts_only_documented_values() { for value in [ diff --git a/crates/perry/src/commands/compile/native_addon_sidecar.rs b/crates/perry/src/commands/compile/native_addon_sidecar.rs index 4976c6a4f8..8c1a4a5765 100644 --- a/crates/perry/src/commands/compile/native_addon_sidecar.rs +++ b/crates/perry/src/commands/compile/native_addon_sidecar.rs @@ -19,6 +19,7 @@ struct SidecarManifest { shipping_model: &'static str, target: String, allowlist: Vec, + path_allowlist: Vec, addons: Vec, } @@ -82,6 +83,9 @@ fn payload_key(addon: &NativeAddonModule) -> String { /// artifact. Nested node_modules and VCS state are separate packages, not /// part of the selected platform payload. pub(super) fn addon_payload_files(addon: &NativeAddonModule) -> Vec { + if !addon.ship_package_payload { + return vec![addon.source_path.clone()]; + } let mut files = walkdir::WalkDir::new(&addon.package_dir) .follow_links(false) .into_iter() @@ -189,6 +193,7 @@ pub(super) fn stage_native_addon_sidecar( shipping_model: SHIPPING_MODEL, target: target_tuple(target), allowlist: ctx.native_addon_packages.iter().cloned().collect(), + path_allowlist: ctx.native_addon_paths.values().cloned().collect(), addons: manifest_addons, }; fs::write( @@ -236,6 +241,7 @@ mod tests { source_path: entry, package_dir: package, entry_relative: PathBuf::from("demo.node"), + ship_package_payload: true, }, ); let root = stage_native_addon_sidecar(&ctx, &output, None) diff --git a/crates/perry/src/commands/compile/types.rs b/crates/perry/src/commands/compile/types.rs index d76e8372bb..c19f502c1e 100644 --- a/crates/perry/src/commands/compile/types.rs +++ b/crates/perry/src/commands/compile/types.rs @@ -673,6 +673,12 @@ pub struct CompilationContext { /// Approved `.node` entries reached by the compile graph, keyed by their /// relocatable package-relative logical id. pub native_addons: BTreeMap, + /// Exact project-relative `.node` paths authorized by the host manifest, + /// keyed by canonical source path. The value is the portable declared path + /// used to derive the sidecar logical id; unlike `native_addon_packages`, + /// these entries never confer trust on an npm package or a containing + /// directory. + pub native_addon_paths: BTreeMap, /// Package aliases: maps npm package name → replacement package name (from perry.packageAliases) pub package_aliases: HashMap, /// Packages to compile natively instead of routing to V8 (from perry.compilePackages) @@ -1202,6 +1208,7 @@ impl CompilationContext { native_libraries: Vec::new(), native_addon_packages: BTreeSet::new(), native_addons: BTreeMap::new(), + native_addon_paths: BTreeMap::new(), package_aliases: HashMap::new(), compile_packages: HashSet::new(), auto_skipped_node_addon_packages: HashSet::new(), @@ -1308,6 +1315,10 @@ pub struct NativeAddonModule { pub source_path: PathBuf, pub package_dir: PathBuf, pub entry_relative: PathBuf, + /// Package entries ship their complete package-local payload so adjacent + /// data/shared libraries remain available. Exact project-path entries ship + /// only the explicitly authorized `.node` file. + pub ship_package_payload: bool, } /// External native library manifest parsed from package.json `perry.nativeLibrary` field diff --git a/crates/perry/tests/node_api_host_e2e.rs b/crates/perry/tests/node_api_host_e2e.rs index 62731a7bb4..a498478361 100644 --- a/crates/perry/tests/node_api_host_e2e.rs +++ b/crates/perry/tests/node_api_host_e2e.rs @@ -127,6 +127,24 @@ fn compile_app(root: &Path, entry: &Path, output: &Path) -> Output { command.output().expect("run perry compile") } +fn compile_bunfs_app(root: &Path, entry: &Path, output: &Path) -> Output { + let mut command = Command::new(perry_bin()); + command + .current_dir(root) + .env("PERRY_WORKSPACE_ROOT", workspace_root()) + .arg("compile") + .arg(entry) + .arg("-o") + .arg(output) + .arg("--bunfs-root") + .arg(root) + .arg("--no-cache"); + if std::env::var_os("PERRY_E2E_VERBOSE").is_some() { + command.arg("-vv"); + } + command.output().expect("run bunfs perry compile") +} + fn find_node_file(path: &Path) -> Option { let mut entries = std::fs::read_dir(path) .ok()? @@ -521,6 +539,158 @@ console.log("node-api-cache", direct.exports === addon) ); } +#[test] +fn bun_import_meta_require_project_addon_survives_source_removal() { + if !require_tool("clang") { + return; + } + #[cfg(windows)] + if !require_tool("llvm-dlltool") { + return; + } + + let dir = tempfile::tempdir().expect("tempdir"); + let extracted = dir.path().join("extracted"); + let native = extracted.join("native"); + let build = dir.path().join("build"); + std::fs::create_dir_all(&native).expect("create project addon directory"); + std::fs::create_dir_all(&build).expect("create build directory"); + std::fs::write( + extracted.join("package.json"), + r#"{ + "name": "perry-bun-root-addon-e2e", + "private": true, + "perry": { + "nativeAddonPaths": ["native/addon.node"] + } +}"#, + ) + .expect("write project addon policy"); + compile_addon(&extracted, &native); + + let entry = extracted.join("main.js"); + std::fs::write( + &entry, + r#"const load = import.meta.require; +const addonPath = new URL("./native/addon.node", import.meta.url).pathname; +const first = load(addonPath); +const r = import.meta.require; +const second = r("./native/addon.node"); +const third = r("/$bunfs/root/native/addon.node"); +const fourth = import.meta.require(new URL("./native/addon.node", import.meta.url).pathname); +console.log("bun-root-node-api", first.add(19, 23), second.answer, third === first, fourth === first); +"#, + ) + .expect("write import.meta.require entry"); + + let executable = build.join(if cfg!(windows) { "app.exe" } else { "app" }); + let compile = compile_bunfs_app(&extracted, &entry, &executable); + assert!( + compile.status.success(), + "project Node-API compile failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + let sidecar = executable.with_file_name(format!( + "{}.perry-native", + executable.file_name().unwrap().to_string_lossy() + )); + let manifest: serde_json::Value = serde_json::from_slice( + &std::fs::read(sidecar.join("manifest.json")).expect("read project addon manifest"), + ) + .expect("parse project addon manifest"); + assert_eq!( + manifest["path_allowlist"], + serde_json::json!(["native/addon.node"]) + ); + assert_eq!( + manifest["addons"][0]["logical_id"], + "$project/native/addon.node" + ); + assert_eq!(manifest["addons"][0]["package"], "$project"); + assert_eq!( + manifest["addons"][0]["files"] + .as_array() + .expect("project addon files") + .len(), + 1, + "an exact project path must not implicitly ship its directory" + ); + + let install = dir.path().join("install"); + std::fs::create_dir_all(&install).expect("create install directory"); + let installed = install.join(executable.file_name().unwrap()); + let installed_sidecar = install.join(sidecar.file_name().unwrap()); + std::fs::rename(&executable, &installed).expect("relocate executable"); + std::fs::rename(&sidecar, &installed_sidecar).expect("relocate addon sidecar"); + std::fs::remove_dir_all(&extracted).expect("remove Bun extraction source tree"); + assert!( + !extracted.exists(), + "source tree must be gone before runtime" + ); + + let output = run( + Command::new(&installed), + "relocated Bun import.meta.require Node-API host", + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("bun-root-node-api 42 8523 true true"), + "stdout: {stdout}" + ); +} + +#[test] +fn dynamic_import_meta_require_addon_path_is_rejected_with_declaration_help() { + let dir = tempfile::tempdir().expect("tempdir"); + let native = dir.path().join("native"); + std::fs::create_dir_all(&native).expect("create native directory"); + // Configuration validates existence and path containment before module + // collection. The dynamic-path diagnostic fires before binary inspection, + // so an inert marker is sufficient for this negative gate. + std::fs::write(native.join("addon.node"), b"not loaded").expect("write addon marker"); + std::fs::write( + dir.path().join("package.json"), + r#"{ + "name": "perry-dynamic-root-addon-e2e", + "private": true, + "perry": { + "nativeAddonPaths": ["native/addon.node"] + } +}"#, + ) + .expect("write project addon policy"); + let entry = dir.path().join("main.js"); + std::fs::write( + &entry, + "const load = import.meta.require; load(process.env.ADDON_PATH);\n", + ) + .expect("write dynamic addon entry"); + let output = dir.path().join(if cfg!(windows) { + "dynamic.exe" + } else { + "dynamic" + }); + let compile = compile_app(dir.path(), &entry, &output); + assert!( + !compile.status.success(), + "dynamic import.meta.require addon path unexpectedly compiled" + ); + let diagnostic = format!( + "{}{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + assert!( + diagnostic.contains("cannot statically prove"), + "{diagnostic}" + ); + assert!( + diagnostic.contains("perry.nativeAddonPaths"), + "{diagnostic}" + ); +} + #[test] fn published_napi_rs_addon_runs_sync_and_async_work() { let dir = tempfile::tempdir().expect("tempdir");