diff --git a/Cargo.lock b/Cargo.lock index 92538a7d4c..dd4ed8f82e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6338,6 +6338,7 @@ dependencies = [ "perry-parser", "rand 0.10.1", "regex", + "regex-syntax", "regress", "resolv-conf", "ryu", diff --git a/changelog.d/9178-lazy-regex-compilation.md b/changelog.d/9178-lazy-regex-compilation.md new file mode 100644 index 0000000000..bdb8cda329 --- /dev/null +++ b/changelog.d/9178-lazy-regex-compilation.md @@ -0,0 +1,29 @@ +A `/…/` literal lowers to `js_regexp_new` at its evaluation site, and +`js_regexp_new` answered "is this a SyntaxError?" by BUILDING the pattern — +`regex_syntax` parse, HIR translate including Unicode case folding, Thompson +NFA construction, meta-engine strategy selection. Every regex a program *had* +was compiled, not every regex it *used*. A symbolized instruction profile of +the claude-code CLI running `--help` — a command that prints text and exits — +put 14.6% of all retired instructions inside regex compilation, against 0.11% +for the whole of its compiled JavaScript. + +Only the program build is deferred; everything observable at construction stays +at construction. A syntactically invalid pattern still throws `SyntaxError` +from `js_regexp_new` / `RegExp.prototype.compile`, at the same point in the +program, because `std_engine_syntax_ok` runs the same parser `build_std_regex` +would run, on the same translated, flag-prefixed, REDoS-collapsed string — +4.6 µs/pattern against 82 µs to build. Anything it rejects falls through to the +unchanged both-engines check, so the fancy-regex fallback for +lookbehind/backreferences still decides and still throws when both refuse. +`.source`, `.flags`, `.global`, `.sticky` and `lastIndex` never touched the +compiled program, and identity is unchanged. The build happens on the first +operation that needs a matcher. + +| literals constructed, 1 used | before | after | node | +|---|---|---|---| +| 50 | 19 ms | 2 ms | 0 ms | +| 200 | 73 ms | 7 ms | 1 ms | +| 400 | 145 ms | 15 ms | 3 ms | +| every distinct literal in cli_2.1.112.js (2,378) | 232 ms | 50 ms | 5 ms | + +Wall clock on the cc corpus 247.5 → 59.9 ms. diff --git a/crates/perry-runtime/Cargo.toml b/crates/perry-runtime/Cargo.toml index 20b797d077..74df755fbc 100644 --- a/crates/perry-runtime/Cargo.toml +++ b/crates/perry-runtime/Cargo.toml @@ -120,7 +120,7 @@ diagnostics = [] # RegExp object's identity/display layer (header, `is_regex_pointer`, `toString`) # stays always compiled, so console-formatting / value-to-string paths keep # working with no engine present. -regex-engine = ["dep:regex", "dep:fancy-regex", "dep:regress"] +regex-engine = ["dep:regex", "dep:regex-syntax", "dep:fancy-regex", "dep:regress"] # The TC39 `Temporal.*` API (`temporal_rs` + its transitive tz/calendar deps: # jiff-tzdb, icu_calendar, timezone_provider, calendrical_calculations — # ~580 KB). Independent of JS `Date` (which has its own `date.rs` impl), so a @@ -286,6 +286,10 @@ libc.workspace = true gimli = { version = "0.34", default-features = false, features = ["read"] } rand = "0.10" regex = { workspace = true, optional = true } +# The parser half of `regex`, used on its own to answer "is this pattern a +# SyntaxError?" without building the automaton — see `regex/lazy.rs`. Not a +# new archive: `regex` already links it, this only names it directly. +regex-syntax = { version = "0.8", optional = true } regress = { workspace = true, features = ["utf16"], optional = true } # Taffy — flexbox / grid layout engine for the perry/tui module # (#358 Phase 3). Same crate Bevy and Dioxus use; pure Rust, no FFI. diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 26c693019a..1bb7ebb479 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -38,6 +38,8 @@ mod exec_array; #[cfg(feature = "regex-engine")] mod grammar; #[cfg(feature = "regex-engine")] +mod lazy; +#[cfg(feature = "regex-engine")] mod match_all; #[cfg(feature = "regex-engine")] mod repeat_matcher; @@ -314,6 +316,16 @@ crate::perry_thread_local! { /// are the patterns where `regex`/`fancy-regex` cannot reproduce /// `RepeatMatcher` capture reset and nullable-iteration semantics (#5897). static REPEAT_MATCHER_CACHE: RefCell>> = RefCell::new(HashMap::new()); + + /// `(pattern, flags)` pairs that have already cleared construction-time + /// validation. Validity is a pure function of the pair, so the answer is + /// worth remembering; `js_regexp_new` used to get this from a + /// `REGEX_CACHE` hit, which stopped being a proxy once the compiled + /// program became lazy (see `regex::lazy`). Same cap and + /// clear-on-overflow policy as the program caches — the cost of a clear + /// is a repeated parse, never a wrong verdict. The unit value keeps + /// `evict_regex_cache_if_full` shared with the three program caches. + static VALIDATED_PATTERNS: RefCell> = RefCell::new(HashMap::new()); } /// Compiled-program size budget handed to both regex engines. @@ -367,8 +379,9 @@ pub(crate) fn build_fancy_regex(pattern: &str) -> Result(cache: &mut HashMap<(String, String), V>) { if cache.len() >= REGEX_CACHE_MAX_ENTRIES { @@ -377,12 +390,15 @@ fn evict_regex_cache_if_full(cache: &mut HashMap<(String, String), V>) { } /// Compile `(pattern, flags)` into the caches if absent, reporting whether -/// SOME engine accepted the flag-prefixed pattern. One NFA build total — -/// `js_regexp_new` used to build every unique pattern TWICE (once discarded -/// for validation at construction, once here for the cache), which doubled -/// regex cost during bundle startup where every module-level literal -/// constructs eagerly (the emoji-regex class of pattern costs milliseconds -/// per build). +/// SOME engine accepted the flag-prefixed pattern. One NFA build total. +/// +/// This is the expensive path — the emoji-regex class of pattern costs +/// milliseconds per build. It no longer runs at construction: `js_regexp_new` +/// validates with the parser alone and `regex::lazy` calls this (through +/// `get_or_compile_regex`) on the first operation that needs a matcher. It is +/// still reached from construction for the patterns the linear engine's parser +/// rejects, where only a build can tell a fancy-regex pattern from a +/// `SyntaxError`. /// /// Returns `true` when the pattern is usable: compiled by the `regex` crate /// (cached in `REGEX_CACHE`), or by `fancy-regex` (cached in `FANCY_CACHE`, @@ -411,29 +427,10 @@ fn compile_and_cache_regex_checked(pattern: &str, flags: &str) -> bool { ); }); } - // Translate JS regex to Rust-compatible pattern - let translated = js_regex_to_rust(pattern); - let case_insensitive = flags.contains('i'); - let multiline = flags.contains('m'); - // #2828: the `s` (dotAll) flag maps directly onto the Rust `regex` - // crate's `(?s)` inline mode, so `.` matches newlines. - let dot_all = flags.contains('s'); - let regex_pattern = if case_insensitive || multiline || dot_all { - let mut prefix = String::from("(?"); - if case_insensitive { - prefix.push('i'); - } - if multiline { - prefix.push('m'); - } - if dot_all { - prefix.push('s'); - } - prefix.push(')'); - format!("{}{}", prefix, translated) - } else { - translated - }; + // Translate JS regex to Rust-compatible pattern, with the inline mode + // prefix the flags imply. Shared with `lazy::std_engine_syntax_ok` so the + // eager syntax check and this build can never inspect different strings. + let regex_pattern = lazy::flag_prefixed_pattern(pattern, flags); let regex = match build_std_regex(®ex_pattern) { Ok(re) => re, Err(_) => { @@ -746,9 +743,12 @@ fn validate_and_canonicalize_flags(flags: &str) -> String { /// Create a new RegExp from pattern and flags strings /// Returns a pointer to RegExpHeader /// -/// Uses the thread-local REGEX_CACHE so repeated regex literals (e.g. in a -/// loop) reuse the same compiled Regex instead of leaking a fresh one each -/// time. The raw pointer stored in RegExpHeader is kept alive by the cache. +/// Validates the pattern and allocates the header; it does NOT build the +/// compiled program. That happens on the first operation that needs a matcher +/// — see `regex::lazy`, and the `regex_ptr`/`fancy_ptr`/`repeat_matcher_ptr` +/// fields, which are null until then. A fresh header per call is required: +/// ECMA-262 evaluates a regex literal to a NEW object every time, and the +/// distinction is observable through `===`, expandos and `lastIndex`. #[cfg(feature = "regex-engine")] #[no_mangle] pub extern "C" fn js_regexp_new( @@ -801,28 +801,20 @@ pub extern "C" fn js_regexp_new( // the fancy fallback. `get_or_compile_regex` populates FANCY_CACHE when // the regex crate fails but fancy-regex succeeds; check both here. // - // PERF (#5777 follow-up): the ENTIRE validation block is gated on a - // REGEX_CACHE miss. Regex validity is a pure function of (pattern, flags): - // an invalid pattern throws here BEFORE `get_or_compile_regex` can ever - // cache it, and both writers of REGEX_CACHE — this function and - // `regex/compile.rs` (`RegExp.prototype.compile`) — run these exact checks - // first, so any entry already in the cache is provably valid and - // re-validating it can only burn CPU. #5777 already skipped the expensive - // both-engines recompile on a hit; this extends the skip to the "cheap" - // JS-syntax checks too, which are not actually cheap: - // `has_invalid_repeated_quantifier` does a - // `pattern.chars().collect::>()` (a ~51 KB allocation for a - // 12,807-char pattern) plus an O(n) scan on EVERY `new RegExp(...)`. The - // common `string-width`/`emoji-regex` npm packages construct a fresh - // ~12,807-char `/…/g` literal on every measurement and a layout pass can - // call them thousands of times, so this re-validation — not the - // already-cached compile — became the top hot frame in profiles. + // PERF (#5777 follow-up): the ENTIRE validation block runs at most once + // per (pattern, flags). Regex validity is a pure function of the pair, so + // a pattern that has already cleared it can never fail it later; the + // cheap JS-syntax checks are not actually cheap + // (`has_invalid_repeated_quantifier` does a + // `pattern.chars().collect::>()` — a ~51 KB allocation for a + // 12,807-char pattern — plus an O(n) scan on EVERY `new RegExp(...)`), + // and the common `string-width`/`emoji-regex` npm packages construct a + // fresh ~12,807-char `/…/g` literal on every measurement, which a layout + // pass calls thousands of times. #5777 keyed that skip off a REGEX_CACHE + // hit, which worked only because construction also COMPILED; with the + // build deferred, the fact is recorded directly in `VALIDATED_PATTERNS`. { - let in_cache = REGEX_CACHE.with(|c| { - c.borrow() - .contains_key(&(pattern_str.to_string(), flags_str.to_string())) - }); - if !in_cache { + if !lazy::pattern_already_validated(pattern_str, flags_str) { if has_invalid_repeated_quantifier(pattern_str) { throw_regexp_syntax_error(&format!( "Invalid regular expression: /{}/: invalid pattern", @@ -860,12 +852,23 @@ pub extern "C" fn js_regexp_new( pattern_str )); } - // The expensive part of validation: compile the pattern. This - // BUILDS AND CACHES in one step (`compile_and_cache_regex_checked`) - // so the `get_or_compile_regex` below is a guaranteed cache hit — - // previously every unique pattern was NFA-compiled twice (once - // discarded here, once for the cache), doubling startup regex cost. - if !compile_and_cache_regex_checked(pattern_str, flags_str) { + // The remaining question — "is this a SyntaxError?" — used to be + // answered by BUILDING the pattern, which is why constructing a + // regex cost an NFA. Ask the standard engine's PARSER instead + // (`lazy::std_engine_syntax_ok`, the same `regex_syntax` parse + // `build_std_regex` performs, on the same string): 17.8x cheaper, + // and it agrees with the full build on every one of the 2,378 + // regex literals in the claude-code bundle (asserted over a + // corpus by `tests::syntax_check_agrees_with_full_build`). + // + // A parser rejection is NOT a verdict: every lookbehind / + // backreference pattern is rejected by the linear engine too. Fall + // through to the unchanged both-engines path, which owns the + // SyntaxError decision and populates the caches for the fancy + // fallback. + if !lazy::std_engine_syntax_ok(pattern_str, flags_str) + && !compile_and_cache_regex_checked(pattern_str, flags_str) + { // Preserve the historical edge: validation used to test the // BARE translated pattern (no `(?ims)` prefix). A pattern that // compiles bare but blows the size limit with the flag prefix @@ -880,17 +883,17 @@ pub extern "C" fn js_regexp_new( )); } } + lazy::mark_pattern_validated(pattern_str, flags_str); } } - // Get or compile the regex from the cache. The header OWNS a leaked `Arc` - // reference (`Arc::into_raw`) to the compiled program — mirroring - // `fancy_ptr` below — so the pointer stays valid even after the capped - // `REGEX_CACHE` (see `REGEX_CACHE_MAX_ENTRIES`) evicts its own reference. - // Previously this borrowed `Arc::as_ptr` and relied on the cache never - // dropping an entry. - let arc = get_or_compile_regex(pattern_str, flags_str); - let regex_ptr = Arc::into_raw(arc) as *mut Regex; + // The compiled program is NOT built here. Validation above has already + // established that the pattern is legal, and a bundle evaluates hundreds + // of module-level literals it never matches with — building each one's + // NFA at construction is what put ~14% of a claude-code `--help` run + // inside `regex_syntax`/`regex_automata`. `regex_ptr` stays null (the + // "not built yet" state) and `lazy::ensure_regex_compiled` installs the + // owned `Arc`s on the first operation that needs a matcher. // ★ Last use of the borrowed pattern text before this function allocates. // `pattern_str` borrows the GC string; the two allocations below can move @@ -946,7 +949,8 @@ pub extern "C" fn js_regexp_new( // Neither `gc_malloc` nor the arena zeroes reused memory, so this // must be set explicitly or the GC follows a garbage pointer. (*ptr).meta = std::ptr::null_mut(); - (*ptr).regex_ptr = regex_ptr; + // Null = not compiled yet; see `lazy::ensure_regex_compiled`. + (*ptr).regex_ptr = std::ptr::null_mut(); (*ptr).pattern_ptr = pattern; (*ptr).flags_ptr = canonical_flags_ptr; // `pattern_ptr` / `flags_ptr` are GC-managed StringHeaders — the GC scans @@ -989,31 +993,14 @@ pub extern "C" fn js_regexp_new( // Wall 18: self-identifying marker so identity checks survive a // duplicate-runtime thread-local split. (*ptr).magic = REGEXP_MAGIC; - // Header-resident fancy-regex fallback (lookahead/lookbehind/backrefs) - // so `.replace(re, fn)` etc. don't depend on the (possibly other-copy) - // FANCY_CACHE thread-local. `get_or_compile_regex` above already - // populated FANCY_CACHE on THIS thread when the std `regex` crate - // rejected the pattern; clone that Arc onto the header (leaked so the - // raw pointer stays valid for the header's lifetime — RegExp headers - // and their compiled programs live for the process today). - (*ptr).fancy_ptr = FANCY_CACHE.with(|fc| { - match fc - .borrow() - .get(&(owned_pattern.clone(), flags_str.to_string())) - { - Some(arc) => Arc::into_raw(arc.clone()) as *const (), - None => std::ptr::null(), - } - }); - (*ptr).repeat_matcher_ptr = REPEAT_MATCHER_CACHE.with(|cache| { - match cache - .borrow() - .get(&(owned_pattern.clone(), flags_str.to_string())) - { - Some(arc) => Arc::into_raw(arc.clone()) as *const (), - None => std::ptr::null(), - } - }); + // The header-resident fancy-regex fallback (lookahead/lookbehind/ + // backrefs) and the ECMAScript backtracking matcher are installed + // alongside `regex_ptr` by `lazy::ensure_regex_compiled`, from the + // same caches, on the first operation that needs a matcher. Keeping + // all three on one publish point is what makes `regex_ptr.is_null()` + // a sound built/not-built flag. + (*ptr).fancy_ptr = std::ptr::null(); + (*ptr).repeat_matcher_ptr = std::ptr::null(); // Record the pointer so that js_string_split can detect // `s.split(regex)` without a dedicated runtime decl. @@ -1199,7 +1186,7 @@ pub extern "C" fn js_regexp_test(re: *const RegExpHeader, s: *const StringHeader }; } - let regex = &*(*re).regex_ptr; + let regex = lazy::header_std_regex(re); if regex.is_match(str_data) { 1 } else { @@ -1213,6 +1200,10 @@ pub extern "C" fn js_regexp_test(re: *const RegExpHeader, s: *const StringHeader /// pattern (backreferences, lookbehind, etc.). #[cfg(feature = "regex-engine")] pub(crate) fn lookup_fancy_regex(re: *const RegExpHeader) -> Option> { + // The header's programs are built on first use; `fancy_ptr` is null until + // then, and a null there is indistinguishable from "this pattern has no + // fancy fallback" — so build before reading it. + lazy::ensure_regex_compiled(re); unsafe { // Wall 18: header-resident fancy Arc first (duplicate-runtime // thread-local resilient). `fancy_ptr` is a leaked `Arc` raw pointer; to @@ -1242,6 +1233,10 @@ pub(crate) fn lookup_fancy_regex(re: *const RegExpHeader) -> Option Option> { + // Same first-use build as `lookup_fancy_regex`: a null + // `repeat_matcher_ptr` means "not built yet" before it can mean "this + // pattern needs no backtracking matcher". + lazy::ensure_regex_compiled(re); unsafe { if regex_header_has_magic(re) && !(*re).repeat_matcher_ptr.is_null() { let raw = (*re).repeat_matcher_ptr as *const repeat_matcher::RepeatMatcherRegex; @@ -1464,7 +1459,7 @@ pub extern "C" fn js_string_replace_regex( return replace_regex_str_fancy(str_data, &fre, (*re).global, repl_str); } - let regex = &*(*re).regex_ptr; + let regex = lazy::header_std_regex(re); let global = (*re).global; let has_named_groups = regex.capture_names().any(|n| n.is_some()); @@ -1585,7 +1580,7 @@ pub extern "C" fn js_string_split_regex_n( // zero-width matches (it emits leading/trailing/consecutive empty // strings the spec's `e == p` skip suppresses) and never splices // captured groups, so walk the string the spec's way instead. - crate::string::spec_regex_split(&*(*re).regex_ptr, &str_data, limit) + crate::string::spec_regex_split(lazy::header_std_regex(re), &str_data, limit) }; let arr = crate::array::js_array_alloc(parts.len() as u32); @@ -1637,7 +1632,7 @@ pub extern "C" fn js_string_search_regex(s: *const StringHeader, re: *const RegE }; } - let regex = &*(*re).regex_ptr; + let regex = lazy::header_std_regex(re); match regex.find(str_data) { Some(m) => { // `String.prototype.search` returns a JS string index — UTF-16 diff --git a/crates/perry-runtime/src/regex/exec.rs b/crates/perry-runtime/src/regex/exec.rs index fadfd9dbe7..76ee03da26 100644 --- a/crates/perry-runtime/src/regex/exec.rs +++ b/crates/perry-runtime/src/regex/exec.rs @@ -38,7 +38,7 @@ pub extern "C" fn js_regexp_exec( // of them may reach Phase 2 (#8449). let (owned, has_indices) = unsafe { let str_data = string_as_str(s); - let regex = &*(*re).regex_ptr; + let regex = super::lazy::header_std_regex(re); let global = (*re).global; let sticky = (*re).sticky; let has_indices = (*re).has_indices; diff --git a/crates/perry-runtime/src/regex/lazy.rs b/crates/perry-runtime/src/regex/lazy.rs new file mode 100644 index 0000000000..027e62cef3 --- /dev/null +++ b/crates/perry-runtime/src/regex/lazy.rs @@ -0,0 +1,267 @@ +//! Lazy compilation of RegExp programs. +//! +//! # Why +//! +//! Constructing a RegExp used to BUILD it. `js_regexp_new` ran +//! `compile_and_cache_regex_checked`, which is a full `regex::Regex::new` — +//! `regex_syntax` parse + HIR translate (Unicode case folding) + a Thompson +//! NFA build + the meta engine's strategy selection. That is the single most +//! expensive thing a JS program can do per regex literal, and a bundle +//! evaluates hundreds of literals at module-init time whether or not the run +//! ever matches with them: a symbolized `perf` profile of the claude-code +//! bundle's `--help` (a run that prints text and exits) put ~14% of ALL +//! retired instructions inside `regex_syntax`/`regex_automata` compilation, +//! against 0.11% for the whole of the compiled JavaScript. +//! +//! Measured on that bundle's own 2,378 distinct regex literals: +//! +//! | step | cost | +//! |---|---| +//! | `regex::Regex::new` (what construction used to do) | 82 µs/pattern | +//! | `regex_syntax::Parser::parse` (syntax only) | 4.6 µs/pattern | +//! +//! and a fixture of 200 literals where exactly ONE is ever executed spent +//! 73 ms of its 79 ms wall clock inside construction (Node: 1 ms). +//! +//! # What is lazy and what is not +//! +//! Only the *program build* moves. Everything observable at construction +//! stays at construction: +//! +//! * a syntactically invalid pattern still throws `SyntaxError` from +//! `js_regexp_new` / `RegExp.prototype.compile`, at the same point in the +//! program — [`std_engine_syntax_ok`] runs the SAME parser +//! `build_std_regex` would run, on the SAME translated + flag-prefixed + +//! REDoS-collapsed string, and anything it rejects falls through to the +//! unchanged both-engines check (so the fancy-regex fallback for +//! lookbehind/backreferences still decides, and still throws when both +//! engines refuse); +//! * `.source` / `.flags` / `.global` / `.sticky` / `lastIndex` are header +//! and side-table reads that never touched the compiled program; +//! * identity is untouched — `js_regexp_new` still `gc_malloc`s a fresh +//! header per evaluation. +//! +//! The build itself happens on the first operation that needs a matcher, +//! through [`ensure_regex_compiled`], and installs exactly the pointers +//! `js_regexp_new` used to install eagerly (`regex_ptr`, `fancy_ptr`, +//! `repeat_matcher_ptr`), each a leaked `Arc` the header owns. + +use std::sync::Arc; + +use regex::Regex; + +use super::grammar::{collapse_redos_guard_quantifiers, js_regex_to_rust}; +use super::{ + evict_regex_cache_if_full, get_or_compile_regex, is_valid_ptr, is_valid_regex_ptr, + string_as_str, RegExpHeader, FANCY_CACHE, REGEX_SOURCE_TABLE, REPEAT_MATCHER_CACHE, + VALIDATED_PATTERNS, +}; + +/// The exact string `build_std_regex` is handed for `(pattern, flags)`: the +/// JS→Rust translation with the inline `(?ims)` mode prefix the flags imply. +/// +/// Extracted so the eager syntax check and the lazy build cannot drift — a +/// validator that inspects a DIFFERENT string than the builder would either +/// throw on a pattern that compiles or accept one that does not. +pub(super) fn flag_prefixed_pattern(pattern: &str, flags: &str) -> String { + let translated = js_regex_to_rust(pattern); + let case_insensitive = flags.contains('i'); + let multiline = flags.contains('m'); + // #2828: the `s` (dotAll) flag maps directly onto the Rust `regex` + // crate's `(?s)` inline mode, so `.` matches newlines. + let dot_all = flags.contains('s'); + if !(case_insensitive || multiline || dot_all) { + return translated; + } + let mut prefix = String::from("(?"); + if case_insensitive { + prefix.push('i'); + } + if multiline { + prefix.push('m'); + } + if dot_all { + prefix.push('s'); + } + prefix.push(')'); + format!("{}{}", prefix, translated) +} + +/// Does the standard engine's PARSER accept this pattern? +/// +/// This is the cheap half of `build_std_regex`: `regex::RegexBuilder::build` +/// parses and then builds an NFA, and only the parse can report a syntax +/// error. Asking the parse alone answers "is this a `SyntaxError`?" without +/// building any automaton. +/// +/// The parse itself has two halves, and the cheap one is enough almost +/// always. `regex_syntax`'s AST parse is pure grammar — unbalanced groups, +/// `a{2,1}`, `[z-a]`, dangling `)` all fail there. Its HIR *translate* pass is +/// where Unicode class expansion and, under `i`, `case_fold_simple` run: +/// `ClassUnicodeRange::case_fold_simple` is 3.53% of a claude-code `--help` +/// profile on its own, and on a corpus of case-folding-heavy literals translate +/// costs 138 µs/pattern against the AST parse's 3.4 µs — 40x. +/// +/// Exactly one class of diagnostic is translate-only for the strings perry +/// produces: an unknown Unicode property name (`\p{Bogus}`). Perry never emits +/// the other translate-only errors' triggers — they all require `(?-u)` / +/// non-UTF-8 matching, and `js_regex_to_rust` always emits Unicode-mode +/// patterns (`\x{…}` escapes, never raw bytes). So: AST-parse everything, and +/// pay for the full translate only when the translated pattern mentions a +/// property. That is 0.7% of the claude-code bundle's literals (16 of 2,378); +/// the substring test deliberately over-triggers (a literal backslash followed +/// by `p` also matches), because over-triggering only costs time. +/// +/// `tests::syntax_check_agrees_with_full_build` pins the whole thing against +/// `build_std_regex` on a corpus, so neither this split nor a `regex` upgrade +/// can silently move where a `SyntaxError` is raised. +/// +/// `false` is NOT a verdict of "invalid": it only means the linear engine +/// refused, which is also how every lookbehind/backreference pattern answers. +/// The caller falls back to the unchanged both-engines path, which owns the +/// `SyntaxError` decision. +pub(super) fn std_engine_syntax_ok(pattern: &str, flags: &str) -> bool { + // `build_std_regex` collapses ReDoS-guard bounded quantifiers before + // building; validate the same post-collapse string. + let collapsed = collapse_redos_guard_quantifiers(&flag_prefixed_pattern(pattern, flags)); + if collapsed.contains("\\p") || collapsed.contains("\\P") { + return regex_syntax::Parser::new().parse(&collapsed).is_ok(); + } + regex_syntax::ast::parse::Parser::new() + .parse(&collapsed) + .is_ok() +} + +/// Has `(pattern, flags)` already cleared validation on this thread? +/// +/// Validity is a pure function of `(pattern, flags)`, so re-deriving it is +/// pure cost. Construction used to reach this conclusion via a `REGEX_CACHE` +/// hit, which only worked because construction also compiled; with the build +/// deferred, the cache can be empty for a pattern that has been constructed a +/// thousand times (`string-width`/`emoji-regex` build a fresh ~12,807-char +/// literal on every measurement), so the fact is recorded separately. +pub(super) fn pattern_already_validated(pattern: &str, flags: &str) -> bool { + VALIDATED_PATTERNS.with(|set| { + set.borrow() + .contains_key(&(pattern.to_string(), flags.to_string())) + }) +} + +/// Record that `(pattern, flags)` passed validation. Capped and +/// cleared-on-overflow exactly like the compiled-program caches — the +/// consequence of a clear is a repeated parse, not a wrong answer. +pub(super) fn mark_pattern_validated(pattern: &str, flags: &str) { + VALIDATED_PATTERNS.with(|set| { + let mut set = set.borrow_mut(); + evict_regex_cache_if_full(&mut set); + set.insert((pattern.to_string(), flags.to_string()), ()); + }); +} + +/// The `(source, flags)` a header was built from. +/// +/// Prefers the GC-survivable side table (issue #637) and falls back to the +/// header's own string payloads, which — unlike the thread-local table — are +/// readable from a second statically-linked copy of the runtime (Wall 18). +pub(super) fn source_and_flags(re: *const RegExpHeader) -> (String, String) { + if let Some(source) = + REGEX_SOURCE_TABLE.with(|table| table.borrow().get(&(re as usize)).cloned()) + { + return source; + } + unsafe { + let pattern = if is_valid_ptr((*re).pattern_ptr) { + string_as_str((*re).pattern_ptr).to_string() + } else { + String::new() + }; + let flags = if is_valid_ptr((*re).flags_ptr) { + string_as_str((*re).flags_ptr).to_string() + } else { + String::new() + }; + (pattern, flags) + } +} + +/// Build this header's compiled program(s) if it has none yet. +/// +/// `regex_ptr == null` is the "not built yet" state. It is published LAST so +/// a header is never observable as built while `fancy_ptr` / +/// `repeat_matcher_ptr` are still stale — every reader that consults those +/// two goes through [`lookup_fancy_regex`](super::lookup_fancy_regex) / +/// `lookup_repeat_matcher`, which call this first. +/// +/// The header OWNS a leaked `Arc` reference to each program (mirroring what +/// `js_regexp_new` used to do inline), so the capped `REGEX_CACHE` / +/// `FANCY_CACHE` / `REPEAT_MATCHER_CACHE` can evict without invalidating a +/// live receiver. +/// +/// Contains no JS allocation and cannot re-enter the interpreter, so it is +/// safe to call from inside a phase that holds a borrow of a GC string. +/// +/// # Preconditions +/// Every call site has already established `is_valid_regex_ptr(re)` — this is +/// reached only from the guarded entry points (`js_regexp_test`, +/// `js_regexp_exec`, the `String.prototype` regex methods, …). That matters +/// for cost, not just for tidiness: `is_valid_regex_ptr` reaches +/// `try_read_gc_header` and heap-space classification, which is far too +/// expensive to repeat on the already-built path, and this runs up to three +/// times per match operation. So the hot path is two loads — is the pointer +/// plausible, is the program already there — and the full validation lives in +/// the cold builder. +#[inline] +pub(crate) fn ensure_regex_compiled(re: *const RegExpHeader) { + if !is_valid_ptr(re) { + return; + } + if unsafe { !(*re).regex_ptr.is_null() } { + return; + } + build_and_install_programs(re); +} + +#[cold] +fn build_and_install_programs(re: *const RegExpHeader) { + // The one place the precondition is re-checked, so a caller that has not + // validated cannot corrupt an unrelated allocation. + if !is_valid_regex_ptr(re) { + return; + } + let (pattern, flags) = source_and_flags(re); + let arc = get_or_compile_regex(&pattern, &flags); + let regex_ptr = Arc::into_raw(arc) as *mut Regex; + let fancy_ptr: *const () = + FANCY_CACHE.with( + |fc| match fc.borrow().get(&(pattern.clone(), flags.clone())) { + Some(arc) => Arc::into_raw(arc.clone()) as *const (), + None => std::ptr::null(), + }, + ); + let repeat_matcher_ptr: *const () = + REPEAT_MATCHER_CACHE.with(|cache| match cache.borrow().get(&(pattern, flags)) { + Some(arc) => Arc::into_raw(arc.clone()) as *const (), + None => std::ptr::null(), + }); + unsafe { + let re = re as *mut RegExpHeader; + (*re).fancy_ptr = fancy_ptr; + (*re).repeat_matcher_ptr = repeat_matcher_ptr; + // Publish last: `regex_ptr` is the built/not-built flag. + (*re).regex_ptr = regex_ptr; + } +} + +/// The header's standard-engine program, building it on first use. +/// +/// Every `&*(*re).regex_ptr` in the tree goes through here — the field is +/// null until something needs a matcher. +/// +/// # Safety +/// `re` must be a live `RegExpHeader` (all callers gate on +/// `is_valid_regex_ptr`); the returned reference borrows a leaked `Arc` the +/// header owns for its lifetime. +pub(crate) unsafe fn header_std_regex<'a>(re: *const RegExpHeader) -> &'a Regex { + ensure_regex_compiled(re); + &*(*re).regex_ptr +} diff --git a/crates/perry-runtime/src/regex/match_all.rs b/crates/perry-runtime/src/regex/match_all.rs index fbe848e8c2..fdd01c5660 100644 --- a/crates/perry-runtime/src/regex/match_all.rs +++ b/crates/perry-runtime/src/regex/match_all.rs @@ -143,7 +143,7 @@ unsafe fn materialize_match_all_results( }); } } else { - let regex = &*(*re).regex_ptr; + let regex = super::lazy::header_std_regex(re); let named_names: Vec<(usize, String)> = regex .capture_names() .enumerate() diff --git a/crates/perry-runtime/src/regex/match_string.rs b/crates/perry-runtime/src/regex/match_string.rs index dd9f0cd497..7f3c38950a 100644 --- a/crates/perry-runtime/src/regex/match_string.rs +++ b/crates/perry-runtime/src/regex/match_string.rs @@ -78,7 +78,7 @@ pub extern "C" fn js_string_match( // UTF-16/WTF-8 metadata while the engine's `Captures` may borrow `s`. let owned = unsafe { let str_data = string_as_str(s); - let regex = &*(*re).regex_ptr; + let regex = super::lazy::header_std_regex(re); let global = (*re).global; let has_indices = (*re).has_indices; diff --git a/crates/perry-runtime/src/regex/repeat_matcher.rs b/crates/perry-runtime/src/regex/repeat_matcher.rs index e962c0e181..107e17dded 100644 --- a/crates/perry-runtime/src/regex/repeat_matcher.rs +++ b/crates/perry-runtime/src/regex/repeat_matcher.rs @@ -301,17 +301,10 @@ pub(super) fn compile(pattern: &str, flags: &str) -> Option } fn source_and_flags(re: *const super::RegExpHeader) -> (String, String) { - if let Some(source) = - super::REGEX_SOURCE_TABLE.with(|table| table.borrow().get(&(re as usize)).cloned()) - { - return source; - } - unsafe { - ( - super::string_as_str((*re).pattern_ptr).to_string(), - super::string_as_str((*re).flags_ptr).to_string(), - ) - } + // One definition, shared with the lazy first-use builder: both need the + // `(source, flags)` a header was constructed from, and a second copy of + // the side-table-then-header fallback would be a place for them to drift. + super::lazy::source_and_flags(re) } fn decode_wtf8_units(bytes: &[u8]) -> Vec { diff --git a/crates/perry-runtime/src/regex/replace_expand.rs b/crates/perry-runtime/src/regex/replace_expand.rs index 43dfa5a3db..7b38134399 100644 --- a/crates/perry-runtime/src/regex/replace_expand.rs +++ b/crates/perry-runtime/src/regex/replace_expand.rs @@ -345,7 +345,7 @@ pub extern "C" fn js_string_replace_regex_fn( } unsafe { - let regex = &*(*re).regex_ptr; + let regex = super::lazy::header_std_regex(re); let global = (*re).global; // Extract closure pointer from NaN-boxed value @@ -482,7 +482,7 @@ pub extern "C" fn js_string_replace_regex_named( return replace_regex_str_fancy(str_data, &fre, (*re).global, repl_str); } - let regex = &*(*re).regex_ptr; + let regex = super::lazy::header_std_regex(re); let global = (*re).global; let has_named_groups = regex.capture_names().any(|n| n.is_some()); diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 42b2edb796..402a4a3984 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -872,3 +872,201 @@ fn search_returns_utf16_index() { let re = js_regexp_new(make_string("x"), make_string("")); assert_eq!(js_string_search_regex(make_string("𝌆x"), re), 2); } + +/// The eager syntax check must accept EXACTLY what the full build accepts. +/// +/// `js_regexp_new` no longer answers "is this a `SyntaxError`?" by building the +/// automaton — it asks the standard engine's parser alone +/// (`lazy::std_engine_syntax_ok`) and only falls through to the both-engines +/// path when the parser refuses. That is sound only while parser-acceptance and +/// builder-acceptance agree; if a future `regex` release moves a diagnostic out +/// of the parser and into the NFA build, a pattern would silently stop throwing +/// at construction. This is the gate for that: it disagrees loudly rather than +/// letting the divergence ship. +/// +/// Both directions matter, so the corpus deliberately contains patterns the +/// linear engine ACCEPTS, ones it rejects for lack of a feature (lookbehind, +/// backreferences — the fancy-regex fallback's territory) and ones that are +/// genuinely malformed. +#[test] +fn syntax_check_agrees_with_full_build() { + let corpus: &[(&str, &str)] = &[ + // Ordinary shapes. + ("abc", ""), + ("^v?(\\d+)\\.(\\d+)\\.(\\d+)$", ""), + ("[A-Za-z0-9_.+-]+@[\\w-]+\\.[\\w.-]+", "i"), + ("(?:https?|ftp)://[^\\s]+", "gi"), + ("\\s+", "gm"), + ("a.b", "s"), + ("(foo|bar|baz){2,4}", "i"), + ("x{0,250}", ""), + ("\\d{1,256}", ""), + // Unicode classes / properties / astral — the case-folding shapes. + ("[A-Za-zÀ-ɏ]+", "i"), + ("[Ѐ-ӿͰ-Ͽ]*", "giu"), + ("\\p{L}+", "u"), + ("\\p{Script=Greek}", "u"), + ("[\\u{1F600}-\\u{1F64F}]", "u"), + ("[←-⇿☀-⛿]", "u"), + ("\\w+\\b", "iu"), + // Fancy-only (the linear engine refuses; fancy-regex accepts). + ("(?<=pre)\\d+", ""), + ("(? x[0]))); +console.log("named:" + JSON.stringify("2026-08".match(/(?\d{4})-(?\d{2})/)?.groups)); + +// RegExp.prototype.compile re-initialises a header whose program was never +// built (the old pointer it releases is null in that case). +const recompiled = /zzz/g; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(recompiled as any).compile("q+", "g"); +console.log("compiled-source:" + recompiled.source + " flags:" + recompiled.flags); +console.log("compiled-match:" + JSON.stringify("aqqqb".match(recompiled)));