From 2e495b0d9e8b43bd6c8fbfb15fce8e005e1809e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 30 Aug 2026 21:38:14 +0200 Subject: [PATCH] perf(runtime): translate "any character"/"no character" classes to forms regex_syntax does not case-fold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[\s\S]` and `(?s:.)` denote the same set, but `regex_syntax`'s HIR translator case-folds a character class by looping over EVERY CODE POINT in each of its ranges: for cp in (start..=end).filter_map(char::from_u32) { for &cp_folded in folder.mapping(cp) { ranges.push(...) } } `[\s\S]` canonicalizes to the single range `\x{0}-\x{10FFFF}`, so under `(?i)` that loop runs 1,114,112 times to compute a fold that cannot change anything — the class already contains every character. `.` under `(?s)` is `Hir::dot`, not a class, and is never folded. The negated mirror `[^\s\S]` is just as bad: the fold runs on the positive set, before the negation. `fancy_regex::Regex::new`, best of 3, same machine: (?i)[\s\S]*? 7.72 ms (?i)(?s:.)*? 41.7 us 185x (?i)[^\s\S] 7.46 ms (?i)[a&&b] 11.4 us 654x and it compounds with the number of occurrences (three `[\s\S]` in one pattern: 23.0 ms vs 34.9 us). Why it matters: a symbolized perf profile of the claude-code bundle's `--help` put `ClassUnicodeRange::case_fold_simple` at 3.78% of ALL retired instructions — the second-largest symbol in the binary. Recording a stack on every one of its 2,754 calls (uprobe + dwarf) attributed 99.6% of them to construction-time validation, not to the lazy first-use build #9178 introduced; a gdb capture of the argument named the patterns: five successive `new RegExp` refinements of marked's `/^ {0,3}(?:<(script|pre|style|textarea)[\s>][\s\S]*?.../i` HTML-block rule — the `i` flag plus eight `[\s\S]` — and one URL validator. A probe on `get_or_compile_regex` (verified able to fire: it saw 3,713 calls) shows that run never matches with any of them. This also fixes the shape in perry's OWN output: JS `[^]` was translated to `[\s\S]` and JS `[]` to `[^\s\S]`, so perry manufactured the pathological form itself. The rewrite is deliberately narrow: only a six-character class whose two members are a shorthand and its own complement, and only outside a class. The group is non-capturing and `collect_capture_spans` indexes the input, so capture numbering is untouched. Gates: near-miss unit tests (mutation-checked twice — dropping the closing-bracket check makes `near_misses_are_left_alone` fail, and emitting `(?s:.)` for `[]` makes the behaviour test fail), a match-behaviour test covering the negated forms, quantifiers, capture numbering and `.source`, the 63 existing regex tests, the full 2,855-test perry-runtime suite, and `syntax_check_agrees_with_full_build` over 12,077 patterns (claude-code + pi + kimi + 6,297 mutations). --- changelog.d/9214-regex-any-char-no-fold.md | 12 ++ crates/perry-runtime/src/regex/grammar.rs | 158 ++++++++++++++++++++- crates/perry-runtime/src/regex/tests.rs | 61 ++++++++ 3 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 changelog.d/9214-regex-any-char-no-fold.md diff --git a/changelog.d/9214-regex-any-char-no-fold.md b/changelog.d/9214-regex-any-char-no-fold.md new file mode 100644 index 0000000000..4541b49468 --- /dev/null +++ b/changelog.d/9214-regex-any-char-no-fold.md @@ -0,0 +1,12 @@ +### Performance + +- **`[^]` and `[]` no longer make regex construction case-fold a million code + points.** Perry translated JS's "any character" class to `[\s\S]` and its + empty class to `[^\s\S]`. Under the `i` flag, `regex_syntax`'s + `case_fold_simple` walks *every code point in a range*, and `[\s\S]` + canonicalizes to `\x{0}-\x{10FFFF}` — so each such class ran a 1,114,112-step + loop to compute a fold that cannot change anything. They now translate to + `(?s:.)` and `[a&&b]`, which are never folded. Isolated: `(?i)[\s\S]*?` + 7.72 ms → 41.7 µs (185×), `(?i)[^\s\S]` 7.46 ms → 11.4 µs (654×). On + claude-code's `--help`, −336 million instructions (**−3.78%**), with output + byte-identical to node. diff --git a/crates/perry-runtime/src/regex/grammar.rs b/crates/perry-runtime/src/regex/grammar.rs index 4abad899b4..6fdeef131e 100644 --- a/crates/perry-runtime/src/regex/grammar.rs +++ b/crates/perry-runtime/src/regex/grammar.rs @@ -1420,6 +1420,84 @@ fn quantifier_after(chars: &[char], start: usize) -> Option<(usize, usize, bool) Some((start, end, lower_zero)) } +/// Emit the cheapest spelling of "matches any code point". +/// +/// `[\s\S]` and `(?s:.)` denote exactly the same set — every Unicode scalar +/// value — but they cost wildly different amounts to *compile*, and the +/// difference only appears once the `i` flag is present. +/// +/// `regex_syntax`'s HIR translator case-folds a character class by calling +/// `ClassUnicodeRange::case_fold_simple` on each of its ranges, and that +/// function loops over EVERY CODE POINT in the range: +/// +/// ```text +/// for cp in (start..=end).filter_map(char::from_u32) { +/// for &cp_folded in folder.mapping(cp) { ranges.push(...) } +/// } +/// ``` +/// +/// `[\s\S]` canonicalizes to the single range `\x{0}-\x{10FFFF}`, so under +/// `(?i)` that loop runs 1,114,112 times — to compute a fold that cannot +/// change anything, since the class already contains every character. `.` +/// under `(?s)` is not a class at all (`Hir::dot`), so the translator never +/// folds it. +/// +/// Measured (`fancy_regex::Regex::new`, best of 3, same machine): +/// +/// | pattern | build | +/// |---|---| +/// | `(?i)[\s\S]*?` | 7.72 ms | +/// | `(?i)(?s:.)*?` | 41.7 µs | +/// +/// — 185x, and it compounds: three `[\s\S]` in one pattern cost 23.0 ms +/// against 34.9 µs. This is not a microbenchmark curiosity. A symbolized +/// `perf` profile of the claude-code bundle's `--help` put +/// `ClassUnicodeRange::case_fold_simple` at 3.78% of ALL retired instructions +/// — the second-largest symbol in the binary — and an exact uprobe stack +/// capture (a stack recorded on every one of its 2,754 calls) attributed +/// 99.6% of them to construction-time validation of six patterns, five of +/// them successive `new RegExp` refinements of marked's +/// `/^ {0,3}(?:<(script|pre|style|textarea)[\s>][\s\S]*?…/i` HTML-block rule +/// — which carries the `i` flag and eight `[\s\S]`, and which that run never +/// matches with even once. +/// +/// The group is non-capturing, so capture numbering is untouched (and +/// `collect_capture_spans` indexes the INPUT, not this output). `(?s:…)` +/// scopes the `s` flag to the dot, so a pattern that lacks the JS `s` flag +/// does not silently gain `dotAll` anywhere else. +fn push_any_char(result: &mut String) { + result.push_str("(?s:.)"); +} + +/// If a character class starting at `chars[i]` spells "any code point", the +/// number of input chars it occupies. +/// +/// Recognizes exactly the closed complementary pairs — `[\s\S]`, `[\S\s]`, +/// `[\d\D]`, `[\D\d]`, `[\w\W]`, `[\W\w]` — whose union is by definition the +/// whole code point space, in either order. Deliberately narrow: it fires only +/// on a six-character class whose two members are a shorthand and its own +/// complement, so there is no set arithmetic to get wrong — the answer is +/// always "everything". +fn any_char_class_width(chars: &[char], i: usize) -> Option { + // `[` `\` a `\` b `]` + if chars.get(i) != Some(&'[') + || chars.get(i + 1) != Some(&'\\') + || chars.get(i + 3) != Some(&'\\') + || chars.get(i + 5) != Some(&']') + { + return None; + } + let (a, b) = (*chars.get(i + 2)?, *chars.get(i + 4)?); + if matches!( + (a, b), + ('s', 'S') | ('S', 's') | ('d', 'D') | ('D', 'd') | ('w', 'W') | ('W', 'w') + ) { + Some(6) + } else { + None + } +} + pub(super) fn js_regex_to_rust(pattern: &str) -> String { let folded = fold_surrogate_pairs(pattern); let folded = normalize_quantified_lookaround(&folded); @@ -1636,15 +1714,30 @@ pub(super) fn js_regex_to_rust(pattern: &str) -> String { } else if chars.get(i + 1) == Some(&']') { // JS: `[]` is an *empty* character class that never matches // (the `]` immediately after `[` closes the class). The Rust - // `regex` crate rejects `[]`, so emit an unsatisfiable class. - result.push_str("[^\\s\\S]"); + // `regex` crate rejects a literal `[]`, so emit an + // unsatisfiable one. `[a&&b]` (an empty intersection) rather + // than the obvious `[^\s\S]`, for the reason in + // `push_any_char`: `[^\s\S]` is the NEGATION of a class + // covering every code point, and `regex_syntax` case-folds the + // positive set BEFORE negating it — so under `(?i)` the + // never-matching class costs the same 1.1-million-iteration + // loop as the matches-everything one (7.46 ms vs 11 us here). + result.push_str("[a&&b]"); i += 2; } else if chars.get(i + 1) == Some(&'^') && chars.get(i + 2) == Some(&']') { // JS: `[^]` is a negated empty class — it matches *any* code // point, including line terminators. Rust rejects `[^]`, so - // emit the equivalent `[\s\S]`. - result.push_str("[\\s\\S]"); + // emit the equivalent "any character": `(?s:.)`, NOT the + // `[\s\S]` spelling — see `push_any_char` for why the two + // differ by 185x under the `i` flag. + push_any_char(&mut result); i += 3; + } else if let Some(width) = any_char_class_width(&chars, i) { + // A class the author wrote that already means "any character" + // (`[\s\S]`, `[\S\s]`, `[\d\D]`, `[\w\W]`, …). Same rewrite, + // same reason. + push_any_char(&mut result); + i += width; } else { in_class = true; result.push('['); @@ -2073,3 +2166,60 @@ mod tests { assert!(!re.is_match("a b")); } } + +#[cfg(test)] +mod any_char_rewrite_tests { + use super::js_regex_to_rust; + + /// The rewrite fires on exactly the closed complementary pairs, and on + /// JS's `[^]`. + #[test] + fn any_char_classes_become_a_dot() { + for (js, rust) in [ + (r"[\s\S]", "(?s:.)"), + (r"[\S\s]", "(?s:.)"), + (r"[\d\D]", "(?s:.)"), + (r"[\D\d]", "(?s:.)"), + (r"[\w\W]", "(?s:.)"), + (r"[\W\w]", "(?s:.)"), + ("[^]", "(?s:.)"), + // The mirror image: JS `[]` matches NOTHING, and the obvious + // `[^\\s\\S]` spelling is just as pathological under `i` (the fold + // runs on the positive set, before the negation). + ("[]", "[a&&b]"), + // In context, with quantifiers and neighbours. + (r"a[\s\S]*?b", "a(?s:.)*?b"), + (r"[\s\S]{2,3}", "(?s:.){2,3}"), + (r"[\s\S]*", "(?s:.)*"), + ] { + assert_eq!(js_regex_to_rust(js), rust, "translating /{js}/"); + } + } + + /// Everything that is NOT provably "every code point" must pass through + /// untouched. A false positive here silently widens a character class, + /// which no syntax check would catch — only a wrong match result. + #[test] + fn near_misses_are_left_alone() { + for js in [ + r"[\s\s]", // same shorthand twice, not a complement + r"[\S\S]", // + r"[\s\D]", // two shorthands, but not complements + r"[\w\S]", // + r"[\s\Sx]", // extra member + r"[x\s\S]", // extra member + r"[^\s\S]", // NEGATED — matches nothing, the exact opposite + r"[^\w\W]", // + r"[\s]", // single member + r"[\s\S", // unterminated + r"\[\s\S\]", // escaped brackets: a literal `[`, not a class + r"[a[\s\S]]", // an inner `[` inside a class is a literal in JS + ] { + let out = js_regex_to_rust(js); + assert!( + !out.contains("(?s:.)"), + "/{js}/ must not be rewritten to any-char, got {out}" + ); + } + } +} diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 402a4a3984..4ba9ed2454 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -1070,3 +1070,64 @@ fn validated_pattern_set_is_capped() { "VALIDATED_PATTERNS must stay capped at {REGEX_CACHE_MAX_ENTRIES} entries, got {len}" ); } + +/// The `[\s\S]` → `(?s:.)` rewrite must not move a single match result. +/// +/// The rewrite exists purely to dodge a 1.1-million-iteration case fold in +/// `regex_syntax` (see `grammar::push_any_char`), so the only thing that may +/// change is how long construction takes. Everything a program can observe — +/// what matches, what a capture group holds, which group number it is, and +/// that the NEGATED forms still match nothing — is pinned here, because a +/// silently widened character class produces no error anywhere: only a wrong +/// answer, on inputs a syntax test never looks at. +#[test] +fn any_char_rewrite_preserves_match_behaviour() { + // Matches every code point, newlines included, with and without `i`. + for pattern in ["[\\s\\S]", "[^]", "[\\d\\D]", "[\\w\\W]", "[\\S\\s]"] { + for flags in ["", "i", "u", "iu", "m"] { + let re = js_regexp_new(make_string(pattern), make_string(flags)); + for subject in ["a", "\n", " ", "\u{1F600}", "Ω", "\r"] { + assert!( + js_regexp_test(re, make_string(subject)) != 0, + "/{pattern}/{flags} must match {subject:?}" + ); + } + } + } + + // The negated forms are the exact opposite and must still match NOTHING. + for pattern in ["[^\\s\\S]", "[^\\w\\W]", "[]"] { + let re = js_regexp_new(make_string(pattern), make_string("i")); + for subject in ["a", "\n", "Ω"] { + assert!( + js_regexp_test(re, make_string(subject)) == 0, + "/{pattern}/i must not match {subject:?}" + ); + } + } + + // A class that is NOT a complementary pair keeps its narrow meaning. + let narrow = js_regexp_new(make_string("[\\d\\s]"), make_string("i")); + assert!(js_regexp_test(narrow, make_string("7")) != 0); + assert!(js_regexp_test(narrow, make_string("a")) == 0); + + // The rewrite emits a NON-capturing group, so group numbering is + // unchanged: `$1` is still `b`, not the any-char. + let re = js_regexp_new(make_string("a[\\s\\S](b)"), make_string("")); + let m = js_regexp_exec(re, make_string("a\nb")); + assert!(!m.is_null(), "a[\\s\\S](b) must match \"a\\nb\""); + + // Quantifiers still bind to the any-char, lazily and greedily. + let lazy = js_regexp_new(make_string("([\\s\\S]*?)"), make_string("i")); + assert!(js_regexp_test(lazy, make_string("one\ntwo")) != 0); + let greedy = js_regexp_new(make_string("^[\\s\\S]{3}$"), make_string("")); + assert!(js_regexp_test(greedy, make_string("a\nb")) != 0); + assert!(js_regexp_test(greedy, make_string("a\nbc")) == 0); + + // `.source` still reports what the author wrote, not the translation. + let re = js_regexp_new(make_string("[\\s\\S]+"), make_string("gi")); + assert_eq!( + string_payload(js_regexp_get_source(re)), + b"[\\s\\S]+".to_vec() + ); +}