From b7b0e8ef33245c5c305cd2c894ea34c9d2cd28b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 31 Aug 2026 06:58:16 +0200 Subject: [PATCH 1/4] test(runtime): add regexp word and dot differential harness --- scripts/regex_9217_9218_differential.mjs | 242 ++++++++++++++++++ .../test_gap_9217_9218_regexp_word_dot.ts | 92 +++++++ 2 files changed, 334 insertions(+) create mode 100644 scripts/regex_9217_9218_differential.mjs create mode 100644 test-files/test_gap_9217_9218_regexp_word_dot.ts diff --git a/scripts/regex_9217_9218_differential.mjs b/scripts/regex_9217_9218_differential.mjs new file mode 100644 index 0000000000..9cfb63a573 --- /dev/null +++ b/scripts/regex_9217_9218_differential.mjs @@ -0,0 +1,242 @@ +#!/usr/bin/env node + +// Differential runner for #9217 / #9218. +// +// With no arguments it runs a focused corpus. To replay a bundle corpus, pass +// a UTF-8 TSV whose first field is the RegExp source and whose optional second +// field is its flags: +// +// node scripts/regex_9217_9218_differential.mjs --corpus /tmp/regexes.tsv +// +// The runner deliberately emits one line per pattern. Its headline divergence +// count is therefore a count of divergent PATTERN RECORDS, not an inflated +// count of individual subject cells. Every RegExp is fresh for every subject, +// so `g` tests matching without carrying lastIndex into the next probe. + +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(scriptDir, ".."); + +function fail(message) { + console.error(message); + process.exit(2); +} + +let corpusPath = null; +let perryPath = join(repoRoot, "target", "release", "perry"); +for (let i = 2; i < process.argv.length; i += 1) { + const arg = process.argv[i]; + if (arg === "--corpus") { + corpusPath = process.argv[++i]; + if (!corpusPath) fail("--corpus needs a path"); + } else if (arg.startsWith("--corpus=")) { + corpusPath = arg.slice("--corpus=".length); + } else if (arg === "--perry") { + perryPath = resolve(process.argv[++i] ?? ""); + } else if (arg.startsWith("--perry=")) { + perryPath = resolve(arg.slice("--perry=".length)); + } else { + fail(`unknown argument: ${arg}`); + } +} + +const subjects = [ + "Az_09-", // ASCII word/non-word + "café", // accented Latin + "Ωμέγα", // Greek + "漢字", // CJK + "😀", // astral emoji + "K", // KELVIN SIGN: word only under i+u + "ſ", // LATIN SMALL LETTER LONG S: word only under i+u + "\n", // LF + "\r", // CR + "
", // LINE SEPARATOR + "
", // PARAGRAPH SEPARATOR + "\t\r\n", // tab + CRLF, including the issue #9218 repro +]; + +const focusedCases = [ + // Word escapes outside classes, with every relevant flag represented. + ["^\\w+$", ""], + ["^\\w+$", "i"], + ["^\\w+$", "u"], + ["^\\w+$", "iu"], + ["^\\w+$", "g"], + ["^\\w+$", "m"], + ["^\\w+$", "s"], + ["^\\w+$", "gimsu"], + ["^\\W+$", ""], + ["^\\W+$", "i"], + ["^\\W+$", "u"], + ["^\\W+$", "iu"], + ["^\\W+$", "g"], + ["\\b\\w+\\b", ""], + ["\\b\\w+\\b", "i"], + ["\\b\\w+\\b", "u"], + ["\\b\\w+\\b", "iu"], + ["\\b\\w+\\b", "gim"], + ["\\B\\W+\\B", ""], + ["\\B\\W+\\B", "iu"], + ["x\\b.", ""], + ["x\\B.", "m"], + + // Word escapes and the class meanings of \b / \B inside classes. + ["^[\\w-]+$", ""], + ["^[\\w-]+$", "i"], + ["^[\\w-]+$", "u"], + ["^[\\w-]+$", "iu"], + ["^[^\\w]+$", ""], + ["^[^\\w]+$", "i"], + ["^[^\\w]+$", "iu"], + ["^[\\W]+$", ""], + ["^[\\W]+$", "i"], + ["^[\\W]+$", "iu"], + ["^[^\\W]+$", ""], + ["^[^\\W]+$", "i"], + ["^[^\\W]+$", "iu"], + ["^[a\\w]+$", "i"], + ["^[^a\\w]+$", "i"], + ["^[a\\W]+$", "i"], + ["^[^a\\W]+$", "i"], + ["^[\\b]$", ""], + ["^[\\b]$", "u"], + ["^[\\B]$", ""], + ["^[\\B]$", "i"], + ["^[\\B]$", "u"], // SyntaxError in both engines + + // Dot without s excludes all four ECMAScript LineTerminators. + ["^.$", ""], + ["^.$", "i"], + ["^.$", "u"], + ["^.$", "m"], + ["^.$", "g"], + ["^.$", "s"], + ["^.$", "is"], + ["^.$", "gimsu"], + [".{2}", ""], + [".{2}", "g"], + [".{2}", "s"], + [".{2}", "gs"], + + // #9216: keep the cheap dotAll/empty-intersection translations intact. + ["[^]", ""], + ["[^]", "i"], + ["[^]", "u"], + ["[^]", "gimsu"], + ["[]", ""], + ["[]", "i"], + ["[]", "u"], + ["[]", "gimsu"], + ["[\\s\\S]", "i"], + ["[\\w\\W]", "i"], + ["[^\\w\\W]", "i"], +]; + +function readCorpus(path) { + const lines = readFileSync(path, "utf8").split(/\r?\n/u); + const cases = []; + for (const line of lines) { + if (line.length === 0 || line.startsWith("#")) continue; + const tab = line.indexOf("\t"); + cases.push(tab === -1 ? [line, ""] : [line.slice(0, tab), line.slice(tab + 1)]); + } + return cases; +} + +const cases = corpusPath ? readCorpus(resolve(corpusPath)) : focusedCases; +if (cases.length === 0) fail("corpus is empty"); + +const temp = mkdtempSync(join(tmpdir(), "perry-regex-diff-")); +const sourcePath = join(temp, "probe.ts"); +const nativePath = join(temp, "probe"); +const source = ` +const cases: string[][] = ${JSON.stringify(cases)}; +const subjects: string[] = ${JSON.stringify(subjects)}; + +for (const entry of cases) { + const pattern = entry[0]; + const flags = entry[1]; + const answers: string[] = []; + for (const subject of subjects) { + try { + const re = new RegExp(pattern, flags); + answers.push(re.test(subject) ? "1" : "0"); + } catch (error) { + answers.push(error instanceof SyntaxError ? "S" : "E"); + } + } + console.log(JSON.stringify([pattern, flags, answers.join("")])); +} +`; +writeFileSync(sourcePath, source); + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: repoRoot, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + ...options, + }); + if (result.error) fail(`${command}: ${result.error.message}`); + if (result.status !== 0) { + fail( + `${command} exited ${result.status}\n${result.stdout ?? ""}${result.stderr ?? ""}`, + ); + } + return result.stdout; +} + +try { + const nodeOutput = run(process.execPath, ["--experimental-strip-types", sourcePath]); + run(perryPath, ["compile", sourcePath, "-o", nativePath], { + env: { + ...process.env, + PERRY_NO_CACHE: "1", + PERRY_RUNTIME_DIR: join(repoRoot, "target", "release"), + }, + }); + const perryOutput = run(nativePath, []); + + const nodeLines = nodeOutput.trimEnd().split("\n"); + const perryLines = perryOutput.trimEnd().split("\n"); + const divergent = []; + const subjectDivergences = new Array(subjects.length).fill(0); + const count = Math.max(nodeLines.length, perryLines.length); + for (let i = 0; i < count; i += 1) { + if (nodeLines[i] === perryLines[i]) continue; + divergent.push(i); + if (nodeLines[i] !== undefined && perryLines[i] !== undefined) { + const nodeRecord = JSON.parse(nodeLines[i]); + const perryRecord = JSON.parse(perryLines[i]); + const nodeAnswers = nodeRecord[2]; + const perryAnswers = perryRecord[2]; + for (let j = 0; j < subjects.length; j += 1) { + if (nodeAnswers[j] !== perryAnswers[j]) subjectDivergences[j] += 1; + } + } + } + + console.log( + `patterns=${cases.length} subjects=${subjects.length} divergent_records=${divergent.length}`, + ); + console.log(`subject_cells=${subjectDivergences.reduce((a, b) => a + b, 0)}`); + for (let i = 0; i < subjects.length; i += 1) { + console.log(`subject[${i}]=${JSON.stringify(subjects[i])} divergences=${subjectDivergences[i]}`); + } + for (const index of divergent.slice(0, 100)) { + console.log(`DIFF ${index + 1} node=${nodeLines[index]}`); + console.log(`DIFF ${index + 1} perry=${perryLines[index]}`); + } + if (divergent.length > 100) { + console.log(`... ${divergent.length - 100} additional divergent records omitted`); + } + process.exitCode = divergent.length === 0 ? 0 : 1; +} finally { + rmSync(temp, { recursive: true, force: true }); +} + diff --git a/test-files/test_gap_9217_9218_regexp_word_dot.ts b/test-files/test_gap_9217_9218_regexp_word_dot.ts new file mode 100644 index 0000000000..c3cc8472e3 --- /dev/null +++ b/test-files/test_gap_9217_9218_regexp_word_dot.ts @@ -0,0 +1,92 @@ +// Gap test for #9217 and #9218. Perry delegates matching to Rust's regex +// engines, so its JS-to-Rust translation must pin the places where the two +// grammars deliberately differ: +// +// * ECMAScript \w is ASCII [A-Za-z0-9_], as are \W and the word-ness used +// by \b / \B. The one exception is i+u, where Unicode simple case folding +// also admits U+212A KELVIN SIGN and U+017F LATIN SMALL LETTER LONG S. +// * a non-dotAll `.` excludes LF, CR, LINE SEPARATOR, and PARAGRAPH +// SEPARATOR. Rust's default dot excludes only LF. +// +// This file is byte-compared with `node --experimental-strip-types` by the gap +// suite. The empty classes at the end guard #9216's cheap `(?s:.)` / `[a&&b]` +// translations while the nearby dot handling changes. + +function show(label: string, re: RegExp, subject: string) { + const match = re.exec(subject); + console.log(label + ":" + (match === null ? "null" : JSON.stringify([match[0], match.index]))); +} + +// ASCII control cases. +show("word-ascii", /^\w+$/, "Az_09"); +show("nonword-ascii", /^\W+$/, "-!?"); +show("word-class-ascii", /^[\w-]+$/, "Az_09-"); + +// Accented Latin, Greek, CJK, and emoji are non-word even with u or i. +for (const entry of ["café", "Ωμέγα", "漢字", "😀"]) { + show("word-nonascii", /^\w+$/, entry); + show("word-nonascii-u", /^\w+$/u, entry); + show("word-nonascii-i", /^\w+$/i, entry); + show("nonword-nonascii", /^\W+$/, entry); + show("class-word-nonascii", /^[\w-]+$/, entry); + show("class-nonword-nonascii", /^[\W]+$/, entry); +} + +// Negated and mixed classes exercise \w/\W while they are nested in a class. +show("negated-word", /^[^\w]+$/, "Ω"); +show("negated-nonword", /^[^\W]+$/, "A_9"); +show("mixed-word-i-hit", /^[a\w]+$/i, "Z"); +show("mixed-word-i-miss", /^[a\w]+$/i, "Ω"); +show("mixed-negated-word-i", /^[^a\w]+$/i, "Ω"); +show("mixed-nonword-i", /^[a\W]+$/i, "Ω"); +show("mixed-negated-nonword-i", /^[^a\W]+$/i, "Z"); + +// \b and \B use the same ASCII word predicate. Around a lone Greek letter, +// both sides are non-word; next to ASCII x there is a boundary. +show("boundary-greek", /^\bΩ\b$/, "Ω"); +show("nonboundary-greek", /^\BΩ\B$/, "Ω"); +show("boundary-ascii-greek", /x\bΩ/, "xΩ"); +show("nonboundary-ascii-greek", /x\BΩ/, "xΩ"); + +// In a class, \b is BACKSPACE. Sloppy-mode \B is the identity escape `B`; +// the /u form is a SyntaxError and is covered by the differential runner. +show("class-backspace", /^[\b]$/, "\b"); +show("class-backspace-not-b", /^[\b]$/, "b"); +show("class-identity-B", new RegExp("^[\\B]$"), "B"); +show("class-identity-B-i", new RegExp("^[\\B]$", "i"), "b"); + +// Kelvin and long-s join the word set only when BOTH i and u are present. +for (const entry of ["K", "ſ"]) { + show("fold-word-plain", /^\w$/, entry); + show("fold-word-i", /^\w$/i, entry); + show("fold-word-u", /^\w$/u, entry); + show("fold-word-iu", /^\w$/iu, entry); + show("fold-nonword-iu", /^\W$/iu, entry); + show("fold-boundary-iu", /^\b.\b$/iu, entry); + show("fold-nonboundary-iu", /^\B.\B$/iu, entry); + show("fold-class-word-iu", /^[\w]$/iu, entry); + show("fold-class-nonword-iu", /^[\W]$/iu, entry); +} + +// Non-dotAll dot excludes every LineTerminator, regardless of i/u/m/g. +const terminators = ["\n", "\r", "
", "
"]; +for (const entry of terminators) { + show("dot", /^.$/, entry); + show("dot-i", /^.$/i, entry); + show("dot-u", /^.$/u, entry); + show("dot-m", /^.$/m, entry); + show("dot-g", /^.$/g, entry); + show("dot-s", /^.$/s, entry); + show("dot-isu", /^.$/isu, entry); +} +show("dot-tab", /^.$/, "\t"); +show("dot-crlf-repro", /.{2}/g, "\t\r\n"); +show("dot-crlf-dotall", /.{2}/gs, "\t\r\n"); + +// #9216 controls: [^] is any character even without s; [] never matches. +for (const entry of ["x", "\n", "\r", "
", "
", "😀"]) { + show("negated-empty", /[^]/i, entry); + show("empty", /[]/i, entry); +} +show("word-complements-any", /[\w\W]/i, "Ω"); +show("word-complements-empty", /[^\w\W]/i, "A"); From dddac131c71680ec24471955aba3a84fcef4be24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 31 Aug 2026 07:25:42 +0200 Subject: [PATCH 2/4] fix(runtime): align regexp word and dot semantics with ECMAScript --- .../9217-9218-regexp-ascii-word-dot.md | 9 + crates/perry-runtime/src/regex/grammar.rs | 269 +++++++++++++++++- crates/perry-runtime/src/regex/lazy.rs | 4 +- crates/perry-runtime/src/regex/tests.rs | 86 +++++- 4 files changed, 360 insertions(+), 8 deletions(-) create mode 100644 changelog.d/9217-9218-regexp-ascii-word-dot.md diff --git a/changelog.d/9217-9218-regexp-ascii-word-dot.md b/changelog.d/9217-9218-regexp-ascii-word-dot.md new file mode 100644 index 0000000000..319795afc5 --- /dev/null +++ b/changelog.d/9217-9218-regexp-ascii-word-dot.md @@ -0,0 +1,9 @@ +### Fixed + +- **RegExp word escapes/boundaries and non-dotAll `.` now follow ECMAScript + instead of Rust's defaults (#9217, #9218).** `\w`, `\W`, `\b`, and `\B` + use the spec's ASCII `[A-Za-z0-9_]` word set; `i`+`u` additionally admits + U+212A KELVIN SIGN and U+017F LATIN SMALL LETTER LONG S. A `.` without `s` + now excludes all four LineTerminators (`\n`, `\r`, U+2028, U+2029), while + dotAll still matches every character. The cheap #9216 translations for + `[^]` and `[]` remain `(?s:.)` and `[a&&b]`, avoiding full-range case folds. diff --git a/crates/perry-runtime/src/regex/grammar.rs b/crates/perry-runtime/src/regex/grammar.rs index f8de27dbea..fa900e6bbd 100644 --- a/crates/perry-runtime/src/regex/grammar.rs +++ b/crates/perry-runtime/src/regex/grammar.rs @@ -379,6 +379,13 @@ pub(super) fn has_unicode_forbidden_pattern(pattern: &str) -> bool { // JS \s excludes U+0085 NEL; Rust's \s includes it. Use explicit class for parity outside char-class. const JS_WHITESPACE_CLASS: &str = r"[\t\n\x0B\x0C\r\x20\x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}]"; const JS_NON_WHITESPACE_CLASS: &str = r"[^\t\n\x0B\x0C\r\x20\x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}]"; +// ECMA-262 CharacterClassEscape: unlike Rust's Unicode-aware `\w`, the JS +// word set starts as exactly these ASCII characters. Under BOTH `i` and +// Unicode mode, Canonicalize adds the two non-ASCII code points whose simple +// case fold lands in that set (LONG S -> s, KELVIN SIGN -> k). +const JS_ASCII_WORD_MEMBERS: &str = r"A-Za-z0-9_"; +const JS_UNICODE_IGNORE_CASE_WORD_MEMBERS: &str = r"A-Za-z0-9_\x{017F}\x{212A}"; +const JS_NON_DOTALL_DOT: &str = r"[^\n\r\x{2028}\x{2029}]"; // ECMAScript allows quantifiers up to 2^53-1; regex-syntax uses u32 and rejects larger values. const MAX_QUANTIFIER: u64 = 65_535; @@ -1272,6 +1279,24 @@ fn next_is_class_shorthand(chars: &[char], i: usize) -> bool { ) } +/// Did the source member immediately before `chars[i]` spell `\w` or `\W`? +/// Those escapes are expanded to explicit members before a following hyphen is +/// processed, so `out_ends_with_class_shorthand` can no longer recognize them. +fn previous_is_word_shorthand(chars: &[char], i: usize) -> bool { + if i < 2 || chars.get(i - 2) != Some(&'\\') || !matches!(chars.get(i - 1), Some('w' | 'W')) { + return false; + } + // `\\w-` is a literal backslash, `w`, hyphen — the candidate slash is + // itself escaped and therefore was not a shorthand. + let mut preceding_backslashes = 0usize; + let mut k = i - 2; + while k > 0 && chars[k - 1] == '\\' { + preceding_backslashes += 1; + k -= 1; + } + preceding_backslashes % 2 == 0 +} + /// Rewrites quantified lookaround groups (`(?=…)?`, `(?!…)*`, `(?:(?=…))?`, etc.) /// into a form `fancy-regex` accepts. Lower-bound-0 → drop the assertion; ≥1 → /// keep the assertion, drop the quantifier. JS/V8 allow these; `fancy-regex` @@ -1517,13 +1542,176 @@ fn any_char_class_width(chars: &[char], i: usize) -> Option { } } +fn js_word_members(unicode_ignore_case: bool) -> &'static str { + if unicode_ignore_case { + JS_UNICODE_IGNORE_CASE_WORD_MEMBERS + } else { + JS_ASCII_WORD_MEMBERS + } +} + +/// A one-code-point atom for ECMAScript's `\w` (or `\W` when `negated`). +/// +/// The local `-i` is essential even when the outer pattern is `(?i)`: Rust's +/// case-folding of `[A-Za-z0-9_]` adds LONG S and KELVIN SIGN in non-Unicode +/// JS mode too. ECMAScript adds those only for the `i`+`u` combination, so the +/// exact post-Canonicalize set is written explicitly and then protected from +/// another fold. +fn js_word_atom(negated: bool, unicode_ignore_case: bool) -> String { + format!( + "(?-i:[{}{}])", + if negated { "^" } else { "" }, + js_word_members(unicode_ignore_case) + ) +} + +/// Emit the `i`+`u` word-boundary assertion. Rust has an ASCII boundary mode +/// for the ordinary case (`(?-iu:\b)`), but no boundary predicate for the +/// exact ASCII-plus-LONG-S-plus-KELVIN set required by ECMAScript. Four +/// one-code-point lookarounds spell the transition (or non-transition) +/// directly; fancy-regex handles this rare form. +fn push_unicode_ignore_case_word_boundary(result: &mut String, non_boundary: bool) { + let word = js_word_atom(false, true); + if non_boundary { + result.push_str(&format!("(?:(?<={word})(?={word})|(? Option<(String, usize)> { + if chars.get(open) != Some(&'[') { + return None; + } + let negated = chars.get(open + 1) == Some(&'^'); + let members_start = open + if negated { 2 } else { 1 }; + + let mut close = members_start; + while close < chars.len() { + if chars[close] == '\\' { + close += 2; + continue; + } + if chars[close] == ']' { + break; + } + close += 1; + } + if chars.get(close) != Some(&']') { + return None; + } + + let mut rest_members = String::new(); + let mut rest_member_count = 0usize; + let mut has_word = false; + let mut has_non_word = false; + let mut previous_was_word_escape = false; + let mut i = members_start; + while i < close { + if chars[i] == '\\' && i + 1 < close && matches!(chars[i + 1], 'w' | 'W') { + has_word |= chars[i + 1] == 'w'; + has_non_word |= chars[i + 1] == 'W'; + previous_was_word_escape = true; + i += 2; + continue; + } + if chars[i] == '\\' && i + 1 < close { + rest_members.push(chars[i]); + rest_members.push(chars[i + 1]); + rest_member_count += 1; + previous_was_word_escape = false; + i += 2; + continue; + } + if chars[i] == '-' { + let next_is_word_escape = + i + 2 < close && chars[i + 1] == '\\' && matches!(chars[i + 2], 'w' | 'W'); + if previous_was_word_escape || next_is_word_escape { + rest_members.push_str("\\-"); + } else { + rest_members.push('-'); + } + } else { + rest_members.push(chars[i]); + } + rest_member_count += 1; + previous_was_word_escape = false; + i += 1; + } + + if !(has_word || has_non_word) { + return None; + } + let width = close + 1 - open; + + // A shorthand and its complement cover every scalar; preserve #9216's + // cheap dotAll / empty-intersection spellings and never build a full-range + // class under `i`. + if has_word && has_non_word { + return Some(( + if negated { + "[a&&b]".to_string() + } else { + "(?s:.)".to_string() + }, + width, + )); + } + + if rest_member_count == 0 { + let atom = if has_word { + js_word_atom(negated, false) + } else { + js_word_atom(!negated, false) + }; + return Some((atom, width)); + } + + let rest_source = format!("[{rest_members}]"); + let mut arms = vec![js_regex_to_rust_with_flags(&rest_source, flags)]; + if has_word { + arms.push(js_word_atom(false, false)); + } + if has_non_word { + arms.push(js_word_atom(true, false)); + } + let union = arms.join("|"); + let rewritten = if negated { + format!("(?!(?:{union}))(?s:.)") + } else { + format!("(?:{union})") + }; + Some((rewritten, width)) +} + pub(super) fn js_regex_to_rust(pattern: &str) -> String { + js_regex_to_rust_with_flags(pattern, "") +} + +pub(super) fn js_regex_to_rust_with_flags(pattern: &str, flags: &str) -> String { let folded = fold_surrogate_pairs(pattern); let folded = normalize_quantified_lookaround(&folded); let folded = clamp_large_quantifiers(&folded); let mut result = String::with_capacity(folded.len()); let chars: Vec = folded.chars().collect(); let capture_spans = collect_capture_spans(&chars); + let case_insensitive = flags.contains('i'); + let unicode = flags.contains('u') || flags.contains('v'); + let unicode_ignore_case = case_insensitive && unicode; + let dot_all = flags.contains('s'); let mut i = 0; let mut in_class = false; // track `[...]` position; JS and Rust disagree on bare `[` inside while i < chars.len() { @@ -1653,6 +1841,48 @@ pub(super) fn js_regex_to_rust(pattern: &str) -> String { } } } + // ECMAScript word escapes are ASCII, unlike Rust's Unicode + // `\w`/`\W`. Inside a class the `i`+non-Unicode case is + // rewritten as a whole at the opening `[` below, because a + // scoped `(?-i:...)` group cannot occur inside a class. Every + // other flag combination can safely use explicit members. + 'w' if in_class => { + result.push_str(js_word_members(unicode_ignore_case)); + i += 2; + } + 'W' if in_class => { + result.push_str("[^"); + result.push_str(js_word_members(unicode_ignore_case)); + result.push(']'); + i += 2; + } + 'w' => { + result.push_str(&js_word_atom(false, unicode_ignore_case)); + i += 2; + } + 'W' => { + result.push_str(&js_word_atom(true, unicode_ignore_case)); + i += 2; + } + // Word boundaries use the same IsWordChar predicate as `\w`. + // Rust's `(?-iu:\b)` is the exact ASCII form. Only `i`+`u` + // needs the explicit augmented-set lookarounds. + 'b' if !in_class => { + if unicode_ignore_case { + push_unicode_ignore_case_word_boundary(&mut result, false); + } else { + result.push_str("(?-iu:\\b)"); + } + i += 2; + } + 'B' if !in_class => { + if unicode_ignore_case { + push_unicode_ignore_case_word_boundary(&mut result, true); + } else { + result.push_str("(?-iu:\\B)"); + } + i += 2; + } // `\s`/`\S` outside class: JS excludes NEL (U+0085), Rust includes it. 's' if !in_class => { result.push_str(JS_WHITESPACE_CLASS); @@ -1757,6 +1987,17 @@ pub(super) fn js_regex_to_rust(pattern: &str) -> String { // same reason. push_any_char(&mut result); i += width; + } else if case_insensitive && !unicode { + if let Some((rewritten, width)) = + rewrite_case_insensitive_ascii_word_class(&chars, i, flags) + { + result.push_str(&rewritten); + i += width; + } else { + in_class = true; + result.push('['); + i += 1; + } } else { in_class = true; result.push('['); @@ -1768,6 +2009,13 @@ pub(super) fn js_regex_to_rust(pattern: &str) -> String { in_class = false; result.push(']'); i += 1; + } else if !in_class && chars[i] == '.' { + if dot_all { + result.push('.'); + } else { + result.push_str(JS_NON_DOTALL_DOT); + } + i += 1; } else if !in_class && chars[i] == '(' && i + 2 < chars.len() && chars[i + 1] == '?' { // Check for JS named group (?...) — convert to (?P...) // But NOT (?<=...) (lookbehind) or (? String { } } else if in_class && chars[i] == '-' - && (out_ends_with_class_shorthand(&result) || next_is_class_shorthand(&chars, i + 1)) + && (out_ends_with_class_shorthand(&result) + || previous_is_word_shorthand(&chars, i) + || next_is_class_shorthand(&chars, i + 1)) { // Inside a class, a `-` adjacent to a shorthand class (`\d`, `\w`, // `\s`, …, or a `\p{…}` property) is a *literal* hyphen in JS — a @@ -2178,9 +2428,9 @@ mod tests { // would otherwise reject `\w-` as `ClassRangeLiteral`. The hyphen must // be escaped to `\-`. for (src, expect) in [ - (r"[\w-\.]", r"[\w\-\.]"), + (r"[\w-\.]", r"[A-Za-z0-9_\-\.]"), (r"[\d-z]", r"[\d\-z]"), - (r"[a\w-]", r"[a\w\-]"), + (r"[a\w-]", r"[aA-Za-z0-9_\-]"), (r"[a-\d]", r"[a\-\d]"), (r"[\p{Greek}-x]", r"[\p{Greek}\-x]"), ] { @@ -2193,7 +2443,7 @@ mod tests { // An ordinary `a-z` range between two single literals is untouched. assert_eq!(js_regex_to_rust("[a-z]"), "[a-z]"); // Outside a class, `-` is never escaped. - assert_eq!(js_regex_to_rust(r"\d-\w"), r"\d-\w"); + assert_eq!(js_regex_to_rust(r"\d-\w"), r"\d-(?-i:[A-Za-z0-9_])"); // A `\w-\.` member must match `\w`, a literal `-`, and `.`. let re = regex::Regex::new(&js_regex_to_rust(r"^[\w-\.]+$")).unwrap(); assert!(re.is_match("a-b.c_d")); @@ -2203,7 +2453,7 @@ mod tests { #[cfg(test)] mod any_char_rewrite_tests { - use super::js_regex_to_rust; + use super::{js_regex_to_rust, js_regex_to_rust_with_flags}; /// The rewrite fires on exactly the closed complementary pairs, and on /// JS's `[^]`. @@ -2256,4 +2506,13 @@ mod any_char_rewrite_tests { ); } } + + #[test] + fn empty_and_any_classes_keep_the_non_folding_rewrite_with_flags() { + for flags in ["", "i", "u", "iu", "s", "gimsu"] { + assert_eq!(js_regex_to_rust_with_flags("[^]", flags), "(?s:.)"); + assert_eq!(js_regex_to_rust_with_flags("[]", flags), "[a&&b]"); + assert_eq!(js_regex_to_rust_with_flags(r"[\w\W]", flags), "(?s:.)"); + } + } } diff --git a/crates/perry-runtime/src/regex/lazy.rs b/crates/perry-runtime/src/regex/lazy.rs index 027e62cef3..a952755781 100644 --- a/crates/perry-runtime/src/regex/lazy.rs +++ b/crates/perry-runtime/src/regex/lazy.rs @@ -50,7 +50,7 @@ use std::sync::Arc; use regex::Regex; -use super::grammar::{collapse_redos_guard_quantifiers, js_regex_to_rust}; +use super::grammar::{collapse_redos_guard_quantifiers, js_regex_to_rust_with_flags}; use super::{ evict_regex_cache_if_full, get_or_compile_regex, is_valid_ptr, is_valid_regex_ptr, string_as_str, RegExpHeader, FANCY_CACHE, REGEX_SOURCE_TABLE, REPEAT_MATCHER_CACHE, @@ -64,7 +64,7 @@ use super::{ /// validator that inspects a DIFFERENT string than the builder would either /// throw on a pattern that compiles or accept one that does not. pub(super) fn flag_prefixed_pattern(pattern: &str, flags: &str) -> String { - let translated = js_regex_to_rust(pattern); + let translated = js_regex_to_rust_with_flags(pattern, flags); let case_insensitive = flags.contains('i'); let multiline = flags.contains('m'); // #2828: the `s` (dotAll) flag maps directly onto the Rust `regex` diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 4ba9ed2454..232ff54fef 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -462,13 +462,97 @@ fn escaped_hyphen_in_class_stays_literal() { } } +#[test] +fn ecmascript_word_escapes_and_boundaries_use_the_spec_word_set() { + fn matches(pattern: &str, flags: &str, subject: &str) -> bool { + let re = js_regexp_new(make_string(pattern), make_string(flags)); + js_regexp_test(re, make_string(subject)) != 0 + } + + // Neither `u` nor `i` alone widens the ASCII set. Rust's native `\w` + // admits all of these, which was the silent wrong-answer bug. + for flags in ["", "i", "u"] { + for subject in ["é", "Ω", "漢", "K", "ſ"] { + assert!( + !matches(r"^\w$", flags, subject), + "/^\\w$/{flags} {subject}" + ); + assert!(matches(r"^\W$", flags, subject), "/^\\W$/{flags} {subject}"); + assert!( + !matches(r"^[\w]$", flags, subject), + "/^[\\w]$/{flags} {subject}" + ); + assert!( + matches(r"^[\W]$", flags, subject), + "/^[\\W]$/{flags} {subject}" + ); + } + } + + // `i`+`u` adds exactly the two non-ASCII simple folds into ASCII. + for subject in ["K", "ſ"] { + assert!(matches(r"^\w$", "iu", subject)); + assert!(!matches(r"^\W$", "iu", subject)); + assert!(matches(r"^[\w]$", "iu", subject)); + assert!(!matches(r"^[\W]$", "iu", subject)); + assert!(matches(r"^\b.\b$", "iu", subject)); + assert!(!matches(r"^\B.\B$", "iu", subject)); + } + for subject in ["é", "Ω", "漢"] { + assert!(!matches(r"^\w$", "iu", subject)); + assert!(matches(r"^\W$", "iu", subject)); + assert!(!matches(r"^\b.\b$", "iu", subject)); + assert!(matches(r"^\B.\B$", "iu", subject)); + } + + // Mixed classes under non-Unicode `i` need separate exact-word and + // normally-folded arms; the outer Rust `(?i)` must not fold the word arm. + assert!(!matches(r"^[a\w]+$", "i", "Ω")); + assert!(matches(r"^[^a\w]+$", "i", "Ω")); + assert!(matches(r"^[a\W]+$", "i", "Ω")); + assert!(!matches(r"^[^a\W]+$", "i", "Ω")); + + // Boundary word-ness is the same predicate as `\w`. + assert!(!matches(r"^\bΩ\b$", "", "Ω")); + assert!(matches(r"^\BΩ\B$", "", "Ω")); + assert!(matches(r"x\bΩ", "", "xΩ")); + assert!(!matches(r"x\BΩ", "", "xΩ")); +} + +#[test] +fn ecmascript_dot_excludes_all_line_terminators_without_dotall() { + fn matches(pattern: &str, flags: &str, subject: &str) -> bool { + let re = js_regexp_new(make_string(pattern), make_string(flags)); + js_regexp_test(re, make_string(subject)) != 0 + } + + for flags in ["", "i", "u", "m", "g"] { + for terminator in ["\n", "\r", "\u{2028}", "\u{2029}"] { + assert!( + !matches(r"^.$", flags, terminator), + "/^.$/{flags} matched {terminator:?}" + ); + } + } + for terminator in ["\n", "\r", "\u{2028}", "\u{2029}"] { + assert!(matches(r"^.$", "s", terminator)); + assert!(matches(r"^.$", "isu", terminator)); + } + assert!(matches(r"^.$", "", "\t")); + assert!(!matches(r".{2}", "g", "\t\r\n")); + assert!(matches(r".{2}", "gs", "\t\r\n")); +} + #[test] fn annexb_legacy_decimal_escapes() { // #5594: a `\` with no matching capture group is an Annex B.1.4 // legacy octal escape, not a backreference — `\1` → `\x01`, never the // bare `\1` the `regex`/`fancy-regex` crates reject. assert_eq!(js_regex_to_rust(r"\1"), r"\x{01}"); - assert_eq!(js_regex_to_rust(r"\b(\w+) \2\b"), r"\b(\w+) \x{02}\b"); + assert_eq!( + js_regex_to_rust(r"\b(\w+) \2\b"), + r"(?-iu:\b)((?-i:[A-Za-z0-9_])+) \x{02}(?-iu:\b)" + ); // Multi-digit octal: `\12` = 0o12 = 0x0A, `\14` = 0o14 = 0x0C. assert_eq!(js_regex_to_rust(r"[\12-\14]"), r"[\x{0A}-\x{0C}]"); // Inside a class a decimal escape is always octal, never a backref — From 5b02d49683b98fffe3b5e1b02fb164fda486d4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 31 Aug 2026 08:07:00 +0200 Subject: [PATCH 3/4] test(runtime): keep rewritten negated regex classes atomic --- crates/perry-runtime/src/regex/grammar.rs | 6 +++++- crates/perry-runtime/src/regex/tests.rs | 2 ++ test-files/test_gap_9217_9218_regexp_word_dot.ts | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/regex/grammar.rs b/crates/perry-runtime/src/regex/grammar.rs index fa900e6bbd..03b34f6445 100644 --- a/crates/perry-runtime/src/regex/grammar.rs +++ b/crates/perry-runtime/src/regex/grammar.rs @@ -1690,7 +1690,11 @@ fn rewrite_case_insensitive_ascii_word_class( } let union = arms.join("|"); let rewritten = if negated { - format!("(?!(?:{union}))(?s:.)") + // Keep the assertion and its consuming dot one atom. Otherwise a + // following quantifier in the JS source (`[^a\W]+`) would bind only + // to the dot, checking the exclusion at the first character but then + // silently admitting forbidden characters in later repetitions. + format!("(?:(?!(?:{union}))(?s:.))") } else { format!("(?:{union})") }; diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 232ff54fef..85342833e2 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -511,6 +511,8 @@ fn ecmascript_word_escapes_and_boundaries_use_the_spec_word_set() { assert!(matches(r"^[^a\w]+$", "i", "Ω")); assert!(matches(r"^[a\W]+$", "i", "Ω")); assert!(!matches(r"^[^a\W]+$", "i", "Ω")); + assert!(matches(r"^[^a\W]+$", "i", "cfx")); + assert!(!matches(r"^[^a\W]+$", "i", "café")); // Boundary word-ness is the same predicate as `\w`. assert!(!matches(r"^\bΩ\b$", "", "Ω")); diff --git a/test-files/test_gap_9217_9218_regexp_word_dot.ts b/test-files/test_gap_9217_9218_regexp_word_dot.ts index c3cc8472e3..c98da173ae 100644 --- a/test-files/test_gap_9217_9218_regexp_word_dot.ts +++ b/test-files/test_gap_9217_9218_regexp_word_dot.ts @@ -40,6 +40,7 @@ show("mixed-word-i-miss", /^[a\w]+$/i, "Ω"); show("mixed-negated-word-i", /^[^a\w]+$/i, "Ω"); show("mixed-nonword-i", /^[a\W]+$/i, "Ω"); show("mixed-negated-nonword-i", /^[^a\W]+$/i, "Z"); +show("mixed-negated-nonword-i-sequence", /^[^a\W]+$/i, "café"); // \b and \B use the same ASCII word predicate. Around a lone Greek letter, // both sides are non-word; next to ASCII x there is a boundary. From cb239609a169446cc10dbb49031a2f11d7a45493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 31 Aug 2026 08:27:00 +0200 Subject: [PATCH 4/4] test(runtime): isolate regexp gap fixture from UTF-16 gap --- test-files/test_gap_9217_9218_regexp_word_dot.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-files/test_gap_9217_9218_regexp_word_dot.ts b/test-files/test_gap_9217_9218_regexp_word_dot.ts index c98da173ae..5dfa2a31e1 100644 --- a/test-files/test_gap_9217_9218_regexp_word_dot.ts +++ b/test-files/test_gap_9217_9218_regexp_word_dot.ts @@ -85,9 +85,13 @@ show("dot-crlf-repro", /.{2}/g, "\t\r\n"); show("dot-crlf-dotall", /.{2}/gs, "\t\r\n"); // #9216 controls: [^] is any character even without s; [] never matches. -for (const entry of ["x", "\n", "\r", "
", "
", "😀"]) { +for (const entry of ["x", "\n", "\r", "
", "
"]) { show("negated-empty", /[^]/i, entry); show("empty", /[]/i, entry); } +// Use `u` for the astral control so this fixture does not conflate #9216 with +// Perry's separate, pre-existing non-u UTF-16 code-unit matching gap. +show("negated-empty-u", /[^]/iu, "😀"); +show("empty-u", /[]/iu, "😀"); show("word-complements-any", /[\w\W]/i, "Ω"); show("word-complements-empty", /[^\w\W]/i, "A");