diff --git a/changelog.d/9408-multiline-anchors-lineterminators.md b/changelog.d/9408-multiline-anchors-lineterminators.md new file mode 100644 index 0000000000..f03ee0adc3 --- /dev/null +++ b/changelog.d/9408-multiline-anchors-lineterminators.md @@ -0,0 +1,14 @@ +### Fixed + +- **`^` and `$` under the `m` flag now hold at every LineTerminator, not just + LF (#9408).** ECMAScript Β§22.2.2.6 defines the multiline anchors over the + same four characters a non-dotAll `.` excludes β€” `\n`, `\r`, U+2028 and + U+2029 β€” but the translation leaned on Rust's `(?m)`, which recognizes LF + alone. `"one\rtwo".match(/^.*$/gm)` returned `null` instead of + `["one","two"]`, and CRLF (which is TWO terminators, with an empty line + between them) reported `["two"]` instead of `["one","","two"]`, so any CRLF + markdown, git output from a Windows checkout, or `/etc/os-release` parse + silently mis-matched. The anchors are now spelled out against the same + LineTerminator set #9218 gave `.`, sharing one definition so the two cannot + drift; a multiline pattern with an anchor consequently compiles on + `fancy-regex` rather than the linear engine. diff --git a/changelog.d/9409-split-empty-code-units.md b/changelog.d/9409-split-empty-code-units.md new file mode 100644 index 0000000000..09f81a3ef1 --- /dev/null +++ b/changelog.d/9409-split-empty-code-units.md @@ -0,0 +1,13 @@ +### Fixed + +- **`split("")` splits into UTF-16 code units, so an astral character yields + two parts (#9409).** Β§22.1.3.23 runs SplitMatch over the code-unit sequence, + making `"πŸ˜€".split("")` a two-element array of lone surrogates β€” matching + `"πŸ˜€".length === 2` and the halves `charAt(0)`/`charAt(1)` already returned. + Perry stepped its WTF-8 payload one sequence at a time, so an astral + character came back as a single part and every emoji-width, truncation and + column calculation built on `split("")` saw one unit where Node sees two. + Each half is now built with the same one-code-unit constructor `charAt` uses, + keeping the `HAS_LONE_SURROGATES` flag so `isWellFormed()` and + `JSON.stringify` still see a broken half; `limit` counts code units and may + legitimately cut a pair. diff --git a/crates/perry-runtime/src/regex/grammar.rs b/crates/perry-runtime/src/regex/grammar.rs index 03b34f6445..55287d4491 100644 --- a/crates/perry-runtime/src/regex/grammar.rs +++ b/crates/perry-runtime/src/regex/grammar.rs @@ -385,7 +385,33 @@ const JS_NON_WHITESPACE_CLASS: &str = r"[^\t\n\x0B\x0C\r\x20\x{A0}\x{1680}\x{200 // case fold lands in that set (LONG S -> s, KELVIN SIGN -> k). const JS_ASCII_WORD_MEMBERS: &str = r"A-Za-z0-9_"; const JS_UNICODE_IGNORE_CASE_WORD_MEMBERS: &str = r"A-Za-z0-9_\x{017F}\x{212A}"; -const JS_NON_DOTALL_DOT: &str = r"[^\n\r\x{2028}\x{2029}]"; +/// The four ECMAScript LineTerminators (Β§12.3), as character-class members. +/// +/// One spelling, three consumers: a non-dotAll `.` excludes exactly this set +/// (#9218), and Β§22.2.2.6 defines the multiline `^`/`$` assertions over exactly +/// the same characters (#9408). A macro rather than a `const` so `concat!` can +/// build the three patterns from it and they cannot drift. +macro_rules! js_line_terminator_members { + () => { + r"\n\r\x{2028}\x{2029}" + }; +} +const JS_NON_DOTALL_DOT: &str = concat!("[^", js_line_terminator_members!(), "]"); +/// `^` under the `m` flag: the start of the input, or any position immediately +/// after a LineTerminator. +/// +/// Rust's `(?m)` mode β€” what the flag prefix used to be translated to on its +/// own β€” recognizes LF and nothing else, so `"one\rtwo"` looked like a single +/// line and `"one\r\ntwo"` like two instead of three (CRLF is TWO terminators, +/// with an empty line between them). Neither engine has a configurable line +/// terminator SET (`regex`'s `line_terminator` is a single byte), so the +/// assertion is spelled out. The lookbehind costs the linear engine: a pattern +/// using it falls back to `fancy-regex`, which is why only `m` patterns are +/// rewritten and `\A`/`\z` are used rather than leaning on the `(?m)` prefix. +const JS_MULTILINE_LINE_START: &str = concat!(r"(?:\A|(?<=[", js_line_terminator_members!(), "]))"); +/// `$` under the `m` flag: the end of the input, or any position immediately +/// before a LineTerminator. Mirror of [`JS_MULTILINE_LINE_START`]. +const JS_MULTILINE_LINE_END: &str = concat!(r"(?:\z|(?=[", js_line_terminator_members!(), "]))"); // ECMAScript allows quantifiers up to 2^53-1; regex-syntax uses u32 and rejects larger values. const MAX_QUANTIFIER: u64 = 65_535; @@ -1716,6 +1742,7 @@ pub(super) fn js_regex_to_rust_with_flags(pattern: &str, flags: &str) -> String let unicode = flags.contains('u') || flags.contains('v'); let unicode_ignore_case = case_insensitive && unicode; let dot_all = flags.contains('s'); + let multiline = flags.contains('m'); let mut i = 0; let mut in_class = false; // track `[...]` position; JS and Rust disagree on bare `[` inside while i < chars.len() { @@ -2020,6 +2047,19 @@ pub(super) fn js_regex_to_rust_with_flags(pattern: &str, flags: &str) -> String result.push_str(JS_NON_DOTALL_DOT); } i += 1; + } else if multiline && !in_class && (chars[i] == '^' || chars[i] == '$') { + // Β§22.2.2.6 Assertion: under `m`, `^`/`$` hold at every + // LineTerminator, not just LF. Only an UNESCAPED anchor outside a + // character class reaches here β€” `\^` / `\$` are consumed by the + // backslash arm above, and inside `[...]` both are ordinary + // members (`in_class` is what the `[`/`]` arms maintain for exactly + // this kind of question). + result.push_str(if chars[i] == '^' { + JS_MULTILINE_LINE_START + } else { + JS_MULTILINE_LINE_END + }); + i += 1; } else if !in_class && chars[i] == '(' && i + 2 < chars.len() && chars[i + 1] == '?' { // Check for JS named group (?...) β€” convert to (?P...) // But NOT (?<=...) (lookbehind) or (? Vec { + re.find_iter(subject) + .map(|m| m.unwrap().as_str().to_string()) + .collect() + }; + assert_eq!(lines("one\ntwo"), ["one", "two"]); + assert_eq!(lines("one\rtwo"), ["one", "two"]); + assert_eq!(lines("one\u{2028}two"), ["one", "two"]); + assert_eq!(lines("one\u{2029}two"), ["one", "two"]); + // CRLF: `$` before the CR, `^` after it, `$` before the LF β€” an empty + // match sits between the pair. + assert_eq!(lines("one\r\ntwo"), ["one", "", "two"]); + assert_eq!(lines("onetwo"), ["onetwo"]); + } + + /// Insertion points, i.e. the `replace(/^/gm, ">")` shape: the anchors are + /// zero-width, so the count and the offsets are the whole answer. + #[test] + fn anchors_are_zero_width_at_every_terminator() { + let starts = build_fancy_regex(&flag_prefixed_pattern("^", "gm")).unwrap(); + let ends = build_fancy_regex(&flag_prefixed_pattern("$", "gm")).unwrap(); + let offsets = |re: &fancy_regex::Regex, subject: &str| -> Vec { + re.find_iter(subject).map(|m| m.unwrap().start()).collect() + }; + assert_eq!(offsets(&starts, "one\r\ntwo"), [0, 4, 5]); + assert_eq!(offsets(&ends, "one\r\ntwo"), [3, 4, 8]); + assert_eq!(offsets(&starts, "\rone"), [0, 1]); + assert_eq!(offsets(&ends, "one\r"), [3, 4]); + } +} diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index c78ba366b5..df6b0d86a5 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -142,6 +142,10 @@ pub use char_ops::{ js_string_from_code_point, js_string_from_code_point_array, js_string_index_get, js_string_index_get_boxed, js_string_index_to_i32, js_string_to_char_array, }; +// The one-UTF-16-code-unit string builder `charAt` uses. `split("")` needs the +// same constructor: both cut a string at code-unit boundaries, so both have to +// be able to produce a lone surrogate (#9409). +pub(crate) use char_ops::string_from_code_unit; pub use compare::{ js_string_compare, js_string_ends_with, js_string_ends_with_at, js_string_equals, js_string_is_well_formed, js_string_locale_compare, js_string_locale_compare_opts, diff --git a/crates/perry-runtime/src/string/split.rs b/crates/perry-runtime/src/string/split.rs index 28d9910934..007879f216 100644 --- a/crates/perry-runtime/src/string/split.rs +++ b/crates/perry-runtime/src/string/split.rs @@ -133,6 +133,55 @@ fn split_part_byte_range(source: &[u8], delimiter: &[u8], target: usize) -> Opti (part_index == target).then_some((part_start, source.len())) } +/// One element of an EMPTY-delimiter split, located by UTF-16 code-unit index +/// (#9409). `split("")` cuts at code-unit boundaries, so an astral character +/// covers two indices and neither of them is a byte range in the source. +enum EmptyDelimiterPart { + /// A byte range to copy verbatim: a BMP character, a WTF-8 lone surrogate + /// already in the payload, or a malformed sequence. + Bytes { start: usize, end: usize }, + /// One synthesized half of an astral character. + Surrogate(u16), +} + +/// The `index`-th part of `source.split("")`, plus the part's own UTF-16 +/// length. `None` once `index` is past the end. +/// +/// Shared by the two scalar-replacement fast paths so `split("")[k]` and +/// `split("")[k].length` cannot disagree with the array `js_string_split_n` +/// builds. A malformed lead byte reports zero units and still occupies one +/// index, exactly as the array walk treats it. +fn empty_delimiter_part(source: &[u8], index: usize) -> Option<(EmptyDelimiterPart, usize)> { + let mut byte_offset = 0usize; + let mut remaining = index; + while byte_offset < source.len() { + let (advance, units, code_point) = crate::string::wtf8_step(source, byte_offset); + let end = (byte_offset + advance).min(source.len()); + let parts = units.max(1); + if remaining < parts { + if units == 2 && code_point >= 0x10000 { + let astral = code_point - 0x10000; + let unit = if remaining == 0 { + 0xD800 + (astral >> 10) as u16 + } else { + 0xDC00 + (astral & 0x3FF) as u16 + }; + return Some((EmptyDelimiterPart::Surrogate(unit), 1)); + } + return Some(( + EmptyDelimiterPart::Bytes { + start: byte_offset, + end, + }, + units, + )); + } + remaining -= parts; + byte_offset = end; + } + None +} + /// Materialize one element of a string-delimiter split as a boxed JS value. /// This is used when codegen proves the result array does not escape and only /// a constant element is observed. A missing element remains `undefined`. @@ -149,30 +198,27 @@ pub extern "C" fn js_string_split_part_value( let delimiter_bytes = unsafe { slice::from_raw_parts(string_data(delimiter), (*delimiter).byte_len as usize) }; if delimiter_bytes.is_empty() { - let mut byte_offset = 0usize; - for _ in 0..index as usize { - if byte_offset >= source.len() { - return f64::from_bits(crate::value::TAG_UNDEFINED); - } - let (advance, _, _) = crate::string::wtf8_step(source, byte_offset); - byte_offset = (byte_offset + advance).min(source.len()); - } - if byte_offset >= source.len() { + let Some((part, _)) = empty_delimiter_part(source, index as usize) else { return f64::from_bits(crate::value::TAG_UNDEFINED); - } - let (advance, _, _) = crate::string::wtf8_step(source, byte_offset); - let end = (byte_offset + advance).min(source.len()); - let mut buf = [0u8; 4]; - let part = &source[byte_offset..end]; - buf[..part.len()].copy_from_slice(part); - let has_lone_surrogate = unsafe { - (*s).flags & STRING_FLAG_HAS_LONE_SURROGATES != 0 - && crate::string::bytes_have_lone_surrogate(part) }; - let result = if has_lone_surrogate { - js_string_from_wtf8_bytes(buf.as_ptr(), part.len() as u32) - } else { - js_string_from_bytes(buf.as_ptr(), part.len() as u32) + let result = match part { + // Half of an astral character: the same one-code-unit constructor + // `charAt` uses, which flags the WTF-8 lone surrogate it produces. + EmptyDelimiterPart::Surrogate(unit) => crate::string::string_from_code_unit(unit), + EmptyDelimiterPart::Bytes { start, end } => { + let mut buf = [0u8; 4]; + let bytes = &source[start..end]; + buf[..bytes.len()].copy_from_slice(bytes); + let has_lone_surrogate = unsafe { + (*s).flags & STRING_FLAG_HAS_LONE_SURROGATES != 0 + && crate::string::bytes_have_lone_surrogate(bytes) + }; + if has_lone_surrogate { + js_string_from_wtf8_bytes(buf.as_ptr(), bytes.len() as u32) + } else { + js_string_from_bytes(buf.as_ptr(), bytes.len() as u32) + } + } }; return crate::value::js_nanbox_string(result as i64); } @@ -228,19 +274,12 @@ pub extern "C" fn js_string_split_part_utf16_length( let delimiter_bytes = unsafe { slice::from_raw_parts(string_data(delimiter), (*delimiter).byte_len as usize) }; if delimiter_bytes.is_empty() { - let mut byte_offset = 0usize; - for _ in 0..index as usize { - if byte_offset >= source.len() { - return 0.0; - } - let (advance, _, _) = crate::string::wtf8_step(source, byte_offset); - byte_offset = (byte_offset + advance).min(source.len()); - } - if byte_offset >= source.len() { - return 0.0; - } - let (_, units, _) = crate::string::wtf8_step(source, byte_offset); - return units as f64; + // Every part of an empty-delimiter split is ONE code unit β€” including + // each half of an astral character, which is the whole point of #9409. + return match empty_delimiter_part(source, index as usize) { + Some((_, units)) => units as f64, + None => 0.0, + }; } let Some((start, end)) = split_part_byte_range(source, delimiter_bytes, index as usize) else { @@ -404,18 +443,34 @@ pub extern "C" fn js_string_split_n( // up to 3 bytes past the end of the allocation. Step the raw bytes with // the bounded WTF-8 decoder instead and emit each sequence verbatim; // well-formed input yields byte-identical parts. - // Pass 1: count the sequences. No allocation happens here, so the - // source payload cannot move under us. + // + // #9409: the unit of the split is a UTF-16 CODE UNIT, not a WTF-8 + // sequence. Β§22.1.3.23 runs SplitMatch over the code-unit sequence, so + // an astral character β€” one 4-byte WTF-8 sequence, but TWO code units + // (`"πŸ˜€".length === 2`) β€” becomes two parts, each a lone surrogate. + // Stepping one WTF-8 sequence per part made `"πŸ˜€".split("")` a + // one-element array while `charAt(0)`/`charAt(1)` on the same string + // already reported the two halves. + // + // Pass 1: count the parts. No allocation happens here, so the source + // payload cannot move under us. let mut n = 0usize; unsafe { let src = slice::from_raw_parts(string_data(s), (*s).byte_len as usize); let mut i = 0usize; - while i < src.len() { - let (advance, _, _) = crate::string::wtf8_step(src, i); + 'count: while i < src.len() { + let (advance, units, _) = crate::string::wtf8_step(src, i); i = (i + advance).min(src.len()); - n += 1; - if limit > 0 && n as i64 >= limit as i64 { - break; + // `units` is 2 for an astral sequence and 1 for anything else, + // including a WTF-8 lone surrogate. A malformed lead byte + // reports 0 units but still owns bytes, and the pre-#9409 code + // emitted it as its own part β€” keep that, or a `Buffer`-derived + // payload would silently lose bytes across a split/join. + for _ in 0..units.max(1) { + n += 1; + if limit > 0 && n as i64 >= limit as i64 { + break 'count; + } } } } @@ -436,32 +491,51 @@ pub extern "C" fn js_string_split_n( let arr_handle = scope.root_raw_mut_ptr(arr); let mut i = 0usize; + // The trailing half of an astral character, produced with its leading + // half and emitted by the next iteration. `limit` can stop the loop + // between the two, which is exactly what `"πŸ˜€".split("", 1)` must do. + let mut pending_low_surrogate: Option = None; for idx in 0..n { // Copy the sequence into a stack buffer BEFORE allocating: // `js_string_from_bytes` allocates first and copies second, so // handing it a pointer into the GC heap is the #5062 dangling-source // class. A WTF-8 sequence is at most 4 bytes. let mut buf = [0u8; 4]; - let seq_len; - unsafe { - let s_now = s_handle.get_raw_const_ptr::(); - let src = slice::from_raw_parts(string_data(s_now), (*s_now).byte_len as usize); - if i >= src.len() { - break; + let mut seq_len = 0usize; + // Set when this part is one half of an astral character; the half + // is synthesized from the code point, not copied from the payload. + let mut code_unit = pending_low_surrogate.take(); + if code_unit.is_none() { + unsafe { + let s_now = s_handle.get_raw_const_ptr::(); + let src = slice::from_raw_parts(string_data(s_now), (*s_now).byte_len as usize); + if i >= src.len() { + break; + } + let (advance, units, code_point) = crate::string::wtf8_step(src, i); + let end = (i + advance).min(src.len()); + if units == 2 && code_point >= 0x10000 { + let astral = code_point - 0x10000; + code_unit = Some(0xD800 + (astral >> 10) as u16); + pending_low_surrogate = Some(0xDC00 + (astral & 0x3FF) as u16); + } else { + seq_len = end - i; + buf[..seq_len].copy_from_slice(&src[i..end]); + } + i = end; } - let (advance, _, _) = crate::string::wtf8_step(src, i); - let end = (i + advance).min(src.len()); - seq_len = end - i; - buf[..seq_len].copy_from_slice(&src[i..end]); - i = end; } // `js_string_from_bytes` derives utf16_len from the bytes (correct // even for a malformed sequence) but hardcodes flags = 0. A lone // surrogate carved out of a WTF-8 source must keep its flag, or // `isWellFormed()` on the part wrongly reports true. + // `string_from_code_unit` β€” what `charAt` already uses β€” applies + // the same rule to a synthesized surrogate half. let seq = &buf[..seq_len]; let (sh, arr_now) = arr_handle.across_mut::(|| { - if src_has_lone_surrogates && crate::string::bytes_have_lone_surrogate(seq) { + if let Some(unit) = code_unit { + crate::string::string_from_code_unit(unit) + } else if src_has_lone_surrogates && crate::string::bytes_have_lone_surrogate(seq) { js_string_from_wtf8_bytes(seq.as_ptr(), seq_len as u32) } else { js_string_from_bytes(seq.as_ptr(), seq_len as u32) diff --git a/crates/perry-runtime/src/string/tests.rs b/crates/perry-runtime/src/string/tests.rs index 68d4d0127a..5968961bce 100644 --- a/crates/perry-runtime/src/string/tests.rs +++ b/crates/perry-runtime/src/string/tests.rs @@ -1123,3 +1123,176 @@ fn concat_memo_governor_recovers_after_backoff() { "backoff must expire into a probation window, got stuck for {windows} windows" ); } + +/// #9409: `split("")` cuts at UTF-16 CODE UNIT boundaries, so an astral +/// character yields TWO parts β€” a high and a low surrogate, each stored as +/// WTF-8 and flagged, exactly as `charAt` already returns them. +mod split_empty_delimiter_code_units { + use super::*; + + fn parts(source: &str, limit: i32) -> Vec> { + let scope = crate::gc::RuntimeHandleScope::new(); + let s = scope.root_string_ptr(js_string_from_bytes(source.as_ptr(), source.len() as u32)); + let empty = scope.root_string_ptr(js_string_from_bytes(b"".as_ptr(), 0)); + let arr = s.with_const_ptr::(|s| { + empty.with_const_ptr::(|e| { + crate::string::js_string_split_n(s, e, limit) + }) + }); + // `split` stores NaN-boxed string pointers with STRING_TAG; the mask is + // how the existing split tests read one back. + const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + unsafe { + (0..crate::array::js_array_length(arr) as usize) + .map(|i| { + let part = (crate::array::js_array_get_f64(arr, i as u32).to_bits() + & POINTER_MASK) as *const StringHeader; + std::slice::from_raw_parts( + crate::string::string_data(part), + (*part).byte_len as usize, + ) + .to_vec() + }) + .collect() + } + } + + fn flags_of(source: &str, index: usize) -> u32 { + let scope = crate::gc::RuntimeHandleScope::new(); + let s = scope.root_string_ptr(js_string_from_bytes(source.as_ptr(), source.len() as u32)); + let empty = scope.root_string_ptr(js_string_from_bytes(b"".as_ptr(), 0)); + let arr = s.with_const_ptr::(|s| { + empty.with_const_ptr::(|e| crate::string::js_string_split_n(s, e, -1)) + }); + const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + unsafe { + let part = (crate::array::js_array_get_f64(arr, index as u32).to_bits() & POINTER_MASK) + as *const StringHeader; + (*part).flags as u32 + } + } + + #[test] + fn an_astral_character_splits_into_its_two_surrogate_halves() { + assert_eq!( + parts("πŸ˜€", -1), + [vec![0xED, 0xA0, 0xBD], vec![0xED, 0xB8, 0x80]] + ); + assert_eq!( + parts("aπŸ˜€b", -1), + [ + b"a".to_vec(), + vec![0xED, 0xA0, 0xBD], + vec![0xED, 0xB8, 0x80], + b"b".to_vec() + ] + ); + } + + /// Each half must carry `HAS_LONE_SURROGATES`, or `isWellFormed()` and + /// `JSON.stringify` would treat a broken half as valid text. + #[test] + fn each_half_is_flagged_as_a_lone_surrogate() { + assert_ne!(flags_of("πŸ˜€", 0) & STRING_FLAG_HAS_LONE_SURROGATES, 0); + assert_ne!(flags_of("πŸ˜€", 1) & STRING_FLAG_HAS_LONE_SURROGATES, 0); + // A BMP part is untouched by the change and stays unflagged. + assert_eq!(flags_of("Γ©", 0) & STRING_FLAG_HAS_LONE_SURROGATES, 0); + } + + /// `limit` counts code units, so it can stop between the halves of one + /// character β€” `"πŸ˜€".split("", 1)` is a one-element array holding the lone + /// high surrogate. + #[test] + fn limit_counts_code_units_and_may_cut_a_pair() { + assert_eq!(parts("πŸ˜€", 1), [vec![0xED, 0xA0, 0xBD]]); + assert_eq!( + parts("πŸ˜€", 2), + [vec![0xED, 0xA0, 0xBD], vec![0xED, 0xB8, 0x80]] + ); + assert_eq!(parts("aπŸ˜€b", 2), [b"a".to_vec(), vec![0xED, 0xA0, 0xBD]]); + assert_eq!(parts("πŸ˜€", 0).len(), 0); + } + + /// BMP text, lone surrogates already in the payload, and the empty string + /// keep their pre-#9409 answers: the change is confined to 4-byte + /// sequences. + #[test] + fn non_astral_payloads_are_unchanged() { + assert_eq!( + parts("abc", -1), + [b"a".to_vec(), b"b".to_vec(), b"c".to_vec()] + ); + assert_eq!( + parts("Γ©ζΌ’", -1), + ["Γ©".as_bytes().to_vec(), "ζΌ’".as_bytes().to_vec()] + ); + assert_eq!(parts("", -1).len(), 0); + } + + fn scalar_part(source: &str, index: i32) -> Vec { + const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + let scope = crate::gc::RuntimeHandleScope::new(); + let s = scope.root_string_ptr(js_string_from_bytes(source.as_ptr(), source.len() as u32)); + let empty = scope.root_string_ptr(js_string_from_bytes(b"".as_ptr(), 0)); + let value = s.with_const_ptr::(|s| { + empty.with_const_ptr::(|e| { + crate::string::split::js_string_split_part_value(s, e, index) + }) + }); + let part = (value.to_bits() & POINTER_MASK) as *const StringHeader; + unsafe { + std::slice::from_raw_parts(crate::string::string_data(part), (*part).byte_len as usize) + .to_vec() + } + } + + fn scalar_part_len(source: &str, index: i32) -> f64 { + let scope = crate::gc::RuntimeHandleScope::new(); + let s = scope.root_string_ptr(js_string_from_bytes(source.as_ptr(), source.len() as u32)); + let empty = scope.root_string_ptr(js_string_from_bytes(b"".as_ptr(), 0)); + s.with_const_ptr::(|s| { + empty.with_const_ptr::(|e| { + crate::string::split::js_string_split_part_utf16_length(s, e, index) + }) + }) + } + + /// The two scalar-replacement fast paths answer `split("")[k]` and + /// `split("")[k].length` WITHOUT building the array, so they need the same + /// code-unit indexing or a scalar-replaced read would disagree with the + /// array form of the identical expression. + #[test] + fn the_scalar_fast_paths_index_the_same_code_units() { + assert_eq!(scalar_part("a\u{1F600}b", 0), b"a".to_vec()); + assert_eq!(scalar_part("a\u{1F600}b", 1), vec![0xED, 0xA0, 0xBD]); + assert_eq!(scalar_part("a\u{1F600}b", 2), vec![0xED, 0xB8, 0x80]); + assert_eq!(scalar_part("a\u{1F600}b", 3), b"b".to_vec()); + for index in 0..4 { + assert_eq!(scalar_part_len("a\u{1F600}b", index), 1.0, "index {index}"); + assert_eq!( + scalar_part("a\u{1F600}b", index), + parts("a\u{1F600}b", -1)[index as usize], + "index {index} must match the array form" + ); + } + assert_eq!(scalar_part_len("a\u{1F600}b", 4), 0.0); + } + + /// A malformed payload (a `Buffer`/FFI slice cut mid-sequence) reports 0 + /// UTF-16 units for its stray lead byte. It still has to come back as its + /// own part, or split/join would silently drop bytes β€” the #6085 guarantee. + #[test] + fn a_malformed_lead_byte_is_still_its_own_part() { + let scope = crate::gc::RuntimeHandleScope::new(); + let bytes = [0x80u8, b'|', 0xF0]; + let s = scope.root_string_ptr(js_string_from_wtf8_bytes( + bytes.as_ptr(), + bytes.len() as u32, + )); + let empty = scope.root_string_ptr(js_string_from_bytes(b"".as_ptr(), 0)); + let arr = s.with_const_ptr::(|s| { + empty.with_const_ptr::(|e| crate::string::js_string_split_n(s, e, -1)) + }); + assert_eq!(crate::array::js_array_length(arr), 3); + } +} diff --git a/test-files/test_gap_9408_multiline_anchors_lineterminators.ts b/test-files/test_gap_9408_multiline_anchors_lineterminators.ts new file mode 100644 index 0000000000..9ed3fca26b --- /dev/null +++ b/test-files/test_gap_9408_multiline_anchors_lineterminators.ts @@ -0,0 +1,113 @@ +// Gap test for #9408. ECMAScript Β§22.2.2.6 makes `^` and `$` under the `m` +// flag match at EVERY LineTerminator β€” LF, CR, U+2028 LINE SEPARATOR and +// U+2029 PARAGRAPH SEPARATOR β€” which is exactly the set #9218 taught a +// non-dotAll `.` to exclude. Rust's `(?m)` recognises LF only, so Perry's +// translation used to see one line where Node sees three. +// +// `\r\n` is the case that most naturally comes out wrong: the pair is TWO +// line terminators, so a `/^.*$/gm` scan must produce an EMPTY match between +// the CR and the LF. +// +// This file is byte-compared with `node --experimental-strip-types` by the gap +// suite. The `u`/non-`m` rows are controls: the translation must not move the +// anchors for a pattern that never asked for multiline. + +const LS = "\u2028"; +const PS = "\u2029"; + +function show(label: string, value: unknown): void { + console.log(label + ":" + JSON.stringify(value)); +} + +function endsWithTerminator(s: string): boolean { + return s.length > 0 && "\n\r\u2028\u2029".includes(s[s.length - 1]); +} + +const subjects: Array<[string, string]> = [ + ["lf", "one\ntwo"], + ["cr", "one\rtwo"], + ["crlf", "one\r\ntwo"], + ["lfcr", "one\n\rtwo"], + ["ls", "one" + LS + "two"], + ["ps", "one" + PS + "two"], + ["mixed", "a\rb\nc" + LS + "d" + PS + "e"], + ["leading", "\rone"], + ["trailing", "one\r"], + ["only", "\r"], + ["empty", ""], + ["none", "onetwo"], +]; + +// The headline scan: every line of the subject, anchors on both ends. +for (const [name, subject] of subjects) { + show("lines-" + name, subject.match(/^.*$/gm)); + show("lines-u-" + name, subject.match(/^.*$/gmu)); + // dotAll makes `.` eat the terminator, so the whole subject is one match. + // Subjects that END with a terminator are skipped here: a global scan must + // then report one more EMPTY match at the end of the input, and dropping it + // is a SEPARATE, pre-existing defect with nothing to do with the anchors + // (`"a".match(/a*/g)` is `["a"]` in Perry and `["a",""]` in Node). + if (!endsWithTerminator(subject)) show("lines-s-" + name, subject.match(/^.*$/gms)); + // Without `m` the anchors stay whole-input anchors. + show("lines-nom-" + name, subject.match(/^.*$/g)); +} + +// Anchors as zero-width insertion points. `^` before each line, `$` after +// each line; `\r\n` must take TWO insertions, not one. +for (const [name, subject] of subjects) { + show("insert-caret-" + name, subject.replace(/^/gm, ">")); + show("insert-dollar-" + name, subject.replace(/$/gm, "<")); + show("insert-both-" + name, subject.replace(/^|$/gm, "|")); + show("insert-caret-nom-" + name, subject.replace(/^/g, ">")); + show("insert-dollar-nom-" + name, subject.replace(/$/g, "<")); +} + +// Non-empty anchored patterns: the shapes the cc bundle actually ships. +show("gitdir", "line\r\ngitdir: /w/t\r\nline".match(/^gitdir:\s*(.+)$/m)); +show("diffgit", /^diff --git /m.test("index x\r\ndiff --git a b\r\n")); +show("osrelease", 'NAME="x"\r\nID="alpine"\r\n'.match(/^ID=["']?(\S+?)["']?\s*$/m)); +show("heading", "text\r\n## Title\r\nmore".match(/^## /gm)); +show("atx", "x\r\n### Deep heading\r\ny".match(/^#+\s+(.+)$/m)); +show("trailing-ws", "a \r\nb\t\r\nc ".replace(/[ \t]+$/gm, "")); +show("bullet", "- a\r\n- b\r\n- c".match(/^- (.+)$/gm)); + +// `$` immediately before each terminator, `^` immediately after. +for (const [name, subject] of subjects) { + show("dollar-idx-" + name, [...subject.matchAll(/$/gm)].map((m) => m.index)); + show("caret-idx-" + name, [...subject.matchAll(/^/gm)].map((m) => m.index)); +} + +// `re.exec` with a non-zero `lastIndex` is NOT covered here. Perry hands the +// engine `&subject[lastIndex..]`, so every assertion loses its left context: +// `\A` (and `\b`, and a lookbehind) is evaluated against the slice rather +// than the string. That is an independent defect β€” `/^b/g` with +// `lastIndex = 1` already matched "ab" before this fixture existed, with no +// `m` flag anywhere β€” and it needs the positional search APIs +// (`captures_at` / `captures_from_pos`), not a translation change. + +// `^` and `$` inside a character class are ordinary literals and must not be +// rewritten. `\^` / `\$` are escaped literals for the same reason. +show("class-literal", "a^b$c".match(/[$^]/g)); +show("class-literal-m", "a^b$c".match(/[$^]/gm)); +show("escaped-caret", "a^b".match(/\^/gm)); +show("escaped-dollar", "a$b".match(/\$/gm)); +show("negated-class-m", "x\ry".match(/[^\r]/gm)); + +// Anchors nested inside groups and alternations still see the flag. +show("group-anchor", "a\rb".match(/(?:^b)/m)); +show("alt-anchor", "a\rb".match(/(^b|^a)/gm)); +show("lookahead-anchor", "a\rb".match(/(?=^b)b/m)); +show("anchored-quantifier", "aa\rbb".match(/^(\w)\1$/gm)); + +// Case-insensitive and dotAll combinations exercise the flag prefix. +show("ci-anchor", "A\rB".match(/^b$/gim)); +show("dotall-anchor", "a\rb".match(/^a$/gms)); +show("all-flags", "A\rB".match(/^./gimsu)); + +// #9263 / #9216 controls: the word-boundary markers and the empty-class +// spellings share this translator and must survive alongside the anchors. +show("word-boundary-m", "a\rb".match(/^\b\w+\b$/gm)); +show("nonword-boundary-m", "Ξ©\rΞ©".match(/^\B/gm)); +show("any-class-m", "a\rb".match(/^[^]$/gm)); +show("empty-class-m", "a\rb".match(/^[]$/gm)); +show("word-complement-m", "a\rb".match(/^[\w\W]$/gm)); diff --git a/test-files/test_gap_9409_split_empty_code_units.ts b/test-files/test_gap_9409_split_empty_code_units.ts new file mode 100644 index 0000000000..51ba937c6a --- /dev/null +++ b/test-files/test_gap_9409_split_empty_code_units.ts @@ -0,0 +1,107 @@ +// Gap test for #9409. `String.prototype.split("")` splits into UTF-16 CODE +// UNITS (Β§22.1.3.23 β†’ SplitMatch over the code-unit sequence), not into +// Unicode code points: an astral character contributes TWO one-unit strings, +// each a lone surrogate. Perry stores strings as WTF-8 and used to step the +// payload one WTF-8 sequence at a time, so a 4-byte astral sequence produced +// ONE element where Node produces two. +// +// The code-point iterators are deliberately included as controls β€” `for…of`, +// the spread form and `Array.from` all iterate CODE POINTS and must keep +// returning one element for an astral character. `split("")` is the odd one +// out, and a fix that unified them would be just as wrong in the other +// direction. +// +// This file is byte-compared with `node --experimental-strip-types` by the gap +// suite. + +function show(label: string, value: unknown): void { + console.log(label + ":" + JSON.stringify(value)); +} + +// Render an array of strings as their code-unit sequences, so lone surrogates +// survive JSON.stringify unambiguously. +function units(parts: string[]): number[][] { + return parts.map((p) => Array.from({ length: p.length }, (_, i) => p.charCodeAt(i))); +} + +const samples: Array<[string, string]> = [ + ["ascii", "abc"], + ["empty", ""], + ["latin1", "cafΓ©"], + ["bmp", "Ωμέγα"], + ["cjk", "ζΌ’ε­—"], + ["astral", "πŸ˜€"], + ["astral-pair", "πŸ˜€πŸ˜€"], + ["mixed", "aπŸ˜€b"], + ["mixed-bmp", "Γ©πŸ˜€ζΌ’"], + ["flag", "πŸ‡©πŸ‡ͺ"], + ["zwj", "πŸ‘¨β€πŸ‘©β€πŸ‘¦"], + ["combining", "e\u0301"], + ["lone-high", "\ud83d"], + ["lone-low", "\ude00"], + ["lone-around", "a\ud83db"], + ["reversed-pair", "\ude00\ud83d"], +]; + +for (const [name, s] of samples) { + const parts = s.split(""); + const wellFormed = s.isWellFormed(); + show("len-" + name, s.length); + show("split-count-" + name, parts.length); + show("split-units-" + name, units(parts)); + show("split-roundtrip-" + name, parts.join("") === s); + // Code-point iterators are NOT code-unit based; these must not move. + show("spread-count-" + name, [...s].length); + // `Array.from` is skipped for the lone-surrogate samples: it takes a + // different lowering from the spread/for-of forms above and returns an EMPTY + // array for any string whose payload is not valid UTF-8. That is a separate + // defect (`js_array_from_string_codepoints` bails on `str::from_utf8`), not + // a code-unit question β€” the spread and for-of rows next to it are the + // code-point controls this fixture actually needs. + if (wellFormed) show("from-count-" + name, Array.from(s).length); + show("forof-count-" + name, (() => { let n = 0; for (const _c of s) n++; return n; })()); + // `charAt` is the code-unit reference the split must agree with. + show("charat-" + name, Array.from({ length: s.length }, (_, i) => s.charAt(i).charCodeAt(0))); +} + +// The `limit` argument counts code units too, and truncation may cut a pair. +show("limit-0", "πŸ˜€".split("", 0)); +show("limit-1", units("πŸ˜€".split("", 1))); +show("limit-2", units("πŸ˜€".split("", 2))); +show("limit-3", units("aπŸ˜€b".split("", 3))); +show("limit-large", units("πŸ˜€".split("", 99))); + +// A dynamic (non-literal) separator takes a different lowering than the +// literal `""` above; both must agree. +const sep = "".concat(""); +show("dyn-count", "aπŸ˜€b".split(sep).length); +show("dyn-units", units("aπŸ˜€b".split(sep))); +show("dyn-var-count", ((sepVar: string) => "πŸ˜€".split(sepVar).length)("")); + +// An empty RegExp separator is a DIFFERENT operation (RegExpExec-driven): it +// runs through the regex engine, which matches Unicode SCALARS, so it still +// reports 3 parts for "ab" where Node reports 4. That is the regex +// half of #9409 and needs a code-unit matching path, not a `split` change. + +// The scalar-replacement fast paths: when codegen proves the result array does +// not escape and only a constant index is read, `split("")[k]` and +// `split("")[k].length` are answered without building the array at all. They +// have to land on the same code units the array does. +show("scalar-part-0", "a\u{1F600}b".split("")[0].charCodeAt(0)); +show("scalar-part-1", "a\u{1F600}b".split("")[1].charCodeAt(0)); +show("scalar-part-2", "a\u{1F600}b".split("")[2].charCodeAt(0)); +show("scalar-part-3", "a\u{1F600}b".split("")[3].charCodeAt(0)); +show("scalar-part-oob", "a\u{1F600}b".split("")[4]); +show("scalar-len-0", "a\u{1F600}b".split("")[0].length); +show("scalar-len-1", "a\u{1F600}b".split("")[1].length); +show("scalar-len-2", "a\u{1F600}b".split("")[2].length); +show("scalar-len-3", "a\u{1F600}b".split("")[3].length); +show("scalar-wellformed-1", "a\u{1F600}b".split("")[1].isWellFormed()); + +// The well-formedness flag must survive: each half of a split pair is a lone +// surrogate, so it is not well-formed on its own. +const halves = "πŸ˜€".split(""); +show("wellformed-source", "πŸ˜€".isWellFormed()); +show("wellformed-parts", halves.map((h) => h.isWellFormed())); +show("wellformed-rejoined", halves.join("").isWellFormed()); +show("tojson-rejoined", JSON.stringify(halves.join("")) === JSON.stringify("πŸ˜€"));