diff --git a/src/memory/store/safety/pii.rs b/src/memory/store/safety/pii.rs index 2f2a631..a0a6cec 100644 --- a/src/memory/store/safety/pii.rs +++ b/src/memory/store/safety/pii.rs @@ -27,7 +27,7 @@ //! dates-of-birth all require NER/LLM and are NOT addressed here. This //! module is honest about its scope. -use regex::{Regex, RegexSet}; +use regex::Regex; use std::sync::LazyLock; use super::{SanitizationReport, Sanitized}; @@ -137,30 +137,12 @@ static RRN_RE: LazyLock = static EMAIL_RE: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b").expect("email")); -// Cheap whole-text pre-filter so we skip the per-pattern scans entirely on -// PII-free text. Each entry roughly corresponds to one of the patterns above. -static SCREEN: LazyLock = LazyLock::new(|| { - RegexSet::new([ - r"\d{11,}", // any long digit run → CPF/CNPJ/CC/Aadhaar/IBAN - r"\d{3}\.\d{3}\.\d{3}-\d{2}", // CPF - r"\d{2}\.\d{3}\.\d{3}/\d{4}-\d{2}", // CNPJ - r"\d{2}-\d{8}-\d", // CUIT - r"(?i)[A-Z]{3,4}\d{6}", // RFC / general alphanumeric ID - r"(?:マイナンバー|個人番号|My\s?Number)", // JP keyword - r"\+\d{7}", // E.164 - r"\(?[2-9]\d{2}\)?[\s.\-]\d{3}[\s.\-]\d{4}", // NANP (parens optional) - r"\d{3}-\d{2}-\d{4}", // SSN - r"\b[A-Z]{2}\d{2}[A-Z0-9]", // IBAN prefix - r"\d{4}[\s\-]\d{4}[\s\-]\d{4}", // Aadhaar formatted - r"(?i)aadhaar|aadhar|आधार|uidai", // Aadhaar keyword - r"(?i)[A-Z]{5}\d{4}[A-Z]", // PAN-IN - r"(?i)[A-Z]{2}\d{6}[A-D]", // NINO - r"\b\d{8}[A-Z]\b", // DNI - r"(?i)[XYZ]\d{7}[A-Z]", // NIE - r"\d{6}-[1-4]\d{6}", // RRN - ]) - .expect("screen regex set") -}); +// ---------- Byte-oriented candidate pre-filter ---------- +// +// The single cheap byte pass that replaces the always-resident combined +// `RegexSet`. Lives in its own module — see `prefilter.rs` for the full rationale. +mod prefilter; +use prefilter::{scan_candidates, Candidates}; // ---------- Public API ---------- @@ -174,26 +156,38 @@ static SCREEN: LazyLock = LazyLock::new(|| { pub fn redact_pii(text: &str) -> Sanitized { let mut report = SanitizationReport::default(); - // Fast path: no candidate at all. - if !SCREEN.is_match(text) { - // Even the screen might miss normalized inputs; check normalized too. + // Fast path: cheap byte pre-filter on the raw text. Fullwidth / Arabic-Indic + // digits and folded punctuation only surface after normalization, so a clean + // raw scan still re-checks the normalized view before declaring the text PII- + // free (mirrors the old two-phase SCREEN check). + let raw_cand = scan_candidates(text); + if !raw_cand.any() { let nview = NormalizedView::build(text); - if !SCREEN.is_match(&nview.normalized) { + let ncand = scan_candidates(&nview.normalized); + if !ncand.any() { + log::trace!( + "[pii] redact_pii: no candidate before or after normalization (len={})", + text.len() + ); return Sanitized { value: text.to_string(), report, }; } + log::debug!("[pii] redact_pii: candidate surfaced only after normalization"); return splice_redactions( text, &nview, - collect_redactions(&nview.normalized), + collect_redactions(&nview.normalized, &ncand), &mut report, ); } let nview = NormalizedView::build(text); - let redactions = collect_redactions(&nview.normalized); + // Gate on candidates from the NORMALIZED text — the precise regexes run + // against it, so normalization-induced classes (folded digits) are included. + let ncand = scan_candidates(&nview.normalized); + let redactions = collect_redactions(&nview.normalized, &ncand); splice_redactions(text, &nview, redactions, &mut report) } @@ -213,13 +207,22 @@ pub fn redact_pii(text: &str) -> Sanitized { /// replace bytes inside a string, not reject the whole write. pub fn has_likely_pii(value: &str) -> bool { let nview = NormalizedView::build(value); - SCREEN.is_match(&nview.normalized) && !collect_strict_redactions(&nview.normalized).is_empty() + let cand = scan_candidates(&nview.normalized); + if !cand.any() { + return false; + } + !collect_strict_redactions(&nview.normalized, &cand).is_empty() } /// True when `value` contains an ordinary email address. Kept separate from /// [`has_likely_pii`] because scanner-built identifiers may legitimately /// contain email-like `@` segments. pub fn has_likely_email(value: &str) -> bool { + // Cheap gate: every email requires an `@`. Skip compiling the regex when + // the byte is absent (the common namespace/key case). + if !value.as_bytes().contains(&b'@') { + return false; + } EMAIL_RE.is_match(value) } @@ -232,8 +235,8 @@ struct Hit { token: &'static str, } -fn collect_redactions(norm: &str) -> Vec { - collect_redactions_inner(norm, true) +fn collect_redactions(norm: &str, cand: &Candidates) -> Vec { + collect_redactions_inner(norm, cand, true) } /// Variant of [`collect_redactions`] that omits bare-numeric patterns @@ -244,54 +247,89 @@ fn collect_redactions(norm: &str) -> Vec { /// where rejection on such a hit alone would have too many false /// positives on scanner-built identifiers (WhatsApp group JIDs /// `-@g.us`, timestamps, padded counters). -fn collect_strict_redactions(norm: &str) -> Vec { - collect_redactions_inner(norm, false) +fn collect_strict_redactions(norm: &str, cand: &Candidates) -> Vec { + collect_redactions_inner(norm, cand, false) } -fn collect_redactions_inner(norm: &str, include_bare_numeric: bool) -> Vec { +/// Run only the precise regexes whose class was flagged by [`scan_candidates`]. +/// Priority order (and therefore overlap-resolution) is byte-identical to the +/// unconditional version; the `if cand.*` guards only decide whether each class +/// runs, so a flagged class produces exactly the hits it always did. +fn collect_redactions_inner(norm: &str, cand: &Candidates, include_bare_numeric: bool) -> Vec { let mut hits: Vec = Vec::new(); // Priority order: most specific / highest-confidence first. - push_checksum(&mut hits, norm, &CPF_FMT_RE, PII_CPF, |s| { - valid_cpf(digits(s).as_slice()) - }); - push_checksum(&mut hits, norm, &CNPJ_FMT_RE, PII_CNPJ, |s| { - valid_cnpj(digits(s).as_slice()) - }); - push_checksum(&mut hits, norm, &CUIT_RE, PII_CUIT, |s| { - valid_cuit(digits(s).as_slice()) - }); + if cand.cpf_fmt { + push_checksum(&mut hits, norm, &CPF_FMT_RE, PII_CPF, |s| { + valid_cpf(digits(s).as_slice()) + }); + } + if cand.cnpj_fmt { + push_checksum(&mut hits, norm, &CNPJ_FMT_RE, PII_CNPJ, |s| { + valid_cnpj(digits(s).as_slice()) + }); + } + if cand.cuit { + push_checksum(&mut hits, norm, &CUIT_RE, PII_CUIT, |s| { + valid_cuit(digits(s).as_slice()) + }); + } // IBAN before credit card: CC can match an IBAN tail of all digits. - push_checksum(&mut hits, norm, &IBAN_RE, PII_IBAN, valid_iban); + if cand.iban { + push_checksum(&mut hits, norm, &IBAN_RE, PII_IBAN, valid_iban); + } if include_bare_numeric { // Credit card before bare CPF/CNPJ to avoid catching a 13-19 digit run as CPF/CNPJ. - push_checksum(&mut hits, norm, &CC_RE, PII_CC, valid_luhn); + if cand.cc { + push_checksum(&mut hits, norm, &CC_RE, PII_CC, valid_luhn); + } + if cand.cnpj_bare { + push_checksum(&mut hits, norm, &CNPJ_BARE_RE, PII_CNPJ, |s| { + valid_cnpj(digits(s).as_slice()) + }); + } + if cand.cpf_bare { + push_checksum(&mut hits, norm, &CPF_BARE_RE, PII_CPF, |s| { + valid_cpf(digits(s).as_slice()) + }); + } + } - push_checksum(&mut hits, norm, &CNPJ_BARE_RE, PII_CNPJ, |s| { - valid_cnpj(digits(s).as_slice()) - }); - push_checksum(&mut hits, norm, &CPF_BARE_RE, PII_CPF, |s| { - valid_cpf(digits(s).as_slice()) + if cand.aadhaar_fmt { + push_checksum(&mut hits, norm, &AADHAAR_FMT_RE, PII_AADHAAR, |s| { + valid_verhoeff(digits(s).as_slice()) }); } - - push_checksum(&mut hits, norm, &AADHAAR_FMT_RE, PII_AADHAAR, |s| { - valid_verhoeff(digits(s).as_slice()) - }); // Keyword-gated Aadhaar redacts only the captured 12-digit group. - push_captured(&mut hits, norm, &AADHAAR_KW_RE, PII_AADHAAR, |digits_str| { - valid_verhoeff(digits(digits_str).as_slice()) - }); + if cand.aadhaar_kw { + push_captured(&mut hits, norm, &AADHAAR_KW_RE, PII_AADHAAR, |digits_str| { + valid_verhoeff(digits(digits_str).as_slice()) + }); + } - push_checksum(&mut hits, norm, &DNI_RE, PII_DNI, valid_dni_es); - push_checksum(&mut hits, norm, &NIE_RE, PII_DNI, valid_nie_es); - push_checksum(&mut hits, norm, &NINO_RE, PII_NINO, valid_nino); - push_checksum(&mut hits, norm, &SSN_RE, PII_SSN, valid_ssn); - push_simple(&mut hits, norm, &RRN_RE, PII_RRN); - push_simple(&mut hits, norm, &RFC_RE, PII_RFC); - push_simple(&mut hits, norm, &PAN_IN_RE, PII_PAN_IN); + if cand.dni { + push_checksum(&mut hits, norm, &DNI_RE, PII_DNI, valid_dni_es); + } + if cand.nie { + push_checksum(&mut hits, norm, &NIE_RE, PII_DNI, valid_nie_es); + } + if cand.nino { + push_checksum(&mut hits, norm, &NINO_RE, PII_NINO, valid_nino); + } + if cand.ssn { + push_checksum(&mut hits, norm, &SSN_RE, PII_SSN, valid_ssn); + } + if cand.rrn { + push_simple(&mut hits, norm, &RRN_RE, PII_RRN); + } + if cand.rfc { + push_simple(&mut hits, norm, &RFC_RE, PII_RFC); + } + if cand.pan_in { + push_simple(&mut hits, norm, &PAN_IN_RE, PII_PAN_IN); + } if include_bare_numeric { // Phones: E.164 first (more specific), then NANP. Both are bare-numeric @@ -300,14 +338,25 @@ fn collect_redactions_inner(norm: &str, include_bare_numeric: bool) -> Vec // Strict callers (boundary checks like `has_likely_pii`) exclude these // so scanner-built namespace/key values (WhatsApp JIDs // `-@g.us`, telegram numeric peer IDs) don't get rejected. - push_simple(&mut hits, norm, &PHONE_E164_RE, PII_PHONE); - push_simple(&mut hits, norm, &PHONE_NANP_RE, PII_PHONE); + if cand.phone_e164 { + push_simple(&mut hits, norm, &PHONE_E164_RE, PII_PHONE); + } + if cand.phone_nanp { + push_simple(&mut hits, norm, &PHONE_NANP_RE, PII_PHONE); + } } // My Number — captured digit group only, keyword remains visible. - push_captured(&mut hits, norm, &MYNUM_RE, PII_MYNUM, |_| true); + if cand.mynumber { + push_captured(&mut hits, norm, &MYNUM_RE, PII_MYNUM, |_| true); + } dedupe_overlaps(&mut hits); + log::debug!( + "[pii] collect_redactions strict={} hits={}", + !include_bare_numeric, + hits.len() + ); hits } @@ -414,79 +463,10 @@ fn splice_redactions( // ---------- Unicode normalization for matching ---------- -struct NormalizedView { - normalized: String, - // For each byte offset i in `normalized`, `byte_map[i]` is the byte offset - // in the original string where the corresponding char *starts*. - // The last entry maps the normalized length to the original length, so - // `norm_to_orig(normalized.len())` is well-defined. - byte_map: Vec, -} - -impl NormalizedView { - fn build(original: &str) -> Self { - let mut normalized = String::with_capacity(original.len()); - let mut byte_map: Vec = Vec::with_capacity(original.len() + 1); - for (idx, ch) in original.char_indices() { - if is_zero_width(ch) { - continue; - } - let mapped = fold_char(ch); - let start = normalized.len(); - normalized.push(mapped); - // One byte_map entry per byte of the normalized char. - let added = normalized.len() - start; - for _ in 0..added { - byte_map.push(idx); - } - } - byte_map.push(original.len()); - Self { - normalized, - byte_map, - } - } - - fn norm_to_orig(&self, norm_byte: usize) -> usize { - if norm_byte >= self.byte_map.len() { - return *self.byte_map.last().unwrap_or(&0); - } - self.byte_map[norm_byte] - } -} - -fn is_zero_width(c: char) -> bool { - matches!( - c, - '\u{200B}' - | '\u{200C}' - | '\u{200D}' - | '\u{200E}' - | '\u{200F}' - | '\u{2060}' - | '\u{180E}' - | '\u{FEFF}' - ) -} - -fn fold_char(c: char) -> char { - match c { - // Fullwidth digits 0-9 - '\u{FF10}'..='\u{FF19}' => char::from_u32(c as u32 - 0xFF10 + 0x30).unwrap_or(c), - // Arabic-Indic digits ٠-٩ - '\u{0660}'..='\u{0669}' => char::from_u32(c as u32 - 0x0660 + 0x30).unwrap_or(c), - // Eastern Arabic-Indic digits ۰-۹ - '\u{06F0}'..='\u{06F9}' => char::from_u32(c as u32 - 0x06F0 + 0x30).unwrap_or(c), - // Common fullwidth punctuation we care about for PII formats - '\u{FF0D}' => '-', - '\u{FF0E}' => '.', - '\u{FF0F}' => '/', - '\u{FF1A}' => ':', - '\u{2010}'..='\u{2015}' => '-', // various unicode hyphens/dashes - '\u{2212}' => '-', // minus sign - other => other, - } -} +// Fullwidth / zero-width normalization used before matching. Lives in its own +// module — see `normalize.rs`. +mod normalize; +use normalize::NormalizedView; // ---------- Checksum helpers ---------- diff --git a/src/memory/store/safety/pii/normalize.rs b/src/memory/store/safety/pii/normalize.rs new file mode 100644 index 0000000..36ee8a5 --- /dev/null +++ b/src/memory/store/safety/pii/normalize.rs @@ -0,0 +1,79 @@ +//! Unicode normalization for PII matching. +//! +//! A pre-pass that defeats fullwidth-digit and zero-width-char bypasses while +//! keeping a byte map back to the original string, so matches found on the +//! normalized view can be spliced onto the exact original bytes. + +pub(super) struct NormalizedView { + pub(super) normalized: String, + // For each byte offset i in `normalized`, `byte_map[i]` is the byte offset + // in the original string where the corresponding char *starts*. + // The last entry maps the normalized length to the original length, so + // `norm_to_orig(normalized.len())` is well-defined. + byte_map: Vec, +} + +impl NormalizedView { + pub(super) fn build(original: &str) -> Self { + let mut normalized = String::with_capacity(original.len()); + let mut byte_map: Vec = Vec::with_capacity(original.len() + 1); + for (idx, ch) in original.char_indices() { + if is_zero_width(ch) { + continue; + } + let mapped = fold_char(ch); + let start = normalized.len(); + normalized.push(mapped); + // One byte_map entry per byte of the normalized char. + let added = normalized.len() - start; + for _ in 0..added { + byte_map.push(idx); + } + } + byte_map.push(original.len()); + Self { + normalized, + byte_map, + } + } + + pub(super) fn norm_to_orig(&self, norm_byte: usize) -> usize { + if norm_byte >= self.byte_map.len() { + return *self.byte_map.last().unwrap_or(&0); + } + self.byte_map[norm_byte] + } +} + +fn is_zero_width(c: char) -> bool { + matches!( + c, + '\u{200B}' + | '\u{200C}' + | '\u{200D}' + | '\u{200E}' + | '\u{200F}' + | '\u{2060}' + | '\u{180E}' + | '\u{FEFF}' + ) +} + +fn fold_char(c: char) -> char { + match c { + // Fullwidth digits 0-9 + '\u{FF10}'..='\u{FF19}' => char::from_u32(c as u32 - 0xFF10 + 0x30).unwrap_or(c), + // Arabic-Indic digits ٠-٩ + '\u{0660}'..='\u{0669}' => char::from_u32(c as u32 - 0x0660 + 0x30).unwrap_or(c), + // Eastern Arabic-Indic digits ۰-۹ + '\u{06F0}'..='\u{06F9}' => char::from_u32(c as u32 - 0x06F0 + 0x30).unwrap_or(c), + // Common fullwidth punctuation we care about for PII formats + '\u{FF0D}' => '-', + '\u{FF0E}' => '.', + '\u{FF0F}' => '/', + '\u{FF1A}' => ':', + '\u{2010}'..='\u{2015}' => '-', // various unicode hyphens/dashes + '\u{2212}' => '-', // minus sign + other => other, + } +} diff --git a/src/memory/store/safety/pii/prefilter.rs b/src/memory/store/safety/pii/prefilter.rs new file mode 100644 index 0000000..abace53 --- /dev/null +++ b/src/memory/store/safety/pii/prefilter.rs @@ -0,0 +1,238 @@ +//! Byte-oriented candidate pre-filter for the PII redactor. +//! +//! Replaces the always-resident combined `RegexSet` (one shared NFA plus a +//! per-thread lazy-DFA cache in *every* process/thread) with a single cheap pass +//! over the raw bytes. The scan derives per-class candidate flags from a handful +//! of structural signals — digit-run lengths, punctuation presence, uppercase / +//! alpha presence, `+`, and case-insensitive keyword probes (including the +//! non-Latin Aadhaar `आधार` and My-Number `マイナンバー` / `個人番号` keywords). +//! Each flag then decides whether that class's precise validation regex is worth +//! compiling and running; the precise `Regex`es stay `LazyLock`, so a class that +//! never sees a candidate is never compiled at all. At 100–1000 concurrent +//! agents that turns "combined NFA + N thread-local DFA caches resident forever" +//! into "only the regexes a workload actually needs, compiled on first hit". +//! +//! Correctness: every flag is a NECESSARY CONDITION of the class's *precise* +//! regex, so a flag can only over-fire (harmless — the precise regex then simply +//! fails to match), never under-fire on real PII. Consequently, whenever a +//! precise pattern would have matched under the old code path, its flag is set +//! and it still runs — output is unchanged. The union of the flags is a superset +//! of the old `SCREEN` set (pinned by `prefilter_is_superset_of_legacy_screen`). +//! The NANP phone class gates on the *screen*-entry necessary condition — an +//! internal `digit sep digit` separator OR a `\d{11,}` run (the old SCREEN reached +//! `PHONE_NANP_RE` through both) — faithfully preserving the documented "a bare +//! 10-digit NANP run is never reached" behavior while still redacting a bare +//! `1`+10-digit country-code number — see +//! `redact_pii_does_not_reach_bare_10_digit_nanp_today`. + +/// Per-class candidate flags produced by [`scan_candidates`]. A set flag means +/// "run this class's precise regex"; an unset flag means the class cannot +/// possibly match, so its regex is skipped (and never compiled). +#[derive(Default, Clone, Copy)] +pub(super) struct Candidates { + pub(super) cpf_fmt: bool, + pub(super) cnpj_fmt: bool, + pub(super) cuit: bool, + pub(super) iban: bool, + pub(super) cc: bool, + pub(super) cnpj_bare: bool, + pub(super) cpf_bare: bool, + pub(super) aadhaar_fmt: bool, + pub(super) aadhaar_kw: bool, + pub(super) dni: bool, + pub(super) nie: bool, + pub(super) nino: bool, + pub(super) ssn: bool, + pub(super) rrn: bool, + pub(super) rfc: bool, + pub(super) pan_in: bool, + pub(super) phone_e164: bool, + pub(super) phone_nanp: bool, + pub(super) mynumber: bool, +} + +impl Candidates { + /// True if any class is a candidate — i.e. the text is worth a precise pass. + pub(super) fn any(&self) -> bool { + self.cpf_fmt + || self.cnpj_fmt + || self.cuit + || self.iban + || self.cc + || self.cnpj_bare + || self.cpf_bare + || self.aadhaar_fmt + || self.aadhaar_kw + || self.dni + || self.nie + || self.nino + || self.ssn + || self.rrn + || self.rfc + || self.pan_in + || self.phone_e164 + || self.phone_nanp + || self.mynumber + } +} + +/// Case-insensitive (ASCII-only case folding) substring test over raw bytes. +/// Non-ASCII bytes compare exactly, so this also serves as an exact matcher for +/// the multibyte Devanagari / Japanese keyword needles. +fn contains_ci(hay: &[u8], needle: &[u8]) -> bool { + if needle.is_empty() { + return true; + } + if hay.len() < needle.len() { + return false; + } + hay.windows(needle.len()) + .any(|w| w.iter().zip(needle).all(|(a, b)| a.eq_ignore_ascii_case(b))) +} + +/// Aadhaar keyword needles — ASCII forms plus Devanagari `आधार`. +const AADHAAR_KEYWORDS: &[&[u8]] = &[b"aadhaar", b"aadhar", b"uidai", b"uid", "आधार".as_bytes()]; +/// My-Number Japanese keyword needles. The English `My\s?Number` variant is +/// handled separately (see `scan_candidates`) so any `\s` separator between the +/// two words is recognised, not just a literal space. +const MYNUMBER_JP_KEYWORDS: &[&[u8]] = &["マイナンバー".as_bytes(), "個人番号".as_bytes()]; + +/// Single linear pass over the bytes deriving every per-class candidate flag. +/// +/// Only ASCII structural bytes carry signal here; multibyte UTF-8 lead / +/// continuation bytes are all `>= 0x80`, so scanning `as_bytes()` for ASCII +/// digits/punctuation/letters is boundary-safe. Keyword probes run over the +/// same byte slice so the non-Latin needles match verbatim. +pub(super) fn scan_candidates(text: &str) -> Candidates { + let bytes = text.as_bytes(); + + let mut total_digits: usize = 0; + let mut max_digit_run: usize = 0; + let mut cur_run: usize = 0; + let mut has_dot = false; + let mut has_dash = false; + let mut has_slash = false; + // Any ASCII whitespace separator (space, tab, newline, CR, form feed, + // vertical tab). The precise Aadhaar pattern separates its groups with + // `[\s-]`, which matches the whole `\s` class — so gating on space/tab + // alone would under-fire on newline-separated Aadhaar (a real PII drop). + let mut has_ws = false; + let mut has_upper = false; + let mut has_alpha = false; + let mut has_xyz = false; + let mut has_plus = false; + // NANP-style "separated group" signal: some `[digit or ')'] [sep] [digit]` + // window exists (sep ∈ space/tab/./-). This is the necessary condition of + // the old SCREEN NANP entry, which required internal separators — keeping + // bare separator-less 10-digit runs out of the phone path. + let mut nanp_sep = false; + + for (i, &b) in bytes.iter().enumerate() { + if b.is_ascii_digit() { + total_digits += 1; + cur_run += 1; + if cur_run > max_digit_run { + max_digit_run = cur_run; + } + } else { + cur_run = 0; + match b { + b'.' => has_dot = true, + b'-' => has_dash = true, + b'/' => has_slash = true, + b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c => has_ws = true, + b'+' => has_plus = true, + b'A'..=b'Z' => { + has_upper = true; + has_alpha = true; + if matches!(b, b'X' | b'Y' | b'Z') { + has_xyz = true; + } + } + b'a'..=b'z' => { + has_alpha = true; + if matches!(b, b'x' | b'y' | b'z') { + has_xyz = true; + } + } + _ => {} + } + } + + if matches!(b, b' ' | b'\t' | b'.' | b'-') && i > 0 && i + 1 < bytes.len() { + let prev = bytes[i - 1]; + let next = bytes[i + 1]; + if (prev.is_ascii_digit() || prev == b')') && next.is_ascii_digit() { + nanp_sep = true; + } + } + } + + let has_digit = total_digits > 0; + let aadhaar_kw = AADHAAR_KEYWORDS.iter().any(|kw| contains_ci(bytes, kw)); + // English `My\s?Number` accepts any single `\s` between the words, so a tab- + // or newline-separated keyword (`My\tNumber`) must still flag. Requiring both + // `my` and `number` substrings is a necessary condition of the precise regex + // and covers every whitespace variant; it may over-fire (harmless — the + // precise `MYNUM_RE` re-checks the separator and the trailing 12 digits). + let mynumber = MYNUMBER_JP_KEYWORDS.iter().any(|kw| contains_ci(bytes, kw)) + || (contains_ci(bytes, b"my") && contains_ci(bytes, b"number")); + + let cand = Candidates { + // Formatted CPF `\d{3}\.\d{3}\.\d{3}-\d{2}` — needs digits, `.`, `-`. + cpf_fmt: has_digit && has_dot && has_dash, + // Formatted CNPJ `\d{2}\.\d{3}\.\d{3}/\d{4}-\d{2}` — adds `/`. + cnpj_fmt: has_digit && has_dot && has_slash && has_dash, + // CUIT `\d{2}-\d{8}-\d` — needs digits and `-`. + cuit: has_digit && has_dash, + // IBAN `[A-Z]{2}\d{2}…` — case-sensitive uppercase letters and digits. + iban: has_upper && has_digit, + // Credit card `(?:\d[\s\-]?){13,19}` — at least 13 digits total. + cc: total_digits >= 13, + // Bare CNPJ `\d{14}` — a 14-long digit run. + cnpj_bare: max_digit_run >= 14, + // Bare CPF `\d{11}` — an 11-long digit run. + cpf_bare: max_digit_run >= 11, + // Formatted Aadhaar `\d{4}[\s-]\d{4}[\s-]\d{4}` — 12 digits + a `\s`/dash + // separator (any ASCII whitespace, matching the precise `[\s-]` class). + aadhaar_fmt: total_digits >= 12 && (has_ws || has_dash), + // Keyword-gated Aadhaar — keyword suffices (precise regex checks digits). + aadhaar_kw, + // Spain DNI `\d{8}[A-Z]` — 8-run plus a letter. + dni: max_digit_run >= 8 && has_alpha, + // Spain NIE `[XYZ]\d{7}[A-Z]` — X/Y/Z, 7-run, letter. + nie: has_xyz && max_digit_run >= 7 && has_alpha, + // UK NINO `[A-Z]{2}\d{6}[A-D]` — letters and a 6-run. + nino: max_digit_run >= 6 && has_alpha, + // US SSN `\d{3}-\d{2}-\d{4}` — digits and `-`. + ssn: has_digit && has_dash, + // Korea RRN `\d{6}-[1-4]\d{6}` — a 6-run and `-`. + rrn: max_digit_run >= 6 && has_dash, + // Mexico RFC `[A-ZÑ&]{3,4}\d{6}[A-Z0-9]{3}` — a 6-run (leading class may + // be all non-ASCII `Ñ`, so gate on the digit run alone, not on letters). + rfc: max_digit_run >= 6, + // India PAN `[A-Z]{5}\d{4}[A-Z]` — letters and a 4-run. + pan_in: max_digit_run >= 4 && has_alpha, + // E.164 `\+\d{7,15}` — a `+` and a 7+ digit run. + phone_e164: has_plus && max_digit_run >= 7, + // NANP — screen-entry necessary condition. The old SCREEN reached + // `PHONE_NANP_RE` via either the separated-group pattern OR the long + // `\d{11,}` run (which covers a bare `1`+10-digit country-code number + // like `12025551234`). A bare 10-digit run still stays out of the phone + // path (no internal separator, run length 10 < 11). + phone_nanp: nanp_sep || max_digit_run >= 11, + // My Number — keyword suffices (precise regex checks the 12 digits). + mynumber, + }; + + log::trace!( + "[pii] scan_candidates bytes={} digits={} max_run={} nanp_sep={} any={}", + bytes.len(), + total_digits, + max_digit_run, + nanp_sep, + cand.any() + ); + + cand +} diff --git a/src/memory/store/safety/pii_tests.rs b/src/memory/store/safety/pii_tests.rs index f97a292..125638b 100644 --- a/src/memory/store/safety/pii_tests.rs +++ b/src/memory/store/safety/pii_tests.rs @@ -75,6 +75,16 @@ fn my_number_redacted_with_keyword() { fn bare_12_digits_without_keyword_kept() { unchanged("Order 123456789012 shipped today."); } +#[test] +fn my_number_keyword_tab_separator_redacted() { + // `My\s?Number` accepts any single `\s`; the byte prefilter must recognise a + // tab-separated keyword, not just a literal space. + redacts("My\tNumber 123456789012", PII_MYNUM); +} +#[test] +fn my_number_keyword_newline_separator_redacted() { + redacts("My\nNumber 123456789012", PII_MYNUM); +} // --- E.164 + NANP phone --- #[test] @@ -93,6 +103,13 @@ fn nanp_with_country_code_redacted() { fn nanp_invalid_area_code_kept() { unchanged("score 115-555-0123 ish"); } +#[test] +fn nanp_bare_country_code_redacted() { + // Separator-less `1`+10-digit NANP: the old SCREEN reached PHONE_NANP_RE via + // the `\d{11,}` run; the prefilter must keep gating this through the phone + // class (the bare-CPF checksum rejects it, so nothing else redacts it). + redacts("12025551234", PII_PHONE); +} // --- SSN --- #[test] @@ -148,6 +165,12 @@ fn aadhaar_keyword_bare_redacted() { fn aadhaar_invalid_verhoeff_kept() { unchanged("Random 2341 2341 2345 nope"); } +#[test] +fn aadhaar_formatted_newline_separator_redacted() { + // AADHAAR_FMT_RE separates groups with `[\s-]`; a newline-separated Aadhaar + // (no keyword, no dash) must still flag the formatted class in the prefilter. + redacts("2341\n2341\n2346", PII_AADHAAR); +} // --- PAN-IN --- #[test] @@ -386,3 +409,154 @@ fn redact_pii_does_not_reach_bare_10_digit_nanp_today() { fn empty_text_is_noop() { unchanged(""); } + +// --- Byte prefilter: per-class positives (incl. non-Latin) --- + +/// Devanagari Aadhaar keyword must still route into the keyword-gated Aadhaar +/// path (the `आधार` needle lives in `AADHAAR_KEYWORDS`). +#[test] +fn aadhaar_devanagari_keyword_redacted() { + redacts("आधार 234123412346", PII_AADHAAR); +} + +/// Japanese My-Number keyword (kanji form) routes into the My-Number path. +#[test] +fn my_number_kanji_keyword_redacted() { + redacts("個人番号 123456789012", PII_MYNUM); +} + +/// `scan_candidates` flags the right class for representative per-class inputs. +#[test] +fn scan_flags_expected_classes() { + assert!(scan_candidates("111.444.777-35").cpf_fmt); + assert!(scan_candidates("11.222.333/0001-81").cnpj_fmt); + assert!(scan_candidates("20-11111111-2").cuit); + assert!(scan_candidates("DE89370400440532013000").iban); + assert!(scan_candidates("4111111111111111").cc); + assert!(scan_candidates("11222333000181").cnpj_bare); + assert!(scan_candidates("11144477735").cpf_bare); + assert!(scan_candidates("2341 2341 2346").aadhaar_fmt); + assert!(scan_candidates("aadhaar 234123412346").aadhaar_kw); + assert!(scan_candidates("आधार 234123412346").aadhaar_kw); + assert!(scan_candidates("12345678Z").dni); + assert!(scan_candidates("X1234567L").nie); + assert!(scan_candidates("AB123456C").nino); + assert!(scan_candidates("123-45-6789").ssn); + assert!(scan_candidates("900101-1234567").rrn); + assert!(scan_candidates("VECJ880326XK4").rfc); + assert!(scan_candidates("ABCDE1234F").pan_in); + assert!(scan_candidates("+15551234567").phone_e164); + assert!(scan_candidates("415-555-0123").phone_nanp); + assert!(scan_candidates("マイナンバー 123456789012").mynumber); + assert!(scan_candidates("My Number 123456789012").mynumber); +} + +/// Clean, PII-free text flags no class at all — the whole precise pass is +/// skipped and every precise regex stays uncompiled. +#[test] +fn scan_clean_text_flags_nothing() { + for clean in [ + "", + "just some ordinary words here", + "memory/global/preferences", + "the quick brown fox", + "https://example.com/path?q=1", + "snake_case_identifier_v2", + ] { + let cand = scan_candidates(clean); + assert!(!cand.any(), "clean text flagged a class: {clean:?}"); + } +} + +/// A bare separator-less 10-digit run must NOT flag the NANP phone class — this +/// is what preserves the documented "bare 10-digit NANP is never reached" +/// behavior even though the precise NANP regex would otherwise match it. +#[test] +fn scan_bare_10_digit_run_does_not_flag_nanp() { + assert!(!scan_candidates("call me at 2025551234 thanks").phone_nanp); +} + +/// Parity oracle: the new byte prefilter must be a SUPERSET of the legacy +/// `SCREEN` regex set. For every corpus input, if the old combined set would +/// have matched the normalized text, the new per-class scan must flag at least +/// one class — otherwise a real PII candidate would be silently dropped. +#[test] +fn prefilter_is_superset_of_legacy_screen() { + use regex::RegexSet; + + // Byte-for-byte the pattern list this PR removed from `pii.rs`. + let legacy_screen = RegexSet::new([ + r"\d{11,}", + r"\d{3}\.\d{3}\.\d{3}-\d{2}", + r"\d{2}\.\d{3}\.\d{3}/\d{4}-\d{2}", + r"\d{2}-\d{8}-\d", + r"(?i)[A-Z]{3,4}\d{6}", + r"(?:マイナンバー|個人番号|My\s?Number)", + r"\+\d{7}", + r"\(?[2-9]\d{2}\)?[\s.\-]\d{3}[\s.\-]\d{4}", + r"\d{3}-\d{2}-\d{4}", + r"\b[A-Z]{2}\d{2}[A-Z0-9]", + r"\d{4}[\s\-]\d{4}[\s\-]\d{4}", + r"(?i)aadhaar|aadhar|आधार|uidai", + r"(?i)[A-Z]{5}\d{4}[A-Z]", + r"(?i)[A-Z]{2}\d{6}[A-D]", + r"\b\d{8}[A-Z]\b", + r"(?i)[XYZ]\d{7}[A-Z]", + r"\d{6}-[1-4]\d{6}", + ]) + .expect("legacy screen"); + + let corpus = [ + // Real PII, one per class. + "CPF: 111.444.777-35.", + "Sem mascara 11144477735 ok", + "CNPJ 11.222.333/0001-81", + "contract 11222333000181 yes", + "CUIT 20-11111111-2", + "Mi RFC VECJ880326XK4 .", + "マイナンバー: 123456789012", + "個人番号 123456789012", + "My Number 123456789012", + // Whitespace-separator variants the precise regexes accept via `\s`. + "My\tNumber 123456789012", + "My\nNumber 123456789012", + "2341\n2341\n2346", + "12025551234", + "phone +15551234567", + "call 415-555-0123 thanks", + "+1 (212) 555-7890", + "ssn 123-45-6789", + "card 4111 1111 1111 1111 thanks", + "card 378282246310005 used", + "IBAN DE89370400440532013000 ok", + "Aadhaar 2341 2341 2346", + "Aadhaar: 234123412346", + "आधार 234123412346", + "uidai 234123412346", + "PAN: ABCDE1234F", + "NI no AB123456C", + "DNI 12345678Z", + "NIE X1234567L", + "주민번호 900101-1234567", + // Scanner-built / borderline identifiers. + "12025551234-1543890267@g.us:2026-05-30", + "+12025551234:2026-05-30", + "accepted:000001747729035001", + "screen_intelligence_vision-1747729035001-VSCode", + "Order 123456789012 shipped today.", + // Clean text (screen won't match; nothing to assert but exercises path). + "memory/global/preferences", + "the quick brown fox jumps", + "just some ordinary words here", + ]; + + for input in corpus { + let nview = NormalizedView::build(input); + if legacy_screen.is_match(&nview.normalized) { + assert!( + scan_candidates(&nview.normalized).any(), + "legacy SCREEN matched but new prefilter flagged nothing: {input:?}" + ); + } + } +}