fix: recognize structured rate claim windows

scripts/validate_storefront_claims.py

@@ -5,7 +5,7 @@
 import csv
 import re
 from pathlib import Path
-from typing import Iterable, List, Pattern, Sequence, Tuple
+from typing import Iterable, Iterator, List, Match, Pattern, Sequence, Set, Tuple
 
 ROOT = Path(__file__).resolve().parents[1]
 CONTRACT = "https://api.oilpriceapi.com/product-facts.json"
@@ -21,19 +21,26 @@
     ".so",
 }
 _RATE_COUNT = r"\d[\d,]*"
-_RATE_ACTION = r"(?:(?:api[- ]+)?(?:requests?|calls?)|reqs?\.?)"
-_RATE_PERIOD = r"(?:minutes?|mins?\.?|hours?|hrs?\.?|days?)"
-_RATE_FREQUENCY = r"(?:minutely|hourly|daily|per[- ]+(?:minute|hour|day))"
-FIXED_RATE = re.compile(
-    rf"\b{_RATE_COUNT}[- ]+{_RATE_ACTION}"
-    rf"(?:(?:[- ]*(?:per|an?|each|every)[- ]+|[- ]*/[- ]*){_RATE_PERIOD}\b|"
-    rf"[- ]+{_RATE_FREQUENCY}\b)|"
-    rf"\b{_RATE_FREQUENCY}[- ]+"
-    rf"(?:(?:api[- ]+)?(?:requests?|calls?)[- ]+)?"
-    rf"(?:limit|allowance|quota|cap)\s*(?:of|is|:|=)?\s*"
-    rf"{_RATE_COUNT}[- ]+{_RATE_ACTION}\b",
+_RATE_ACTION = r"(?:(?:api[- ]+)?(?:requests?|calls?|queries?|hits?|credits?)|reqs?\.?)"
+_RATE_UNIT_SINGULAR = r"(?:second|sec|minute|min|hour|hr|day|week|month|year)"
+_RATE_UNIT = rf"{_RATE_UNIT_SINGULAR}s?\.?"
+_RATE_ADVERB = r"(?:secondly|minutely|hourly|daily|weekly|monthly|yearly)"
+_RATE_DURATION = rf"(?:(?:a|an|one|any|rolling|{_RATE_COUNT})[- ]+){{0,3}}{_RATE_UNIT}"
+_RATE_NUMBER_PATTERN = re.compile(rf"(?<![\w.]){_RATE_COUNT}(?![\w.])")
+_RATE_ACTION_PATTERN = re.compile(rf"\b{_RATE_ACTION}\b", re.IGNORECASE)
+_RATE_CADENCE_PATTERN = re.compile(
+    rf"\b{_RATE_ADVERB}\b|"
+    rf"(?:/[- ]*|\b(?:per|each|every|in|within|over|during|for)\b[- ]+)"
+    rf"{_RATE_DURATION}\b|"
+    rf"\b(?:a|an|one|any|rolling|{_RATE_COUNT})[- ]+{_RATE_UNIT}\b",
     re.IGNORECASE,
 )
+_RATE_BOUNDARY_PATTERN = re.compile(
+    r"(?:\r?\n)+|\s+#\s+|[!?;]+(?:\s+|$)|\.(?:\s+|$)"
+)
+_HTML_TAG_PATTERN = re.compile(r"<[^>]{1,500}>")
+_MAX_ACTION_COUNT_GAP = 64
+_MAX_RATE_SPAN = 200
 BLOCKED: Sequence[Tuple[str, Pattern[str]]] = (
     ("real-time claim", re.compile(r"\breal[ -]?time\b", re.IGNORECASE)),
     (
@@ -69,8 +76,6 @@
     (
         "fixed allowance",
         re.compile(
-            r"\b\d[\d,]*\s+(?:free\s+)?(?:api\s+requests?|station\s+queries?)"
-            r"\s*(?:/|per\s+)month\b|"
             r"\bmonthly\s+station\s+(?:query|request)\s+limit\b",
             re.IGNORECASE,
         ),
@@ -99,10 +104,6 @@
             re.IGNORECASE,
         ),
     ),
-    (
-        "fixed demo rate",
-        FIXED_RATE,
-    ),
 )
 
 
@@ -146,16 +147,102 @@ def discover_installed_surfaces(package_root: Path) -> List[Path]:
     return sorted(set(surfaces))
 
 
+def _bounded_rate_segments(text: str) -> Iterator[Tuple[int, str]]:
+    start = 0
+    for boundary in _RATE_BOUNDARY_PATTERN.finditer(text):
+        segment = text[start : boundary.start()]
+        if segment.strip():
+            yield start, segment
+        start = boundary.end()
+    if text[start:].strip():
+        yield start, text[start:]
+
+
+def _token_gap(left: Match[str], right: Match[str]) -> int:
+    if left.end() <= right.start():
+        return right.start() - left.end()
+    if right.end() <= left.start():
+        return left.start() - right.end()
+    return 0
+
+
+def _claim_span(
+    action: Match[str], count: Match[str], cadence: Match[str]
+) -> Tuple[int, int]:
+    return (
+        min(action.start(), count.start(), cadence.start()),
+        max(action.end(), count.end(), cadence.end()),
+    )
+
+
+def _fixed_rate_claims(text: str) -> List[str]:
+    """Find count + API action + cadence triples in a bounded sentence window."""
+    claims: List[str] = []
+    seen: Set[Tuple[int, str]] = set()
+
+    for segment_offset, segment in _bounded_rate_segments(text):
+        searchable = _HTML_TAG_PATTERN.sub(" ", segment)
+        counts = list(_RATE_NUMBER_PATTERN.finditer(searchable))
+        cadences = list(_RATE_CADENCE_PATTERN.finditer(searchable))
+        if not counts or not cadences:
+            continue
+        allowance_counts = [
+            count
+            for count in counts
+            if not any(
+                cadence.start() <= count.start() and count.end() <= cadence.end()
+                for cadence in cadences
+            )
+        ]
+        if not allowance_counts:
+            continue
+
+        for action in _RATE_ACTION_PATTERN.finditer(searchable):
+            nearby_counts = [
+                count
+                for count in allowance_counts
+                if _token_gap(action, count) <= _MAX_ACTION_COUNT_GAP
+            ]
+            if not nearby_counts:
+                continue
+            count = min(nearby_counts, key=lambda token: _token_gap(action, token))
+
+            bounded_cadences = [
+                cadence
+                for cadence in cadences
+                if _claim_span(action, count, cadence)[1]
+                - _claim_span(action, count, cadence)[0]
+                <= _MAX_RATE_SPAN
+            ]
+            if not bounded_cadences:
+                continue
+            cadence = min(
+                bounded_cadences,
+                key=lambda token: _claim_span(action, count, token)[1]
+                - _claim_span(action, count, token)[0],
+            )
+            claim_start, claim_end = _claim_span(action, count, cadence)
+            claim = re.sub(r"\s+", " ", searchable[claim_start:claim_end]).strip()
+            key = (segment_offset + claim_start, claim)
+            if key not in seen:
+                seen.add(key)
+                claims.append(claim)
+    return claims
+
+
 def _claim_failures(root: Path, surfaces: Iterable[Path]) -> List[str]:
     failures: List[str] = []
     for path in surfaces:
         text = path.read_text(encoding="utf-8")
         for label, pattern in BLOCKED:
-            match = pattern.search(text)
-            if match:
+            for match in pattern.finditer(text):
                 failures.append(
                     f"{path.relative_to(root)}: {label} matched {match.group(0)!r}"
                 )
+        for claim in _fixed_rate_claims(text):
+            failures.append(
+                f"{path.relative_to(root)}: fixed demo rate matched {claim!r}"
+            )
     return failures
 
 

tests/test_storefront_claims.py

@@ -157,6 +157,25 @@ def test_rejects_claim_in_future_installed_package_data(tmp_path: Path) -> None:
         "100 API requests every hour",
         "daily cap is 50 calls",
         "50-call-per-day allowance",
+        "50 requests per 24 hours",
+        "50 requests every 24 hours",
+        "50 API calls in a day",
+        "daily 50-request limit",
+        "24-hour quota of 50 calls",
+        "API call rate limit is 200 every hour",
+        "weekly 5,000-credit allowance",
+        "2,000 queries per 30 days",
+        "50/day API calls",
+        "50 daily API calls",
+        "API calls: 50 per day",
+        "API calls daily: 50",
+        "daily API calls: 50",
+        "50 requests over a rolling 24-hour window",
+        "50 API calls during any one-hour period",
+        "50 request limit per day",
+        "50-call limit per day",
+        "50 requests allowed daily",
+        "<p>daily <strong>50</strong> API calls</p>",
     ],
 )
 def test_rejects_fixed_rate_aliases_in_installed_text(tmp_path: Path, claim: str) -> None:
@@ -165,13 +184,26 @@ def test_rejects_fixed_rate_aliases_in_installed_text(tmp_path: Path, claim: str
     assert any("fixed demo rate" in failure for failure in failures), failures
 
 
+def test_reports_one_failure_for_overlapping_monthly_rate(tmp_path: Path) -> None:
+    failures = _installed_text_failures(tmp_path, "50 API requests per month")
+
+    matching = [failure for failure in failures if "50 API requests per month" in failure]
+    assert len(matching) == 1, failures
+
+
 @pytest.mark.parametrize(
     "text",
     [
         "SDK version 1.12.4 supports Python 3.8.",
         "Run 50 tests daily.",
         "The response contains 50 records per page.",
         "Retry attempt 50 failed.",
+        "Daily 50-test limit.",
+        "A 24-hour test window contains 50 assertions.",
+        "The monthly report contains 50 records.",
+        "1 week queries",
+        "return 120  # 2 minutes for year queries",
+        "50 records are returned. Requests include timestamps updated daily.",
     ],
 )
 def test_fixed_rate_aliases_do_not_match_versions_or_test_counts(