From 52d9db47ca5c84c3fda81214c0d4d42fcbc624f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?=
 <tonghuaroot@gmail.com>
Date: Fri, 17 Jul 2026 13:54:46 +0800
Subject: [PATCH] gh-153404: Silently ignore non-decimal digits in robots.txt
 Crawl-delay and Request-rate (GH-153405)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

str.isdigit() returned True for non-decimal Unicode digits such as
U+00B2 SUPERSCRIPT TWO, which int() then rejected with ValueError.
(cherry picked from commit 597ec2d7e5f8bbd5a1eb9e6780444cae46cec3ea)

Co-authored-by: tonghuaroot (童话) <tonghuaroot@gmail.com>
---
 Lib/test/test_robotparser.py                  | 28 +++++++++++++++++++
 Lib/urllib/robotparser.py                     |  6 ++--
 ...-07-09-16-35-47.gh-issue-153404.daRkes.rst |  4 +++
 3 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst

diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index 620e579c6f21518..415d9a76077552c 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -188,6 +188,8 @@ def test_request_rate(self):
                         parsed_request_rate.seconds,
                         self.request_rate.seconds
                     )
+                else:
+                    self.assertIsNone(parsed_request_rate)
 
 
 class EmptyFileTest(BaseRequestRateTest, unittest.TestCase):
@@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase):
     bad = []
 
 
+class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase):
+    # Non-decimal Unicode digits pass str.isdigit() but int() rejects
+    # them, so the directive must be silently ignored, not raise.
+    robots_txt = """\
+User-Agent: *
+Disallow: /tmp/
+Crawl-delay: ²
+Request-rate: ²/5
+    """
+    good = ['/foo.html']
+    bad = ['/tmp/']
+    crawl_delay = None
+    request_rate = None
+
+
+class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase):
+    robots_txt = """\
+User-agent: *
+Disallow: /tmp/
+Request-rate: 5/²
+    """
+    good = ['/foo.html']
+    request_rate = None
+    bad = ['/tmp/']
+
+
 class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
     # also test that Allow and Diasallow works well with each other
     robots_txt = """\
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index 469af30d0d7aa8c..3e25b6c73863410 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -143,15 +143,15 @@ def parse(self, lines):
                         # before trying to convert to int we need to make
                         # sure that robots.txt has valid syntax otherwise
                         # it will crash
-                        if line[1].strip().isdigit():
+                        if line[1].strip().isdecimal():
                             entry.delay = int(line[1])
                         state = 2
                 elif line[0] == "request-rate":
                     if state != 0:
                         numbers = line[1].split('/')
                         # check if all values are sane
-                        if (len(numbers) == 2 and numbers[0].strip().isdigit()
-                            and numbers[1].strip().isdigit()):
+                        if (len(numbers) == 2 and numbers[0].strip().isdecimal()
+                            and numbers[1].strip().isdecimal()):
                             entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
                         state = 2
                 elif line[0] == "sitemap":
diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
new file mode 100644
index 000000000000000..0746d3f12149d52
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
@@ -0,0 +1,4 @@
+:class:`urllib.robotparser.RobotFileParser` now silently ignores a
+``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits
+(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError`
+and aborting the parse of the whole ``robots.txt`` file.