diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 725c6e3b09e1d04..9693cfc259286d4 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -188,6 +188,8 @@ def test_request_rate(self): parsed_request_rate.seconds, self.request_rate.seconds ) + else: + self.assertIsNone(parsed_request_rate) class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): @@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): bad = [] +class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase): + # Non-decimal Unicode digits pass str.isdigit() but int() rejects + # them, so the directive must be silently ignored, not raise. + robots_txt = """\ +User-Agent: * +Disallow: /tmp/ +Crawl-delay: ² +Request-rate: ²/5 + """ + good = ['/foo.html'] + bad = ['/tmp/'] + crawl_delay = None + request_rate = None + + +class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /tmp/ +Request-rate: 5/² + """ + good = ['/foo.html'] + request_rate = None + bad = ['/tmp/'] + + class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): # also test that Allow and Diasallow works well with each other robots_txt = """\ diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 61772d90e2d53bf..8d0311d96f5e0b4 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -148,15 +148,15 @@ def parse(self, lines): # before trying to convert to int we need to make # sure that robots.txt has valid syntax otherwise # it will crash - if line[1].strip().isdigit(): + if line[1].strip().isdecimal(): entry.delay = int(line[1]) state = 2 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') # check if all values are sane - if (len(numbers) == 2 and numbers[0].strip().isdigit() - and numbers[1].strip().isdigit()): + if (len(numbers) == 2 and numbers[0].strip().isdecimal() + and numbers[1].strip().isdecimal()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 elif line[0] == "sitemap": diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst new file mode 100644 index 000000000000000..0746d3f12149d52 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst @@ -0,0 +1,4 @@ +:class:`urllib.robotparser.RobotFileParser` now silently ignores a +``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits +(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError` +and aborting the parse of the whole ``robots.txt`` file.