From f1042ecc7943d972e594e387eb50119a36aae199 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Thu, 9 Jul 2026 16:37:17 +0800 Subject: [PATCH 1/3] gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and Request-rate str.isdigit() returns True for non-decimal Unicode digits such as U+00B2 SUPERSCRIPT TWO, which int() then rejects with ValueError. In RobotFileParser.parse() this raised a raw ValueError that aborted parsing of the entire robots.txt file, unlike every other malformed directive which is silently ignored. Guard the Crawl-delay and Request-rate conversions with str.isdecimal() so a value written with non-decimal digits is ignored instead of raising. --- Lib/test/test_robotparser.py | 22 +++++++++++++++++++ Lib/urllib/robotparser.py | 6 ++--- ...-07-09-16-35-47.gh-issue-153404.daRkes.rst | 4 ++++ 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 725c6e3b09e1d04..44cd67f268c6883 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -246,6 +246,28 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): bad = [] +class NonDecimalCrawlDelayTest(BaseRequestRateTest, unittest.TestCase): + # Non-decimal Unicode digits pass str.isdigit() but int() rejects + # them, so the directive must be silently ignored, not raise. + robots_txt = """\ +User-Agent: * +Disallow: /. +Crawl-delay: ² + """ + good = ['/foo.html'] + bad = [] + + +class NonDecimalRequestRateTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /tmp/ +Request-rate: ²/5 + """ + good = ['/foo.html'] + bad = ['/tmp/'] + + class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): # also test that Allow and Diasallow works well with each other robots_txt = """\ diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 61772d90e2d53bf..8d0311d96f5e0b4 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -148,15 +148,15 @@ def parse(self, lines): # before trying to convert to int we need to make # sure that robots.txt has valid syntax otherwise # it will crash - if line[1].strip().isdigit(): + if line[1].strip().isdecimal(): entry.delay = int(line[1]) state = 2 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') # check if all values are sane - if (len(numbers) == 2 and numbers[0].strip().isdigit() - and numbers[1].strip().isdigit()): + if (len(numbers) == 2 and numbers[0].strip().isdecimal() + and numbers[1].strip().isdecimal()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 elif line[0] == "sitemap": diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst new file mode 100644 index 000000000000000..0746d3f12149d52 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst @@ -0,0 +1,4 @@ +:class:`urllib.robotparser.RobotFileParser` now silently ignores a +``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits +(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError` +and aborting the parse of the whole ``robots.txt`` file. From 9ab3e9cb20d401666acb10430aa142414c436682 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Fri, 17 Jul 2026 08:37:17 +0800 Subject: [PATCH 2/3] Consolidate test classes and add non-decimal denominator case --- Lib/test/test_robotparser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 44cd67f268c6883..ac14add76a7aa08 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -246,23 +246,24 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): bad = [] -class NonDecimalCrawlDelayTest(BaseRequestRateTest, unittest.TestCase): +class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase): # Non-decimal Unicode digits pass str.isdigit() but int() rejects # them, so the directive must be silently ignored, not raise. robots_txt = """\ User-Agent: * -Disallow: /. +Disallow: /tmp/ Crawl-delay: ² +Request-rate: ²/5 """ good = ['/foo.html'] - bad = [] + bad = ['/tmp/'] -class NonDecimalRequestRateTest(BaseRequestRateTest, unittest.TestCase): +class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): robots_txt = """\ User-agent: * Disallow: /tmp/ -Request-rate: ²/5 +Request-rate: 5/² """ good = ['/foo.html'] bad = ['/tmp/'] From 7c2bc43bbe06209962f8b3707df134940ad91833 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Fri, 17 Jul 2026 12:20:42 +0800 Subject: [PATCH 3/3] Add explicit expected values and assertIsNone for request_rate --- Lib/test/test_robotparser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index ac14add76a7aa08..9693cfc259286d4 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -188,6 +188,8 @@ def test_request_rate(self): parsed_request_rate.seconds, self.request_rate.seconds ) + else: + self.assertIsNone(parsed_request_rate) class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): @@ -257,6 +259,8 @@ class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase): """ good = ['/foo.html'] bad = ['/tmp/'] + crawl_delay = None + request_rate = None class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): @@ -266,6 +270,7 @@ class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): Request-rate: 5/² """ good = ['/foo.html'] + request_rate = None bad = ['/tmp/']