diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 9693cfc259286d..c63e5e486e3899 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -399,6 +399,80 @@ class UserAgentStartsGroupTest(BaseRobotTest, unittest.TestCase): Disallow: /another/path\ """ + +class CrawlDelayInGroupTest(BaseRobotTest, unittest.TestCase): + # Crawl-delay and Request-rate do not end a group (RFC 9309, + # section 2.2.4), but only apply to the user agents directly above them. + robots_txt = """\ +User-agent: spambot +Crawl-delay: 3 +User-agent: eggsbot +Crawl-delay: 5 +Request-rate: 9/30 +User-agent: hambot +Disallow: /some/path + """ + good = [('spambot', '/'), ('eggsbot', '/'), ('hambot', '/')] + bad = [('spambot', '/some/path'), ('eggsbot', '/some/path'), + ('hambot', '/some/path')] + expected_output = """\ +User-agent: spambot +Crawl-delay: 3 +Disallow: /some/path + +User-agent: eggsbot +Crawl-delay: 5 +Request-rate: 9/30 +Disallow: /some/path + +User-agent: hambot +Disallow: /some/path\ +""" + + def test_crawl_delay_and_request_rate(self): + parser = self.parser + self.assertEqual(parser.crawl_delay('spambot'), 3) + self.assertIsNone(parser.request_rate('spambot')) + self.assertEqual(parser.crawl_delay('eggsbot'), 5) + self.assertEqual(parser.request_rate('eggsbot'), (9, 30)) + self.assertIsNone(parser.crawl_delay('hambot')) + self.assertIsNone(parser.request_rate('hambot')) + + +class InvalidCrawlDelayInGroupTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Crawl-delay: pears +User-agent: eggsbot +Request-rate: 9/banana +User-agent: hambot +Disallow: /some/path + """ + good = [('spambot', '/'), ('eggsbot', '/'), ('hambot', '/')] + bad = [('spambot', '/some/path'), ('eggsbot', '/some/path'), + ('hambot', '/some/path')] + + +class CrawlDelayAfterRulesTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path +Crawl-delay: 3 +User-agent: eggsbot +Disallow: /another/path + """ + good = [('spambot', '/another/path'), ('eggsbot', '/some/path')] + bad = [('spambot', '/some/path'), ('eggsbot', '/another/path')] + expected_output = """\ +User-agent: spambot +Crawl-delay: 3 +Disallow: /some/path + +User-agent: eggsbot +Disallow: /another/path\ +""" + + class IgnoreEmptyLinesTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: spambot diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 985333c7100438..b0acfd45530c30 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -97,6 +97,14 @@ def _add_entry(self, entry): else: self.groups[agent] = merge_entries(self.groups[agent], entry) + def _add_group(self, group): + # Only the last entry of a group has rules, but they apply to + # all entries in the group. + rulelines = group[-1].rulelines + for entry in group: + entry.rulelines = rulelines + self._add_entry(entry) + def parse(self, lines): """Parse the input lines from a robots.txt file. @@ -108,8 +116,16 @@ def parse(self, lines): # 0: start state # 1: saw user-agent line # 2: saw an allow or disallow line + # 3: saw a crawl-delay or request-rate line, but no allow or + # disallow line yet + # Crawl-delay and request-rate lines do not end a group (RFC 9309, + # section 2.2.4). A user-agent line after them starts a new entry + # in the same group: the rules apply to the whole group, but the + # crawl delay and request rate only apply to the user agents + # listed directly before them. state = 0 entry = Entry() + group = [entry] self.modified() for line in lines: @@ -126,8 +142,12 @@ def parse(self, lines): line[1] = line[1].strip() if line[0] == "user-agent": if state == 2: - self._add_entry(entry) + self._add_group(group) + entry = Entry() + group = [entry] + elif state == 3: entry = Entry() + group.append(entry) product_token = line[1] entry.useragents.append(product_token) state = 1 @@ -152,7 +172,8 @@ def parse(self, lines): # it will crash if line[1].strip().isdecimal(): entry.delay = int(line[1]) - state = 2 + if state == 1: + state = 3 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') @@ -160,7 +181,8 @@ def parse(self, lines): if (len(numbers) == 2 and numbers[0].strip().isdecimal() and numbers[1].strip().isdecimal()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) - state = 2 + if state == 1: + state = 3 elif line[0] == "sitemap": # According to http://www.sitemaps.org/protocol.html # "This directive is independent of the user-agent line, @@ -168,7 +190,7 @@ def parse(self, lines): # Therefore we do not change the state of the parser. self.sitemaps.append(line[1]) if state != 0: - self._add_entry(entry) + self._add_group(group) def _find_entry(self, useragent): entry = self.groups.get(useragent.lower()) diff --git a/Misc/NEWS.d/next/Library/2026-11-19-23-54-36.gh-issue-162263.Q_m9bH.rst b/Misc/NEWS.d/next/Library/2026-11-19-23-54-36.gh-issue-162263.Q_m9bH.rst new file mode 100644 index 00000000000000..6721e02491d394 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-19-23-54-36.gh-issue-162263.Q_m9bH.rst @@ -0,0 +1,4 @@ +Fix :mod:`urllib.robotparser` ending a group at a ``Crawl-delay`` or +``Request-rate`` line. The rules of the group now apply to all of its +user agents, as :rfc:`9309` requires, while the crawl delay and request +rate still apply only to the user agents listed directly before them.