Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions Lib/test/test_robotparser.py
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,80 @@ class UserAgentStartsGroupTest(BaseRobotTest, unittest.TestCase):
Disallow: /another/path\
"""


class CrawlDelayInGroupTest(BaseRobotTest, unittest.TestCase):
# Crawl-delay and Request-rate do not end a group (RFC 9309,
# section 2.2.4), but only apply to the user agents directly above them.
robots_txt = """\
User-agent: spambot
Crawl-delay: 3
User-agent: eggsbot
Crawl-delay: 5
Request-rate: 9/30
User-agent: hambot
Disallow: /some/path
"""
good = [('spambot', '/'), ('eggsbot', '/'), ('hambot', '/')]
bad = [('spambot', '/some/path'), ('eggsbot', '/some/path'),
('hambot', '/some/path')]
expected_output = """\
User-agent: spambot
Crawl-delay: 3
Disallow: /some/path

User-agent: eggsbot
Crawl-delay: 5
Request-rate: 9/30
Disallow: /some/path

User-agent: hambot
Disallow: /some/path\
"""

def test_crawl_delay_and_request_rate(self):
parser = self.parser
self.assertEqual(parser.crawl_delay('spambot'), 3)
self.assertIsNone(parser.request_rate('spambot'))
self.assertEqual(parser.crawl_delay('eggsbot'), 5)
self.assertEqual(parser.request_rate('eggsbot'), (9, 30))
self.assertIsNone(parser.crawl_delay('hambot'))
self.assertIsNone(parser.request_rate('hambot'))


class InvalidCrawlDelayInGroupTest(BaseRequestRateTest, unittest.TestCase):
robots_txt = """\
User-agent: spambot
Crawl-delay: pears
User-agent: eggsbot
Request-rate: 9/banana
User-agent: hambot
Disallow: /some/path
"""
good = [('spambot', '/'), ('eggsbot', '/'), ('hambot', '/')]
bad = [('spambot', '/some/path'), ('eggsbot', '/some/path'),
('hambot', '/some/path')]


class CrawlDelayAfterRulesTest(BaseRobotTest, unittest.TestCase):
robots_txt = """\
User-agent: spambot
Disallow: /some/path
Crawl-delay: 3
User-agent: eggsbot
Disallow: /another/path
"""
good = [('spambot', '/another/path'), ('eggsbot', '/some/path')]
bad = [('spambot', '/some/path'), ('eggsbot', '/another/path')]
expected_output = """\
User-agent: spambot
Crawl-delay: 3
Disallow: /some/path

User-agent: eggsbot
Disallow: /another/path\
"""


class IgnoreEmptyLinesTest(BaseRobotTest, unittest.TestCase):
robots_txt = """\
User-agent: spambot
Expand Down
30 changes: 26 additions & 4 deletions Lib/urllib/robotparser.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@ def _add_entry(self, entry):
else:
self.groups[agent] = merge_entries(self.groups[agent], entry)

def _add_group(self, group):
# Only the last entry of a group has rules, but they apply to
# all entries in the group.
rulelines = group[-1].rulelines
for entry in group:
entry.rulelines = rulelines
self._add_entry(entry)

def parse(self, lines):
"""Parse the input lines from a robots.txt file.

Expand All @@ -108,8 +116,16 @@ def parse(self, lines):
# 0: start state
# 1: saw user-agent line
# 2: saw an allow or disallow line
# 3: saw a crawl-delay or request-rate line, but no allow or
# disallow line yet
# Crawl-delay and request-rate lines do not end a group (RFC 9309,
# section 2.2.4). A user-agent line after them starts a new entry
# in the same group: the rules apply to the whole group, but the
# crawl delay and request rate only apply to the user agents
# listed directly before them.
state = 0
entry = Entry()
group = [entry]

self.modified()
for line in lines:
Expand All @@ -126,8 +142,12 @@ def parse(self, lines):
line[1] = line[1].strip()
if line[0] == "user-agent":
if state == 2:
self._add_entry(entry)
self._add_group(group)
entry = Entry()
group = [entry]
elif state == 3:
entry = Entry()
group.append(entry)
product_token = line[1]
entry.useragents.append(product_token)
state = 1
Expand All @@ -152,23 +172,25 @@ def parse(self, lines):
# it will crash
if line[1].strip().isdecimal():
entry.delay = int(line[1])
state = 2
if state == 1:
state = 3
elif line[0] == "request-rate":
if state != 0:
numbers = line[1].split('/')
# check if all values are sane
if (len(numbers) == 2 and numbers[0].strip().isdecimal()
and numbers[1].strip().isdecimal()):
entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
state = 2
if state == 1:
state = 3
elif line[0] == "sitemap":
# According to http://www.sitemaps.org/protocol.html
# "This directive is independent of the user-agent line,
# so it doesn't matter where you place it in your file."
# Therefore we do not change the state of the parser.
self.sitemaps.append(line[1])
if state != 0:
self._add_entry(entry)
self._add_group(group)

def _find_entry(self, useragent):
entry = self.groups.get(useragent.lower())
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix :mod:`urllib.robotparser` ending a group at a ``Crawl-delay`` or
``Request-rate`` line. The rules of the group now apply to all of its
user agents, as :rfc:`9309` requires, while the crawl delay and request
rate still apply only to the user agents listed directly before them.
Loading