diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 9693cfc259286d4..6bab9b6719e4f89 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -875,5 +875,23 @@ def test_read_404(self): self.assertIsNone(parser.crawl_delay('*')) self.assertIsNone(parser.request_rate('*')) + +class RepeatedUserAgentPerformanceTest(unittest.TestCase): + """Parsing many identical ``User-agent:`` lines must stay linear.""" + + def test_many_duplicate_user_agents(self): + import time + body = ("User-agent: evil\n" * 50_000) + "Disallow: /\n" + parser = urllib.robotparser.RobotFileParser() + parser.modified() + start = time.perf_counter() + parser.parse(body.splitlines()) + elapsed = time.perf_counter() - start + self.assertLess(elapsed, 5.0, + f"parse of 50_000 duplicate User-agent lines " + f"took {elapsed:.3f}s") + self.assertFalse(parser.can_fetch("evil", "/")) + + if __name__=='__main__': unittest.main() diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 985333c71004389..844e35c5a5b20c7 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -90,8 +90,12 @@ def read(self): def _add_entry(self, entry): self.entries.append(entry) + seen = set() for agent in entry.useragents: agent = agent.lower() + if agent in seen: + continue + seen.add(agent) if agent not in self.groups: self.groups[agent] = entry else: diff --git a/Misc/NEWS.d/next/Library/2026-11-19-17-55-31.gh-issue-162282.1WW-xb.rst b/Misc/NEWS.d/next/Library/2026-11-19-17-55-31.gh-issue-162282.1WW-xb.rst new file mode 100644 index 000000000000000..0d72db13e0c2bec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-19-17-55-31.gh-issue-162282.1WW-xb.rst @@ -0,0 +1 @@ +Fix quadratic-time cost in :meth:`urllib.robotparser.RobotFileParser.parse` when a single group contains many identical ``User-agent:`` lines.