diff --git a/Lib/robotparser.py b/Lib/robotparser.py index 447563fe654..726854b49f5 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -68,7 +68,9 @@ class RobotFileParser: def _add_entry(self, entry): if "*" in entry.useragents: # the default entry is considered last - self.default_entry = entry + if self.default_entry is None: + # the first default entry wins + self.default_entry = entry else: self.entries.append(entry) @@ -120,7 +122,7 @@ class RobotFileParser: entry.rulelines.append(RuleLine(line[1], True)) state = 2 if state == 2: - self.entries.append(entry) + self._add_entry(entry) def can_fetch(self, useragent, url): diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 431b8ffbd98..650b603fdb5 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -202,6 +202,20 @@ bad = ['/folder1/anotherfile.html'] RobotTest(13, doc, good, bad, agent="googlebot") +# 14. For issue #4108 (obey first * entry) +doc = """ +User-agent: * +Disallow: /some/path + +User-agent: * +Disallow: /another/path +""" + +good = ['/another/path'] +bad = ['/some/path'] + +RobotTest(14, doc, good, bad) + class TestCase(unittest.TestCase): def runTest(self): diff --git a/Misc/NEWS b/Misc/NEWS index b0e66a39b26..37b143b51b1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -35,6 +35,9 @@ Core and Builtins when turned into an exception: in this case the exception simply gets ignored. +- Issue #4108: In urllib.robotparser, if there are multiple 'User-agent: *' + entries, consider the first one. + - Issue #9354: Provide getsockopt() in asyncore's file_wrapper. - In the unicode/str.format(), raise a ValueError when indexes to arguments are