Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Correctly set default entry in all cases.

This commit is contained in:
Martin v. Löwis 2002-03-18 10:43:18 +00:00
parent d22368ffb3
commit 73f570ba08
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Lib/robotparser.py
View File

@ -58,6 +58,13 @@ class RobotFileParser:
_debug("parse lines")
self.parse(lines)
def _add_entry(self, entry):
if "*" in entry.useragents:
# the default entry is considered last
self.default_entry = entry
else:
self.entries.append(entry)
def parse(self, lines):
"""parse the input lines from a robot.txt file.
We allow that a user-agent: line is not preceded by
@ -76,11 +83,7 @@ class RobotFileParser:
entry = Entry()
state = 0
elif state==2:
if "*" in entry.useragents:
# the default entry is considered last
self.default_entry = entry
else:
self.entries.append(entry)
self._add_entry(entry)
entry = Entry()
state = 0
# remove optional comment and strip line
@ -99,7 +102,7 @@ class RobotFileParser:
_debug("line %d: warning: you should insert a blank"
" line before any user-agent"
" directive" % linenumber)
self.entries.append(entry)
self._add_entry(entry)
entry = Entry()
entry.useragents.append(line[1])
state = 1