mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-15 11:22:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

Correctly set default entry in all cases.

Browse source
This commit is contained in:
Martin v. Löwis 2002-03-18 10:43:18 +00:00
parent d22368ffb3
commit 73f570ba08
1 changed files with 9 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

15
Lib/robotparser.py
View file

@ -58,6 +58,13 @@ class RobotFileParser:
_debug("parse lines") _debug("parse lines")
self.parse(lines) self.parse(lines)
def _add_entry(self, entry):
if "*" in entry.useragents:
# the default entry is considered last
self.default_entry = entry
else:
self.entries.append(entry)
def parse(self, lines): def parse(self, lines):
"""parse the input lines from a robot.txt file. """parse the input lines from a robot.txt file.
We allow that a user-agent: line is not preceded by We allow that a user-agent: line is not preceded by
@ -76,11 +83,7 @@ class RobotFileParser:
entry = Entry() entry = Entry()
state = 0 state = 0
elif state==2: elif state==2:
if "*" in entry.useragents: self._add_entry(entry)
# the default entry is considered last
self.default_entry = entry
else:
self.entries.append(entry)
entry = Entry() entry = Entry()
state = 0 state = 0
# remove optional comment and strip line # remove optional comment and strip line
@ -99,7 +102,7 @@ class RobotFileParser:
_debug("line %d: warning: you should insert a blank" _debug("line %d: warning: you should insert a blank"
" line before any user-agent" " line before any user-agent"
" directive" % linenumber) " directive" % linenumber)
self.entries.append(entry) self._add_entry(entry)
entry = Entry() entry = Entry()
entry.useragents.append(line[1]) entry.useragents.append(line[1])
state = 1 state = 1