#4108: the first default entry (User-agent: *) wins.

This commit is contained in:
Georg Brandl 2010-07-29 17:55:01 +00:00
parent 70120e202d
commit 0a0fc07d37
3 changed files with 21 additions and 2 deletions

View file

@ -66,7 +66,9 @@ class RobotFileParser:
def _add_entry(self, entry):
if "*" in entry.useragents:
# the default entry is considered last
self.default_entry = entry
if self.default_entry is None:
# the first default entry wins
self.default_entry = entry
else:
self.entries.append(entry)
@ -118,7 +120,7 @@ class RobotFileParser:
entry.rulelines.append(RuleLine(line[1], True))
state = 2
if state == 2:
self.entries.append(entry)
self._add_entry(entry)
def can_fetch(self, useragent, url):