mirror of
https://github.com/python/cpython.git
synced 2025-07-23 11:15:24 +00:00
#4108: the first default entry (User-agent: *) wins.
This commit is contained in:
parent
70120e202d
commit
0a0fc07d37
3 changed files with 21 additions and 2 deletions
|
@ -66,7 +66,9 @@ class RobotFileParser:
|
|||
def _add_entry(self, entry):
|
||||
if "*" in entry.useragents:
|
||||
# the default entry is considered last
|
||||
self.default_entry = entry
|
||||
if self.default_entry is None:
|
||||
# the first default entry wins
|
||||
self.default_entry = entry
|
||||
else:
|
||||
self.entries.append(entry)
|
||||
|
||||
|
@ -118,7 +120,7 @@ class RobotFileParser:
|
|||
entry.rulelines.append(RuleLine(line[1], True))
|
||||
state = 2
|
||||
if state == 2:
|
||||
self.entries.append(entry)
|
||||
self._add_entry(entry)
|
||||
|
||||
|
||||
def can_fetch(self, useragent, url):
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue