mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-18 07:41:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merged revisions 83449 via svnmerge from

Browse source 
svn+ssh://pythondev@svn.python.org/python/branches/release27-maint

................
  r83449 | georg.brandl | 2010-08-01 22:59:03 +0200 (So, 01 Aug 2010) | 9 lines

  Merged revisions 83238 via svnmerge from
  svn+ssh://pythondev@svn.python.org/python/branches/py3k

  ........
    r83238 | georg.brandl | 2010-07-29 19:55:01 +0200 (Do, 29 Jul 2010) | 1 line

    #4108: the first default entry (User-agent: *) wins.
  ........
................
This commit is contained in:
Georg Brandl 2010-08-01 22:00:39 +00:00
parent 86edb14057
commit 0ba1f01adf
3 changed files with 21 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Lib/robotparser.py
View file

@ -68,7 +68,9 @@ class RobotFileParser:
def _add_entry(self, entry):
if "*" in entry.useragents:
# the default entry is considered last
self.default_entry = entry
if self.default_entry is None:
# the first default entry wins
self.default_entry = entry
else:
self.entries.append(entry)
@ -120,7 +122,7 @@ class RobotFileParser:
entry.rulelines.append(RuleLine(line[1], True))
state = 2
if state == 2:
self.entries.append(entry)
self._add_entry(entry)
def can_fetch(self, useragent, url):