mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-17 22:20:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

#1778443 robotparser fixes from Aristotelis Mikropoulos

Browse source
This commit is contained in:
Benjamin Peterson 2008-07-12 23:41:19 +00:00
parent 24122593f0
commit 0522a9f1eb
2 changed files with 4 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Lib/robotparser.py
View file

@ -55,11 +55,8 @@ class RobotFileParser:
"""Reads the robots.txt URL and feeds it to the parser.""" """Reads the robots.txt URL and feeds it to the parser."""
opener = URLopener() opener = URLopener()
f = opener.open(self.url) f = opener.open(self.url)
lines = [] lines = [line.strip() for line in f]
line = f.readline() f.close()
while line:
lines.append(line.strip())
line = f.readline()
self.errcode = opener.errcode self.errcode = opener.errcode
if self.errcode in (401, 403): if self.errcode in (401, 403):
self.disallow_all = True self.disallow_all = True
@ -84,7 +81,7 @@ class RobotFileParser:
entry = Entry() entry = Entry()
for line in lines: for line in lines:
linenumber = linenumber + 1 linenumber += 1
if not line: if not line:
if state == 1: if state == 1:
entry = Entry() entry = Entry()

1
Misc/ACKS
View file

@ -460,6 +460,7 @@ Luke Mewburn
Mike Meyer Mike Meyer
Steven Miale Steven Miale
Trent Mick Trent Mick
Aristotelis Mikropoulos
Damien Miller Damien Miller
Chad Miller Chad Miller
Jay T. Miller Jay T. Miller