mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-19 01:05:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Bug 3347: robotparser failed because it didn't convert bytes to string.

Browse source 
The solution is to convert bytes to text via utf-8.  I'm not entirely
sure if this is safe, but it looks like robots.txt is expected to be
ascii.
This commit is contained in:
Jeremy Hylton 2008-07-18 20:59:44 +00:00
parent 48577d1944
commit 73fd46d24e
2 changed files with 18 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Lib/urllib/robotparser.py
View file

@ -60,7 +60,8 @@ class RobotFileParser:
elif err.code >= 400:
self.allow_all = True
else:
self.parse(f.read().splitlines())
raw = f.read()
self.parse(raw.decode("utf-8").splitlines())
def _add_entry(self, entry):
if "*" in entry.useragents:
@ -123,7 +124,10 @@ class RobotFileParser:
return True
# search for given user agent matches
# the first match counts
url = urllib.parse.quote(urllib.parse.urlparse(urllib.parse.unquote(url))[2]) or "/"
url = urllib.parse.quote(
urllib.parse.urlparse(urllib.parse.unquote(url))[2])
if not url:
url = "/"
for entry in self.entries:
if entry.applies_to(useragent):
return entry.allowance(url)