mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-24 11:44:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-32861: urllib.robotparser fix incomplete __str__ methods. (GH-5711)

Browse source 
The urllib.robotparser's __str__ representation now includes wildcard
entries and the "Crawl-delay" and "Request-rate" fields. Also removes extra
newlines that were being appended to the end of the string.
This commit is contained in:
Michael Lazar 2018-05-14 10:10:41 -04:00 • committed by Serhiy Storchaka
parent 5c0d462689
commit bd08a0af2d
3 changed files with 42 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

17
Lib/urllib/robotparser.py
View file

@ -190,7 +190,10 @@ class RobotFileParser:
return self.default_entry.req_rate
def __str__(self):
return ''.join([str(entry) + "\n" for entry in self.entries])
entries = self.entries
if self.default_entry is not None:
entries = entries + [self.default_entry]
return '\n\n'.join(map(str, entries))
class RuleLine:
@ -222,10 +225,14 @@ class Entry:
def __str__(self):
ret = []
for agent in self.useragents:
ret.extend(["User-agent: ", agent, "\n"])
for line in self.rulelines:
ret.extend([str(line), "\n"])
return ''.join(ret)
ret.append(f"User-agent: {agent}")
if self.delay is not None:
ret.append(f"Crawl-delay: {self.delay}")
if self.req_rate is not None:
rate = self.req_rate
ret.append(f"Request-rate: {rate.requests}/{rate.seconds}")
ret.extend(map(str, self.rulelines))
return '\n'.join(ret)
def applies_to(self, useragent):
"""check if this entry applies to the specified agent"""