mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 08:59:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

bpo-35922: Fix RobotFileParser when robots.txt has no relevant crawl delay or request rate (GH-11791)

Browse source 
Co-Authored-By: Tal Einat <taleinat+github@gmail.com>
This commit is contained in:
Rémi Lapeyre 2019-06-16 08:48:57 +02:00 • committed by Tal Einat
parent 3a1d50e7e5
commit 8047e0e1c6
3 changed files with 26 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Lib/urllib/robotparser.py
View file

@ -186,7 +186,9 @@ class RobotFileParser:
for entry in self.entries:
if entry.applies_to(useragent):
return entry.delay
return self.default_entry.delay
if self.default_entry:
return self.default_entry.delay
return None
def request_rate(self, useragent):
if not self.mtime():
@ -194,7 +196,9 @@ class RobotFileParser:
for entry in self.entries:
if entry.applies_to(useragent):
return entry.req_rate
return self.default_entry.req_rate
if self.default_entry:
return self.default_entry.req_rate
return None
def site_maps(self):
if not self.sitemaps: