mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-07 07:31:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

bpo-35922: Fix RobotFileParser when robots.txt has no relevant crawl delay or request rate (GH-11791)

Browse source 
Co-Authored-By: Tal Einat <taleinat+github@gmail.com>
This commit is contained in:
Rémi Lapeyre 2019-06-16 08:48:57 +02:00 • committed by Tal Einat
parent 3a1d50e7e5
commit 8047e0e1c6
3 changed files with 26 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

28
Lib/test/test_robotparser.py
View file

@ -97,30 +97,38 @@ Disallow: /
class BaseRequestRateTest(BaseRobotTest):
request_rate = None
crawl_delay = None
def test_request_rate(self):
parser = self.parser
for url in self.good + self.bad:
agent, url = self.get_agent_and_url(url)
with self.subTest(url=url, agent=agent):
if self.crawl_delay:
self.assertEqual(
self.parser.crawl_delay(agent), self.crawl_delay
)
if self.request_rate:
self.assertEqual(parser.crawl_delay(agent), self.crawl_delay)
parsed_request_rate = parser.request_rate(agent)
self.assertEqual(parsed_request_rate, self.request_rate)
if self.request_rate is not None:
self.assertIsInstance(
self.parser.request_rate(agent),
parsed_request_rate,
urllib.robotparser.RequestRate
)
self.assertEqual(
self.parser.request_rate(agent).requests,
parsed_request_rate.requests,
self.request_rate.requests
)
self.assertEqual(
self.parser.request_rate(agent).seconds,
parsed_request_rate.seconds,
self.request_rate.seconds
)
class EmptyFileTest(BaseRequestRateTest, unittest.TestCase):
robots_txt = ''
good = ['/foo']
class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase):
robots_txt = """\
User-agent: figtree
@ -141,10 +149,6 @@ Disallow: /%7ejoe/index.html
class DifferentAgentTest(CrawlDelayAndRequestRateTest):
agent = 'FigTree Robot libwww-perl/5.04'
# these are not actually tested, but we still need to parse it
# in order to accommodate the input parameters
request_rate = None
crawl_delay = None
class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase):