Merged upstream changes.

Lib/test/test_robotparser.py

@@ -10,84 +10,49 @@ except ImportError:
     threading = None
 
 
-class RobotTestCase(unittest.TestCase):
+class BaseRobotTest:
-    def __init__(self, index=None, parser=None, url=None, good=None,
+    robots_txt = ''
-                 agent=None, request_rate=None, crawl_delay=None):
+    agent = 'test_robotparser'
-        # workaround to make unittest discovery work (see #17066)
+    good = []
-        if not isinstance(index, int):
+    bad = []
-            return
-        unittest.TestCase.__init__(self)
-        if good:
-            self.str = "RobotTest(%d, good, %s)" % (index, url)
-        else:
-            self.str = "RobotTest(%d, bad, %s)" % (index, url)
-        self.parser = parser
-        self.url = url
-        self.good = good
-        self.agent = agent
-        self.request_rate = request_rate
-        self.crawl_delay = crawl_delay
 
-    def runTest(self):
+    def setUp(self):
-        if isinstance(self.url, tuple):
+        lines = io.StringIO(self.robots_txt).readlines()
-            agent, url = self.url
+        self.parser = urllib.robotparser.RobotFileParser()
-        else:
+        self.parser.parse(lines)
-            url = self.url
+
-            agent = self.agent
+    def get_agent_and_url(self, url):
-        if self.good:
+        if isinstance(url, tuple):
+            agent, url = url
+            return agent, url
+        return self.agent, url
+
+    def test_good_urls(self):
+        for url in self.good:
+            agent, url = self.get_agent_and_url(url)
+            with self.subTest(url=url, agent=agent):
                 self.assertTrue(self.parser.can_fetch(agent, url))
-            self.assertEqual(self.parser.crawl_delay(agent), self.crawl_delay)
+
-            # if we have actual values for request rate
+    def test_bad_urls(self):
-            if self.request_rate and self.parser.request_rate(agent):
+        for url in self.bad:
-                self.assertEqual(
+            agent, url = self.get_agent_and_url(url)
-                    self.parser.request_rate(agent).requests,
+            with self.subTest(url=url, agent=agent):
-                    self.request_rate.requests
-                )
-                self.assertEqual(
-                    self.parser.request_rate(agent).seconds,
-                    self.request_rate.seconds
-                )
-            self.assertEqual(self.parser.request_rate(agent), self.request_rate)
-        else:
                 self.assertFalse(self.parser.can_fetch(agent, url))
 
-    def __str__(self):
-        return self.str
 
-tests = unittest.TestSuite()
+class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase):
-
+    robots_txt = """\
-def RobotTest(index, robots_txt, good_urls, bad_urls,
-              request_rate, crawl_delay, agent="test_robotparser"):
-
-    lines = io.StringIO(robots_txt).readlines()
-    parser = urllib.robotparser.RobotFileParser()
-    parser.parse(lines)
-    for url in good_urls:
-        tests.addTest(RobotTestCase(index, parser, url, 1, agent,
-                      request_rate, crawl_delay))
-    for url in bad_urls:
-        tests.addTest(RobotTestCase(index, parser, url, 0, agent,
-                      request_rate, crawl_delay))
-
-# Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002)
-
-# 1.
-doc = """
 User-agent: *
 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
 Disallow: /tmp/ # these will soon disappear
 Disallow: /foo.html
     """
-
     good = ['/', '/test.html']
     bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html']
-request_rate = None
-crawl_delay = None
 
-RobotTest(1, doc, good, bad, request_rate, crawl_delay)
 
-# 2.
+class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase):
-doc = """
+    robots_txt = """\
 # robots.txt for http://www.example.com/
 
 User-agent: *
@@ -98,34 +63,23 @@ Disallow: /cyberworld/map/ # This is an infinite virtual URL space
 # Cybermapper knows where to go.
 User-agent: cybermapper
 Disallow:
-
     """
-
     good = ['/', '/test.html', ('cybermapper', '/cyberworld/map/index.html')]
     bad = ['/cyberworld/map/index.html']
-request_rate = None  # The parameters should be equal to None since they
-crawl_delay = None   # don't apply to the cybermapper user agent
 
-RobotTest(2, doc, good, bad, request_rate, crawl_delay)
 
-# 3.
+class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase):
-doc = """
+    robots_txt = """\
 # go away
 User-agent: *
 Disallow: /
     """
-
     good = []
     bad = ['/cyberworld/map/index.html', '/', '/tmp/']
-request_rate = None
-crawl_delay = None
 
-RobotTest(3, doc, good, bad, request_rate, crawl_delay)
 
-# Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002)
+class CrawlDelayAndRequestRateTest(BaseRobotTest, unittest.TestCase):
-
+    robots_txt = """\
-# 4.
-doc = """
 User-agent: figtree
 Crawl-delay: 3
 Request-rate: 9/30
@@ -134,27 +88,42 @@ Disallow: /a%3cd.html
 Disallow: /a%2fb.html
 Disallow: /%7ejoe/index.html
     """
-
+    agent = 'figtree'
-good = [] # XFAIL '/a/b.html'
+    request_rate = namedtuple('req_rate', 'requests seconds')(9, 30)
-bad = ['/tmp','/tmp.html','/tmp/a.html',
-       '/a%3cd.html','/a%3Cd.html','/a%2fb.html',
-       '/~joe/index.html'
-       ]
-
-request_rate = namedtuple('req_rate', 'requests seconds')
-request_rate.requests = 9
-request_rate.seconds = 30
     crawl_delay = 3
-request_rate_bad = None  # not actually tested, but we still need to parse it
+    good = [('figtree', '/foo.html')]
-crawl_delay_bad = None  # in order to accommodate the input parameters
+    bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html',
+           '/a%2fb.html', '/~joe/index.html']
+
+    def test_request_rate(self):
+        for url in self.good:
+            agent, url = self.get_agent_and_url(url)
+            with self.subTest(url=url, agent=agent):
+                if self.crawl_delay:
+                    self.assertEqual(
+                        self.parser.crawl_delay(agent), self.crawl_delay
+                    )
+                if self.request_rate and self.parser.request_rate(agent):
+                    self.assertEqual(
+                        self.parser.request_rate(agent).requests,
+                        self.request_rate.requests
+                    )
+                    self.assertEqual(
+                        self.parser.request_rate(agent).seconds,
+                        self.request_rate.seconds
+                    )
 
 
-RobotTest(4, doc, good, bad, request_rate, crawl_delay, 'figtree' )
+class DifferentAgentTest(CrawlDelayAndRequestRateTest):
-RobotTest(5, doc, good, bad, request_rate_bad, crawl_delay_bad,
+    agent = 'FigTree Robot libwww-perl/5.04'
-          'FigTree Robot libwww-perl/5.04')
+    # these are not actually tested, but we still need to parse it
+    # in order to accommodate the input parameters
+    request_rate = None
+    crawl_delay = None
 
-# 6.
+
-doc = """
+class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
+    robots_txt = """\
 User-agent: *
 Disallow: /tmp/
 Disallow: /a%3Cd.html
@@ -163,141 +132,102 @@ Disallow: /%7ejoe/index.html
 Crawl-delay: 3
 Request-rate: 9/banana
     """
-
+    good = ['/tmp']
-good = ['/tmp',] # XFAIL: '/a%2fb.html'
+    bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html',
-bad = ['/tmp/','/tmp/a.html',
-       '/a%3cd.html','/a%3Cd.html',"/a/b.html",
            '/%7Ejoe/index.html']
     crawl_delay = 3
-request_rate = None  # since request rate has invalid syntax, return None
 
-RobotTest(6, doc, good, bad, None, None)
 
+class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase):
     # From bug report #523041
-
+    robots_txt = """\
-# 7.
-doc = """
 User-Agent: *
 Disallow: /.
 Crawl-delay: pears
     """
-
     good = ['/foo.html']
-bad = []  # bug report says "/" should be denied, but that is not in the RFC
+    # bug report says "/" should be denied, but that is not in the RFC
+    bad = []
 
-crawl_delay = None  # since crawl delay has invalid syntax, return None
-request_rate = None
 
-RobotTest(7, doc, good, bad, crawl_delay, request_rate)
+class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
-
+    # also test that Allow and Diasallow works well with each other
-# From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364
+    robots_txt = """\
-
-# 8.
-doc = """
 User-agent: Googlebot
 Allow: /folder1/myfile.html
 Disallow: /folder1/
 Request-rate: whale/banana
     """
-
+    agent = 'Googlebot'
     good = ['/folder1/myfile.html']
     bad = ['/folder1/anotherfile.html']
-crawl_delay = None
-request_rate = None  # invalid syntax, return none
 
-RobotTest(8, doc, good, bad, crawl_delay, request_rate, agent="Googlebot")
 
-# 9.  This file is incorrect because "Googlebot" is a substring of
+class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase):
-#     "Googlebot-Mobile", so test 10 works just like test 9.
+    # the order of User-agent should be correct. note
-doc = """
+    # that this file is incorrect because "Googlebot" is a
+    # substring of "Googlebot-Mobile"
+    robots_txt = """\
 User-agent: Googlebot
 Disallow: /
 
 User-agent: Googlebot-Mobile
 Allow: /
     """
-
+    agent = 'Googlebot'
-good = []
     bad = ['/something.jpg']
 
-RobotTest(9, doc, good, bad, None, None, agent="Googlebot")
 
-good = []
+class UserAgentGoogleMobileTest(UserAgentOrderingTest):
-bad = ['/something.jpg']
+    agent = 'Googlebot-Mobile'
-
-RobotTest(10, doc, good, bad, None, None, agent="Googlebot-Mobile")
-
-# 11.  Get the order correct.
-doc = """
-User-agent: Googlebot-Mobile
-Allow: /
-
-User-agent: Googlebot
-Disallow: /
-"""
-
-good = []
-bad = ['/something.jpg']
-
-RobotTest(11, doc, good, bad, None, None, agent="Googlebot")
-
-good = ['/something.jpg']
-bad = []
-
-RobotTest(12, doc, good, bad, None, None, agent="Googlebot-Mobile")
 
 
-# 13.  Google also got the order wrong in #8.  You need to specify the
+class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase):
-#      URLs from more specific to more general.
+    # Google also got the order wrong. You need
-doc = """
+    # to specify the URLs from more specific to more general
+    robots_txt = """\
 User-agent: Googlebot
 Allow: /folder1/myfile.html
 Disallow: /folder1/
     """
-
+    agent = 'googlebot'
     good = ['/folder1/myfile.html']
     bad = ['/folder1/anotherfile.html']
 
-RobotTest(13, doc, good, bad, None, None, agent="googlebot")
 
-
+class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase):
-# 14. For issue #6325 (query string support)
+    # see issue #6325 for details
-doc = """
+    robots_txt = """\
 User-agent: *
 Disallow: /some/path?name=value
     """
-
     good = ['/some/path']
     bad = ['/some/path?name=value']
 
-RobotTest(14, doc, good, bad, None, None)
 
-# 15. For issue #4108 (obey first * entry)
+class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase):
-doc = """
+    # obey first * entry (#4108)
+    robots_txt = """\
 User-agent: *
 Disallow: /some/path
 
 User-agent: *
 Disallow: /another/path
     """
-
     good = ['/another/path']
     bad = ['/some/path']
 
-RobotTest(15, doc, good, bad, None, None)
 
-# 16. Empty query (issue #17403). Normalizing the url first.
+class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase):
-doc = """
+    # normalize the URL first (#17403)
+    robots_txt = """\
 User-agent: *
 Allow: /some/path?
 Disallow: /another/path?
     """
-
     good = ['/some/path?']
     bad = ['/another/path?']
 
-RobotTest(16, doc, good, bad, None, None)
-
 
 class RobotHandler(BaseHTTPRequestHandler):
 
@@ -329,9 +259,6 @@ class PasswordProtectedSiteTestCase(unittest.TestCase):
         self.t.join()
         self.server.server_close()
 
-    def runTest(self):
-        self.testPasswordProtectedSite()
-
     def testPasswordProtectedSite(self):
         addr = self.server.server_address
         url = 'http://' + support.HOST + ':' + str(addr[1])
@@ -341,8 +268,6 @@ class PasswordProtectedSiteTestCase(unittest.TestCase):
         parser.read()
         self.assertFalse(parser.can_fetch("*", robots_url))
 
-    def __str__(self):
-        return '%s' % self.__class__.__name__
 
 class NetworkTestCase(unittest.TestCase):
 
@@ -356,11 +281,5 @@ class NetworkTestCase(unittest.TestCase):
             self.assertTrue(
                 parser.can_fetch("*", "http://www.python.org/robots.txt"))
 
-def load_tests(loader, suite, pattern):
-    suite = unittest.makeSuite(NetworkTestCase)
-    suite.addTest(tests)
-    suite.addTest(PasswordProtectedSiteTestCase())
-    return suite
-
 if __name__=='__main__':
     unittest.main()