diff -r 50d93fe73726 Doc/library/urllib.robotparser.rst --- a/Doc/library/urllib.robotparser.rst Tue May 27 03:31:32 2014 -0400 +++ b/Doc/library/urllib.robotparser.rst Tue May 27 10:20:14 2014 +0100 @@ -52,6 +52,23 @@ Sets the time the ``robots.txt`` file was last fetched to the current time. + + .. method:: crawl_delay(useragent) + + Returns the value of the ``Crawl-delay`` parameter from ``robots.txt`` + for the *useragent* in question. If there is no such parameter or it + doesn't apply to the *useragent* specified or the ``robots.txt`` entry + for this parameter has invalid syntax, return ``None`` + .. versionadded:: 3.5 + + .. method:: request_rate(useragent) + + Returns the contents of the ``Request-rate`` parameter from ``robots.txt`` + in the form of a :func:`~collections.namedtuple` ``(requests, seconds)``. + If there is no such parameter or it doesn't apply to the *useragent* + specified or the ``robots.txt`` entry for this parameter has invalid + syntax, return ``None`` + .. versionadded:: 3.5 The following example demonstrates basic use of the RobotFileParser class. @@ -60,8 +77,14 @@ >>> rp = urllib.robotparser.RobotFileParser() >>> rp.set_url("http://www.musi-cal.com/robots.txt") >>> rp.read() + >>> rrate = rp.request_rate("*") + >>> rrate.requests + 3 + >>> rrate.seconds + 20 + >>> rp.crawl_delay("*") + 6 >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") False >>> rp.can_fetch("*", "http://www.musi-cal.com/") True - diff -r 50d93fe73726 Lib/test/test_robotparser.py --- a/Lib/test/test_robotparser.py Tue May 27 03:31:32 2014 -0400 +++ b/Lib/test/test_robotparser.py Tue May 27 10:20:14 2014 +0100 @@ -4,9 +4,11 @@ from urllib.error import URLError, HTTPError from urllib.request import urlopen from test import support +from collections import namedtuple class RobotTestCase(unittest.TestCase): - def __init__(self, index=None, parser=None, url=None, good=None, agent=None): + def __init__(self, index=None, parser=None, url=None, good=None, agent=None, + request_rate=None, crawl_delay=None): # workaround to make unittest discovery work (see #17066) if not isinstance(index, int): return @@ -19,6 +21,8 @@ self.url = url self.good = good self.agent = agent + self.request_rate = request_rate + self.crawl_delay = crawl_delay def runTest(self): if isinstance(self.url, tuple): @@ -28,6 +32,19 @@ agent = self.agent if self.good: self.assertTrue(self.parser.can_fetch(agent, url)) + self.assertEqual(self.parser.crawl_delay(agent), self.crawl_delay) + # If we have actual values for request rate + if self.request_rate and self.parser.request_rate(agent): + self.assertEqual(self.parser.request_rate(agent).requests, + self.request_rate.requests) + self.assertEqual(self.parser.request_rate(agent).seconds, + self.request_rate.seconds) + else: + self.assertEqual(self.parser.request_rate(agent), self.request_rate) + # As the good tests and the bad tests sometimes use different + # user agent strings you have to choose only one of them to + # test crawl-delay and request-rate since their output depends + # on the user agent string. else: self.assertFalse(self.parser.can_fetch(agent, url)) @@ -37,15 +54,17 @@ tests = unittest.TestSuite() def RobotTest(index, robots_txt, good_urls, bad_urls, - agent="test_robotparser"): + request_rate, crawl_delay, agent="test_robotparser"): lines = io.StringIO(robots_txt).readlines() parser = urllib.robotparser.RobotFileParser() parser.parse(lines) for url in good_urls: - tests.addTest(RobotTestCase(index, parser, url, 1, agent)) + tests.addTest(RobotTestCase(index, parser, url, 1, agent, + request_rate, crawl_delay)) for url in bad_urls: - tests.addTest(RobotTestCase(index, parser, url, 0, agent)) + tests.addTest(RobotTestCase(index, parser, url, 0, agent, + request_rate, crawl_delay)) # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002) @@ -59,14 +78,18 @@ good = ['/','/test.html'] bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html'] +request_rate = None +crawl_delay = None -RobotTest(1, doc, good, bad) +RobotTest(1, doc, good, bad, request_rate, crawl_delay) # 2. doc = """ # robots.txt for http://www.example.com/ User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. @@ -77,8 +100,10 @@ good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')] bad = ['/cyberworld/map/index.html'] +request_rate = None # The parameters should be equal to None since they +crawl_delay = None # don't apply to the cybermapper user agent -RobotTest(2, doc, good, bad) +RobotTest(2, doc, good, bad, request_rate, crawl_delay) # 3. doc = """ @@ -89,14 +114,18 @@ good = [] bad = ['/cyberworld/map/index.html','/','/tmp/'] +request_rate = None +crawl_delay = None -RobotTest(3, doc, good, bad) +RobotTest(3, doc, good, bad, request_rate, crawl_delay) # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002) # 4. doc = """ User-agent: figtree +Crawl-delay: 3 +Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html @@ -109,8 +138,17 @@ '/~joe/index.html' ] -RobotTest(4, doc, good, bad, 'figtree') -RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04') +request_rate = namedtuple('req_rate', 'requests seconds') +request_rate.requests = 9 +request_rate.seconds = 30 +crawl_delay = 3 +request_rate_bad = None # Not actually tested, but we still need to parse it +crawl_delay_bad = None # In order to accomodate the input parameters + + +RobotTest(4, doc, good, bad, request_rate, crawl_delay, 'figtree' ) +RobotTest(5, doc, good, bad, request_rate_bad, crawl_delay_bad, + 'FigTree Robot libwww-perl/5.04') # 6. doc = """ @@ -119,14 +157,18 @@ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html +Crawl-delay: 3 +Request-rate: 9/banana """ good = ['/tmp',] # XFAIL: '/a%2fb.html' bad = ['/tmp/','/tmp/a.html', '/a%3cd.html','/a%3Cd.html',"/a/b.html", '/%7Ejoe/index.html'] +crawl_delay = 3 +request_rate = None # Since request rate has invalid sytax, return None -RobotTest(6, doc, good, bad) +RobotTest(6, doc, good, bad, None, None) # From bug report #523041 @@ -134,12 +176,16 @@ doc = """ User-Agent: * Disallow: /. +Crawl-delay: pears """ good = ['/foo.html'] -bad = [] # Bug report says "/" should be denied, but that is not in the RFC +bad = [] # Bug report says "/" should be denied, but that is not in the RFC -RobotTest(7, doc, good, bad) +crawl_delay = None # Since crawl delay has invalid sytax, return None +request_rate = None + +RobotTest(7, doc, good, bad, crawl_delay, request_rate) # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364 @@ -148,12 +194,15 @@ User-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ +Request-rate: whale/banana """ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] +crawl_delay = None +request_rate = None # Invalid sytax, return none -RobotTest(8, doc, good, bad, agent="Googlebot") +RobotTest(8, doc, good, bad, crawl_delay, request_rate, agent="Googlebot") # 9. This file is incorrect because "Googlebot" is a substring of # "Googlebot-Mobile", so test 10 works just like test 9. @@ -168,12 +217,12 @@ good = [] bad = ['/something.jpg'] -RobotTest(9, doc, good, bad, agent="Googlebot") +RobotTest(9, doc, good, bad, None, None, agent="Googlebot") good = [] bad = ['/something.jpg'] -RobotTest(10, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(10, doc, good, bad, None, None, agent="Googlebot-Mobile") # 11. Get the order correct. doc = """ @@ -187,12 +236,12 @@ good = [] bad = ['/something.jpg'] -RobotTest(11, doc, good, bad, agent="Googlebot") +RobotTest(11, doc, good, bad, None, None, agent="Googlebot") good = ['/something.jpg'] bad = [] -RobotTest(12, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(12, doc, good, bad, None, None, agent="Googlebot-Mobile") # 13. Google also got the order wrong in #8. You need to specify the @@ -206,7 +255,7 @@ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] -RobotTest(13, doc, good, bad, agent="googlebot") +RobotTest(13, doc, good, bad, None, None, agent="googlebot") # 14. For issue #6325 (query string support) @@ -218,7 +267,7 @@ good = ['/some/path'] bad = ['/some/path?name=value'] -RobotTest(14, doc, good, bad) +RobotTest(14, doc, good, bad, None, None) # 15. For issue #4108 (obey first * entry) doc = """ @@ -232,7 +281,7 @@ good = ['/another/path'] bad = ['/some/path'] -RobotTest(15, doc, good, bad) +RobotTest(15, doc, good, bad, None, None) # 16. Empty query (issue #17403). Normalizing the url first. doc = """ @@ -244,7 +293,7 @@ good = ['/some/path?'] bad = ['/another/path?'] -RobotTest(16, doc, good, bad) +RobotTest(16, doc, good, bad, None, None) class NetworkTestCase(unittest.TestCase): diff -r 50d93fe73726 Lib/urllib/robotparser.py --- a/Lib/urllib/robotparser.py Tue May 27 03:31:32 2014 -0400 +++ b/Lib/urllib/robotparser.py Tue May 27 10:20:14 2014 +0100 @@ -11,6 +11,7 @@ """ import urllib.parse, urllib.request +from collections import namedtuple __all__ = ["RobotFileParser"] @@ -120,6 +121,23 @@ if state != 0: entry.rulelines.append(RuleLine(line[1], True)) state = 2 + elif line[0] == "crawl-delay": + if state != 0: + # Before trying to convert to int we need to make sure + # that robots.txt has valid syntax otherwise it will crash. + if line[1].strip().isdigit(): + entry.delay = int(line[1]) + state = 2 + elif line[0] == "request-rate": + if state != 0: + numbers = line[1].split('/') + # Check if all values are sane + if len(numbers) == 2 and numbers[0].strip().isdigit()\ + and numbers[1].strip().isdigit(): + entry.req_rate = namedtuple('req_rate', ['requests', 'seconds']) + entry.req_rate.requests = int(numbers[0]) + entry.req_rate.seconds = int(numbers[1]) + state = 2 if state == 2: self._add_entry(entry) @@ -153,6 +171,18 @@ # agent not found ==> access granted return True + def crawl_delay(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.delay # current user agent + return None # If there is no crawl delay defined + + def request_rate(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.req_rate + return None # If there is no request_rate defined + def __str__(self): return ''.join([str(entry) + "\n" for entry in self.entries]) @@ -180,6 +210,11 @@ def __init__(self): self.useragents = [] self.rulelines = [] + # Support for request rate and crawl delay here. Default values + # `None` are returned if they don't apply to the current user agent + # or are malformed/missing in the robots.txt + self.delay = None + self.req_rate = None def __str__(self): ret = []