diff -r 73aa6d672b81 Lib/test/test_robotparser.py --- a/Lib/test/test_robotparser.py Sun Feb 23 18:01:08 2014 -0500 +++ b/Lib/test/test_robotparser.py Mon Feb 24 17:45:19 2014 +0800 @@ -4,6 +4,9 @@ from urllib.error import URLError, HTTPError from urllib.request import urlopen from test import support +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + class RobotTestCase(unittest.TestCase): def __init__(self, index=None, parser=None, url=None, good=None, agent=None): @@ -247,33 +250,51 @@ RobotTest(16, doc, good, bad) -class NetworkTestCase(unittest.TestCase): +class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_error(403, "Forbidden access") + + def log_message(self, format, *args): + pass + + +class PasswordProtectedSiteTestCase(unittest.TestCase): + + def setUp(self): + self.server = HTTPServer((support.HOST, 0), RobotHandler) + + self.t = threading.Thread( + name='HTTPServer serving', + target=self.server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval':0.01}) + self.t.daemon = True # In case this function raises. + self.t.start() + + def tearDown(self): + self.server.shutdown() + self.t.join() + self.server.server_close() + + def runTest(self): + self.testPasswordProtectedSite() def testPasswordProtectedSite(self): - support.requires('network') - with support.transient_internet('mueblesmoraleda.com'): - url = 'http://mueblesmoraleda.com' - robots_url = url + "/robots.txt" - # First check the URL is usable for our purposes, since the - # test site is a bit flaky. - try: - urlopen(robots_url) - except HTTPError as e: - if e.code not in {401, 403}: - self.skipTest( - "%r should return a 401 or 403 HTTP error, not %r" - % (robots_url, e.code)) - else: - self.skipTest( - "%r should return a 401 or 403 HTTP error, not succeed" - % (robots_url)) - parser = urllib.robotparser.RobotFileParser() - parser.set_url(url) - try: - parser.read() - except URLError: - self.skipTest('%s is unavailable' % url) - self.assertEqual(parser.can_fetch("*", robots_url), False) + addr = self.server.server_address + url = 'http://' + support.HOST + ':' + str(addr[1]) + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + +tests.addTest(PasswordProtectedSiteTestCase()) + + +class NetworkTestCase(unittest.TestCase): def testPythonOrg(self): support.requires('network')