diff -r de982d8b7b15 Lib/urllib/robotparser.py --- a/Lib/urllib/robotparser.py Tue Oct 13 21:26:35 2015 +0300 +++ b/Lib/urllib/robotparser.py Tue Oct 13 17:41:20 2015 -0700 @@ -178,13 +178,19 @@ for entry in self.entries: if entry.applies_to(useragent): return entry.delay - return None + # Check to see if robots.txt has been read, if not, return None + if not self.last_checked: + return None + return self.default_entry.delay def request_rate(self, useragent): for entry in self.entries: if entry.applies_to(useragent): return entry.req_rate - return None + # Check to see if robots.txt has been read, if not, return None + if not self.last_checked: + return None + return self.default_entry.req_rate def __str__(self): return ''.join([str(entry) + "\n" for entry in self.entries])