Index: Lib/robotparser.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/robotparser.py,v retrieving revision 1.17 diff -u -r1.17 robotparser.py --- Lib/robotparser.py 27 Feb 2003 20:14:40 -0000 1.17 +++ Lib/robotparser.py 7 Mar 2004 14:26:14 -0000 @@ -83,7 +83,7 @@ self.entries.append(entry) def parse(self, lines): - """parse the input lines from a robot.txt file. + """parse the input lines from a robots.txt file. We allow that a user-agent: line is not preceded by one or more blank lines.""" state = 0 @@ -148,7 +148,7 @@ def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" - _debug("Checking robot.txt allowance for:\n user agent: %s\n url: %s" % + _debug("Checking robots.txt allowance for:\n user agent: %s\n url: %s" % (useragent, url)) if self.disallow_all: return False Index: Misc/cheatsheet =================================================================== RCS file: /cvsroot/python/python/dist/src/Misc/cheatsheet,v retrieving revision 1.7 diff -u -r1.7 cheatsheet --- Misc/cheatsheet 12 Jul 2003 23:55:57 -0000 1.7 +++ Misc/cheatsheet 7 Mar 2004 14:26:18 -0000 @@ -1962,7 +1962,7 @@ rexec Restricted execution facilities ("safe" exec, eval, etc). rfc822 RFC-822 message manipulation class. rlcompleter Word completion for GNU readline 2.0. -robotparser Parse robot.txt files, useful for web spiders. +robotparser Parse robots.txt files, useful for web spiders. sched A generally useful event scheduler class. sets Module for a set datatype. sgmllib A parser for SGML.