import robotparser

class MyURLopener(robotparser.URLopener):
    version = 'Some bot/0.9'

if __name__ == '__main__':

    rp = robotparser.RobotFileParser('http://en.wikipedia.org/robots.txt')
    rp.read()
    print rp.can_fetch('*', 'http://en.wikipedia.org/robots.txt')

    rp = robotparser.RobotFileParser('http://en.wikipedia.org/robots.txt')
    rp.opener = MyURLopener()
    rp.read()
    print rp.can_fetch('*', 'http://en.wikipedia.org/robots.txt')