Index: consts.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/compiler/consts.py,v retrieving revision 1.5 diff -c -r1.5 consts.py *** consts.py 14 Sep 2001 22:54:48 -0000 1.5 --- consts.py 4 Sep 2005 13:12:34 -0000 *************** *** 8,13 **** --- 8,14 ---- SC_FREE = 3 SC_CELL = 4 SC_UNKNOWN = 5 + SC_DEFAULT = 6 CO_OPTIMIZED = 0x0001 CO_NEWLOCALS = 0x0002 Index: future.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/compiler/future.py,v retrieving revision 1.3 diff -c -r1.3 future.py *** future.py 18 Oct 2001 21:57:37 -0000 1.3 --- future.py 4 Sep 2005 13:12:34 -0000 *************** *** 31,39 **** for name, asname in stmt.names: if name in self.features: self.found[name] = 1 else: ! raise SyntaxError, \ ! "future feature %s is not defined" % name stmt.valid_future = 1 return 1 return 0 --- 31,44 ---- for name, asname in stmt.names: if name in self.features: self.found[name] = 1 + elif name=="*": + raise SyntaxError( + "future statement does not support import *", + ( stmt.filename, stmt.lineno, 0, "" ) ) else: ! raise SyntaxError( ! "future feature %s is not defined" % name, ! ( stmt.filename, stmt.lineno, 0, "" ) ) stmt.valid_future = 1 return 1 return 0 *************** *** 43,56 **** return self.found.keys() class BadFutureParser: ! """Check for invalid future statements""" def visitFrom(self, node): if hasattr(node, 'valid_future'): return if node.modname != "__future__": return ! raise SyntaxError, "invalid future statement" def find_futures(node): p1 = FutureParser() --- 48,64 ---- return self.found.keys() class BadFutureParser: ! """Check for invalid future statements ! Those not marked valid are appearing after other statements ! """ def visitFrom(self, node): if hasattr(node, 'valid_future'): return if node.modname != "__future__": return ! raise SyntaxError( "from __future__ imports must occur at the beginning of the file", ! ( node.filename, node.lineno, 0, "" ) ) def find_futures(node): p1 = FutureParser() Index: pyassem.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/compiler/pyassem.py,v retrieving revision 1.31 diff -c -r1.31 pyassem.py *** pyassem.py 31 Dec 2002 18:17:42 -0000 1.31 --- pyassem.py 4 Sep 2005 13:12:34 -0000 *************** *** 244,250 **** op = inst[0] if op[:4] == 'JUMP': self.outEdges.add(inst[1]) ! self.insts.append(inst) def getInstructions(self): return self.insts --- 244,250 ---- op = inst[0] if op[:4] == 'JUMP': self.outEdges.add(inst[1]) ! self.insts.append( list(inst) ) def getInstructions(self): return self.insts *************** *** 314,320 **** class PyFlowGraph(FlowGraph): super_init = FlowGraph.__init__ ! def __init__(self, name, filename, args=(), optimized=0, klass=None): self.super_init() self.name = name self.filename = filename --- 314,321 ---- class PyFlowGraph(FlowGraph): super_init = FlowGraph.__init__ ! def __init__(self, name, filename, args=(), optimized=0, ! klass=None, newlocals=0): self.super_init() self.name = name self.filename = filename *************** *** 322,331 **** self.args = args # XXX self.argcount = getArgCount(args) self.klass = klass if optimized: ! self.flags = CO_OPTIMIZED | CO_NEWLOCALS ! else: ! self.flags = 0 self.consts = [] self.names = [] # Free variables found by the symbol table scan, including --- 323,333 ---- self.args = args # XXX self.argcount = getArgCount(args) self.klass = klass + self.flags = 0 if optimized: ! self.flags |= CO_OPTIMIZED ! if newlocals: ! self.flags |= CO_NEWLOCALS self.consts = [] self.names = [] # Free variables found by the symbol table scan, including *************** *** 343,348 **** --- 345,351 ---- if isinstance(var, TupleArg): self.varnames[i] = var.getName() self.stage = RAW + self.orderedblocks = [] def setDocstring(self, doc): self.docstring = doc *************** *** 366,375 **** """Get a Python code object""" if self.stage == RAW: self.computeStackDepth() - self.flattenGraph() - if self.stage == FLAT: self.convertArgs() if self.stage == CONV: self.makeByteCode() if self.stage == DONE: return self.newCodeObject() --- 369,378 ---- """Get a Python code object""" if self.stage == RAW: self.computeStackDepth() self.convertArgs() if self.stage == CONV: + self.flattenGraph() + if self.stage == FLAT: self.makeByteCode() if self.stage == DONE: return self.newCodeObject() *************** *** 425,459 **** def flattenGraph(self): """Arrange the blocks in order and resolve jumps""" ! assert self.stage == RAW self.insts = insts = [] pc = 0 begin = {} end = {} ! for b in self.getBlocksInOrder(): begin[b] = pc for inst in b.getInstructions(): - insts.append(inst) if len(inst) == 1: pc = pc + 1 elif inst[0] != "SET_LINENO": ! # arg takes 2 bytes ! pc = pc + 3 end[b] = pc pc = 0 ! for i in range(len(insts)): ! inst = insts[i] ! if len(inst) == 1: ! pc = pc + 1 ! elif inst[0] != "SET_LINENO": ! pc = pc + 3 ! opname = inst[0] if self.hasjrel.has_elt(opname): ! oparg = inst[1] ! offset = begin[oparg] - pc ! insts[i] = opname, offset ! elif self.hasjabs.has_elt(opname): ! insts[i] = opname, begin[inst[1]] self.stage = FLAT hasjrel = misc.Set() --- 428,491 ---- def flattenGraph(self): """Arrange the blocks in order and resolve jumps""" ! assert self.stage == CONV self.insts = insts = [] pc = 0 begin = {} end = {} ! forward_refs = [] ! for b in self.orderedblocks: begin[b] = pc for inst in b.getInstructions(): if len(inst) == 1: + insts.append(inst) pc = pc + 1 elif inst[0] != "SET_LINENO": ! opname, arg = inst ! if self.hasjrel.has_elt(opname): ! # relative jump - no extended arg ! forward_refs.append( (arg, inst, pc ) ) ! insts.append(inst) ! pc = pc + 3 ! elif self.hasjabs.has_elt(opname): ! # absolute jump - can be extended if backward ! if arg in begin: ! # can only extend argument if backward ! offset = begin[arg] ! hi, lo = divmod(offset,65536) ! if hi>0: ! # extended argument ! insts.append( ["EXTENDED_ARG", hi ] ) ! pc = pc + 3 ! inst[1] = lo ! else: ! forward_refs.append( (arg, inst, pc ) ) ! insts.append(inst) ! pc = pc + 3 ! else: ! # numerical arg ! assert type(arg)==int ! hi,lo = divmod(arg,65536) ! if hi>0: ! # extended argument ! insts.append( ["EXTENDED_ARG", hi ] ) ! inst[1] = lo ! pc = pc + 3 ! insts.append(inst) ! pc = pc + 3 ! else: ! insts.append(inst) end[b] = pc pc = 0 ! ! for arg, inst, pc in forward_refs: ! opname, block = inst ! abspos = begin[block] if self.hasjrel.has_elt(opname): ! offset = abspos - pc - 3 ! inst[1] = offset ! else: ! inst[1] = abspos self.stage = FLAT hasjrel = misc.Set() *************** *** 465,480 **** def convertArgs(self): """Convert arguments from symbolic to concrete form""" ! assert self.stage == FLAT self.consts.insert(0, self.docstring) self.sort_cellvars() ! for i in range(len(self.insts)): ! t = self.insts[i] ! if len(t) == 2: ! opname, oparg = t ! conv = self._converters.get(opname, None) ! if conv: ! self.insts[i] = opname, conv(self, oparg) self.stage = CONV def sort_cellvars(self): --- 497,514 ---- def convertArgs(self): """Convert arguments from symbolic to concrete form""" ! assert self.stage == RAW ! self.orderedblocks = self.getBlocksInOrder() self.consts.insert(0, self.docstring) self.sort_cellvars() ! ! for b in self.orderedblocks: ! for inst in b.getInstructions(): ! if len(inst) == 2: ! opname, oparg = inst ! conv = self._converters.get(opname, None) ! if conv: ! inst[1] = conv(self, oparg) self.stage = CONV def sort_cellvars(self): *************** *** 563,572 **** del name, obj, opname def makeByteCode(self): ! assert self.stage == CONV self.lnotab = lnotab = LineAddrTable() for t in self.insts: opname = t[0] if len(t) == 1: lnotab.addCode(self.opnum[opname]) else: --- 597,611 ---- del name, obj, opname def makeByteCode(self): ! assert self.stage == FLAT self.lnotab = lnotab = LineAddrTable() for t in self.insts: opname = t[0] + if self._debug: + if len(t)==1: + print "x",opname + else: + print "x",opname, t[1] if len(t) == 1: lnotab.addCode(self.opnum[opname]) else: Index: pycodegen.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/compiler/pycodegen.py,v retrieving revision 1.73 diff -c -r1.73 pycodegen.py *** pycodegen.py 20 Aug 2004 03:47:13 -0000 1.73 --- pycodegen.py 4 Sep 2005 13:12:34 -0000 *************** *** 8,14 **** from compiler import ast, parse, walk, syntax from compiler import pyassem, misc, future, symbols ! from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\ CO_NESTED, CO_GENERATOR, CO_GENERATOR_ALLOWED, CO_FUTURE_DIVISION from compiler.pyassem import TupleArg --- 8,14 ---- from compiler import ast, parse, walk, syntax from compiler import pyassem, misc, future, symbols ! from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_DEFAULT from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\ CO_NESTED, CO_GENERATOR, CO_GENERATOR_ALLOWED, CO_FUTURE_DIVISION from compiler.pyassem import TupleArg *************** *** 193,198 **** --- 193,201 ---- defined. """ + scopeambiguity = False + parentscopeambiguity = False + optimized = 0 # is namespace access optimized? __initialized = None class_name = None # provide default for instance variable *************** *** 265,273 **** --- 268,285 ---- self._nameOp('STORE', name) def loadName(self, name): + if (self.scope.nested and self.scopeambiguity and + name in self.scope.hasbeenfree): + raise SyntaxError("cannot reference variable '%s' because " + "of ambiguity between " + "scopes" % name) self._nameOp('LOAD', name) def delName(self, name): + scope = self.scope.check_name(name) + if scope == SC_CELL: + raise SyntaxError("can not delete variable '%s' " + "referenced in nested scope" % name) self._nameOp('DELETE', name) def _nameOp(self, prefix, name): *************** *** 279,290 **** else: self.emit(prefix + '_FAST', name) elif scope == SC_GLOBAL: ! if not self.optimized: ! self.emit(prefix + '_NAME', name) ! else: ! self.emit(prefix + '_GLOBAL', name) elif scope == SC_FREE or scope == SC_CELL: self.emit(prefix + '_DEREF', name) else: raise RuntimeError, "unsupported scope for var %s: %d" % \ (name, scope) --- 291,304 ---- else: self.emit(prefix + '_FAST', name) elif scope == SC_GLOBAL: ! self.emit(prefix + '_GLOBAL', name) elif scope == SC_FREE or scope == SC_CELL: self.emit(prefix + '_DEREF', name) + elif scope == SC_DEFAULT: + if self.optimized and self.localsfullyknown: + self.emit(prefix + '_GLOBAL', name) + else: + self.emit(prefix + '_NAME', name) else: raise RuntimeError, "unsupported scope for var %s: %d" % \ (name, scope) *************** *** 374,380 **** ndecorators = 0 gen = self.FunctionGen(node, self.scopes, isLambda, ! self.class_name, self.get_module()) walk(node.code, gen) gen.finish() self.set_lineno(node) --- 388,395 ---- ndecorators = 0 gen = self.FunctionGen(node, self.scopes, isLambda, ! self.class_name, self.get_module(), ! parentscopeambiguity = self.scopeambiguity or self.parentscopeambiguity) walk(node.code, gen) gen.finish() self.set_lineno(node) *************** *** 395,401 **** def visitClass(self, node): gen = self.ClassGen(node, self.scopes, ! self.get_module()) walk(node.code, gen) gen.finish() self.set_lineno(node) --- 410,417 ---- def visitClass(self, node): gen = self.ClassGen(node, self.scopes, ! self.get_module(), ! parentscopeambiguity = self.scopeambiguity or self.parentscopeambiguity) walk(node.code, gen) gen.finish() self.set_lineno(node) *************** *** 503,510 **** def visitContinue(self, node): if not self.setups: ! raise SyntaxError, "'continue' outside loop (%s, %d)" % \ ! (node.filename, node.lineno) kind, block = self.setups.top() if kind == LOOP: self.set_lineno(node) --- 519,525 ---- def visitContinue(self, node): if not self.setups: ! raise SyntaxError, "'continue' not properly in loop" # (%s, %d)" % (node.filename, node.lineno) kind, block = self.setups.top() if kind == LOOP: self.set_lineno(node) *************** *** 520,532 **** if kind == LOOP: break if kind != LOOP: ! raise SyntaxError, "'continue' outside loop (%s, %d)" % \ ! (node.filename, node.lineno) self.emit('CONTINUE_LOOP', loop_block) self.nextBlock() elif kind == END_FINALLY: ! msg = "'continue' not allowed inside 'finally' clause (%s, %d)" ! raise SyntaxError, msg % (node.filename, node.lineno) def visitTest(self, node, jump): end = self.newBlock() --- 535,546 ---- if kind == LOOP: break if kind != LOOP: ! raise SyntaxError, "'continue' not properly in loop" # (%s, %d)" % (node.filename, node.lineno) self.emit('CONTINUE_LOOP', loop_block) self.nextBlock() elif kind == END_FINALLY: ! msg = "'continue' not supported inside 'finally' clause" # " (%s, %d)" ! raise SyntaxError, msg # % (node.filename, node.lineno) def visitTest(self, node, jump): end = self.newBlock() *************** *** 631,637 **** def visitGenExpr(self, node): gen = GenExprCodeGenerator(node, self.scopes, self.class_name, ! self.get_module()) walk(node.code, gen) gen.finish() self.set_lineno(node) --- 645,652 ---- def visitGenExpr(self, node): gen = GenExprCodeGenerator(node, self.scopes, self.class_name, ! self.get_module(), ! parentscopeambiguity=self.scopeambiguity or self.parentscopeambiguity) walk(node.code, gen) gen.finish() self.set_lineno(node) *************** *** 811,816 **** --- 826,836 ---- # misc def visitDiscard(self, node): + # Important: this function is overridden in InteractiveCodeGenerator, + # which also has the effect that the following test only occurs in + # non-'single' modes. + if isinstance(node.expr, ast.Const): + return # skip LOAD_CONST/POP_TOP pairs (for e.g. docstrings) self.set_lineno(node) self.visit(node.expr) self.emit('POP_TOP') *************** *** 1221,1229 **** scopes = None ! def __init__(self, tree): self.graph = pyassem.PyFlowGraph("", tree.filename) self.futures = future.find_futures(tree) self.__super_init() walk(tree, self) --- 1241,1252 ---- scopes = None ! def __init__(self, tree, futures = []): self.graph = pyassem.PyFlowGraph("", tree.filename) self.futures = future.find_futures(tree) + for f in futures: + if f not in self.futures: + self.futures.append(f) self.__super_init() walk(tree, self) *************** *** 1234,1243 **** __super_init = CodeGenerator.__init__ scopes = None - futures = () ! def __init__(self, tree): self.graph = pyassem.PyFlowGraph("", tree.filename) self.__super_init() walk(tree, self) --- 1257,1266 ---- __super_init = CodeGenerator.__init__ scopes = None ! def __init__(self, tree, futures=[]): self.graph = pyassem.PyFlowGraph("", tree.filename) + self.futures = futures[:] self.__super_init() walk(tree, self) *************** *** 1249,1258 **** __super_init = CodeGenerator.__init__ scopes = None - futures = () ! def __init__(self, tree): self.graph = pyassem.PyFlowGraph("", tree.filename) self.__super_init() self.set_lineno(tree) walk(tree, self) --- 1272,1284 ---- __super_init = CodeGenerator.__init__ scopes = None ! def __init__(self, tree, futures=[]): self.graph = pyassem.PyFlowGraph("", tree.filename) + self.futures = future.find_futures(tree) + for f in futures: + if f not in self.futures: + self.futures.append(f) self.__super_init() self.set_lineno(tree) walk(tree, self) *************** *** 1269,1289 **** class AbstractFunctionCode: optimized = 1 - lambdaCount = 0 def __init__(self, func, scopes, isLambda, class_name, mod): self.class_name = class_name self.module = mod if isLambda: klass = FunctionCodeGenerator ! name = "" % klass.lambdaCount ! klass.lambdaCount = klass.lambdaCount + 1 else: name = func.name args, hasTupleArg = generateArgList(func.argnames) self.graph = pyassem.PyFlowGraph(name, func.filename, args, ! optimized=1) self.isLambda = isLambda self.super_init() --- 1295,1314 ---- class AbstractFunctionCode: optimized = 1 def __init__(self, func, scopes, isLambda, class_name, mod): self.class_name = class_name self.module = mod if isLambda: klass = FunctionCodeGenerator ! name = "" else: name = func.name args, hasTupleArg = generateArgList(func.argnames) self.graph = pyassem.PyFlowGraph(name, func.filename, args, ! optimized=self.localsfullyknown, ! newlocals=1) self.isLambda = isLambda self.super_init() *************** *** 1336,1344 **** __super_init = AbstractFunctionCode.__init__ ! def __init__(self, func, scopes, isLambda, class_name, mod): self.scopes = scopes self.scope = scopes[func] self.__super_init(func, scopes, isLambda, class_name, mod) self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setCellVars(self.scope.get_cell_vars()) --- 1361,1374 ---- __super_init = AbstractFunctionCode.__init__ ! def __init__(self, func, scopes, isLambda, class_name, mod, parentscopeambiguity): self.scopes = scopes self.scope = scopes[func] + + self.localsfullyknown = self.scope.localsfullyknown + self.parentscopeambiguity = parentscopeambiguity + self.scopeambiguity = (not self.localsfullyknown or parentscopeambiguity) + self.__super_init(func, scopes, isLambda, class_name, mod) self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setCellVars(self.scope.get_cell_vars()) *************** *** 1352,1360 **** __super_init = AbstractFunctionCode.__init__ ! def __init__(self, gexp, scopes, class_name, mod): self.scopes = scopes self.scope = scopes[gexp] self.__super_init(gexp, scopes, 1, class_name, mod) self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setCellVars(self.scope.get_cell_vars()) --- 1382,1395 ---- __super_init = AbstractFunctionCode.__init__ ! def __init__(self, gexp, scopes, class_name, mod, parentscopeambiguity): self.scopes = scopes self.scope = scopes[gexp] + + self.localsfullyknown = self.scope.localsfullyknown + self.parentscopeambiguity = parentscopeambiguity + self.scopeambiguity = (not self.localsfullyknown or parentscopeambiguity) + self.__super_init(gexp, scopes, 1, class_name, mod) self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setCellVars(self.scope.get_cell_vars()) *************** *** 1388,1396 **** __super_init = AbstractClassCode.__init__ ! def __init__(self, klass, scopes, module): self.scopes = scopes self.scope = scopes[klass] self.__super_init(klass, scopes, module) self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setCellVars(self.scope.get_cell_vars()) --- 1423,1435 ---- __super_init = AbstractClassCode.__init__ ! def __init__(self, klass, scopes, module, parentscopeambiguity): self.scopes = scopes self.scope = scopes[klass] + + self.parentscopeambiguity = parentscopeambiguity + self.scopeambiguity = parentscopeambiguity + self.__super_init(klass, scopes, module) self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setCellVars(self.scope.get_cell_vars()) Index: symbols.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/compiler/symbols.py,v retrieving revision 1.16 diff -c -r1.16 symbols.py *** symbols.py 2 Aug 2004 06:09:53 -0000 1.16 --- symbols.py 4 Sep 2005 13:12:34 -0000 *************** *** 1,16 **** """Module symbol-table generator""" from compiler import ast ! from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN from compiler.misc import mangle import types ! import sys MANGLE_LEN = 256 class Scope: # XXX how much information do I need about each name? def __init__(self, name, module, klass=None): self.name = name --- 1,27 ---- """Module symbol-table generator""" from compiler import ast ! from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, \ ! SC_UNKNOWN, SC_DEFAULT from compiler.misc import mangle import types ! import warnings import sys MANGLE_LEN = 256 + def issue_warning(msg, filename, lineno): + try: + warnings.warn_explicit(msg, SyntaxWarning, filename, lineno) + except SyntaxWarning: + err = SyntaxError(msg) + err.filename = filename + err.lineno = lineno + raise err + class Scope: + localsfullyknown = True # XXX how much information do I need about each name? def __init__(self, name, module, klass=None): self.name = name *************** *** 20,25 **** --- 31,37 ---- self.globals = {} self.params = {} self.frees = {} + self.hasbeenfree = {} self.cells = {} self.children = [] # nested is true if the class could contain free variables, *************** *** 100,106 **** if self.nested: return SC_UNKNOWN else: ! return SC_GLOBAL def get_free_vars(self): if not self.nested: --- 112,118 ---- if self.nested: return SC_UNKNOWN else: ! return SC_DEFAULT def get_free_vars(self): if not self.nested: *************** *** 111,116 **** --- 123,129 ---- if not (self.defs.has_key(name) or self.globals.has_key(name)): free[name] = 1 + self.hasbeenfree.update(free) return free.keys() def handle_children(self): *************** *** 133,139 **** Be careful to stop if a child does not think the name is free. """ ! self.globals[name] = 1 if self.frees.has_key(name): del self.frees[name] for child in self.children: --- 146,153 ---- Be careful to stop if a child does not think the name is free. """ ! if name not in self.defs: ! self.globals[name] = 1 if self.frees.has_key(name): del self.frees[name] for child in self.children: *************** *** 237,242 **** --- 251,262 ---- self.visit(node.code, scope) self.handle_free_vars(scope, parent) + def visitExec(self, node, parent): + if not (node.globals or node.locals): + parent.localsfullyknown = False # bare exec statement + for child in node.getChildNodes(): + self.visit(child, parent) + def visitGenExpr(self, node, parent): scope = GenExprScope(self.module, self.klass); if parent.nested or isinstance(parent, FunctionScope) \ *************** *** 331,336 **** --- 351,357 ---- def visitFrom(self, node, scope): for name, asname in node.names: if name == "*": + scope.localsfullyknown = False continue scope.add_def(asname or name) *************** *** 343,348 **** --- 364,378 ---- def visitGlobal(self, node, scope): for name in node.names: + namescope = scope.check_name(name) + if namescope == SC_LOCAL: + issue_warning("name '%s' is assigned to before " + "global declaration" %(name,), + node.filename, node.lineno) + elif namescope != SC_GLOBAL and name in scope.uses: + issue_warning("name '%s' is used prior " + "to global declaration" %(name,), + node.filename, node.lineno) scope.add_global(name) def visitAssign(self, node, scope): Index: transformer.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/compiler/transformer.py,v retrieving revision 1.49.2.1 diff -c -r1.49.2.1 transformer.py *** transformer.py 2 Jul 2005 18:38:12 -0000 1.49.2.1 --- transformer.py 4 Sep 2005 13:12:34 -0000 *************** *** 102,109 **** tree = parsefile(fileob | filename) """ ! def __init__(self): self._dispatch = {} for value, name in symbol.sym_name.items(): if hasattr(self, name): self._dispatch[value] = getattr(self, name) --- 102,110 ---- tree = parsefile(fileob | filename) """ ! def __init__(self, filename='?'): self._dispatch = {} + self.filename = filename for value, name in symbol.sym_name.items(): if hasattr(self, name): self._dispatch[value] = getattr(self, name) *************** *** 118,123 **** --- 119,137 ---- } self.encoding = None + def syntaxerror(self, msg, node): + offset = 0 + text = "" + lineno = extractLineNo( node ) + args = ( self.filename, lineno, offset, text ) + raise SyntaxError( msg, args ) + + def none_assignment_error(self, assigning, node): + if assigning==OP_DELETE: + self.syntaxerror( "deleting None", node ) + else: + self.syntaxerror( "assignment to None", node ) + def transform(self, tree): """Transform an AST into a modified parse tree.""" if not (isinstance(tree, tuple) or isinstance(tree, list)): *************** *** 168,192 **** raise WalkerError, ('unexpected node type', n) def single_input(self, node): - ### do we want to do anything about being "interactive" ? - # NEWLINE | simple_stmt | compound_stmt NEWLINE n = node[0][0] if n != token.NEWLINE: ! return self.com_stmt(node[0]) ! ! return Pass() def file_input(self, nodelist): doc = self.get_docstring(nodelist, symbol.file_input) - if doc is not None: - i = 1 - else: - i = 0 stmts = [] ! for node in nodelist[i:]: if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: self.com_append_stmt(stmts, node) return Module(doc, Stmt(stmts)) def eval_input(self, nodelist): --- 182,207 ---- raise WalkerError, ('unexpected node type', n) def single_input(self, node): # NEWLINE | simple_stmt | compound_stmt NEWLINE n = node[0][0] if n != token.NEWLINE: ! stmt = self.com_stmt(node[0]) ! else: ! stmt = Pass() ! return Module(None, stmt) def file_input(self, nodelist): doc = self.get_docstring(nodelist, symbol.file_input) stmts = [] ! for node in nodelist: if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: self.com_append_stmt(stmts, node) + + if doc is not None: + assert isinstance(stmts[0], Discard) + assert isinstance(stmts[0].expr, Const) + del stmts[0] + return Module(doc, Stmt(stmts)) def eval_input(self, nodelist): *************** *** 252,258 **** if args[0] == symbol.varargslist: names, defaults, flags = self.com_arglist(args[1:]) else: ! names = defaults = () flags = 0 doc = self.get_docstring(nodelist[-1]) --- 267,274 ---- if args[0] == symbol.varargslist: names, defaults, flags = self.com_arglist(args[1:]) else: ! names = [] ! defaults = [] flags = 0 doc = self.get_docstring(nodelist[-1]) *************** *** 263,268 **** --- 279,286 ---- assert isinstance(code, Stmt) assert isinstance(code.nodes[0], Discard) del code.nodes[0] + if name == "None": + self.none_assignment_error( OP_ASSIGN, nodelist[-4] ) return Function(decorators, name, names, defaults, flags, doc, code, lineno=lineno) *************** *** 271,277 **** if nodelist[2][0] == symbol.varargslist: names, defaults, flags = self.com_arglist(nodelist[2][1:]) else: ! names = defaults = () flags = 0 # code for lambda --- 289,296 ---- if nodelist[2][0] == symbol.varargslist: names, defaults, flags = self.com_arglist(nodelist[2][1:]) else: ! names = [] ! defaults = [] flags = 0 # code for lambda *************** *** 297,302 **** --- 316,323 ---- assert isinstance(code.nodes[0], Discard) del code.nodes[0] + if name == "None": + self.none_assignment_error(OP_ASSIGN, nodelist[1]) return Class(name, bases, doc, code, lineno=nodelist[1][2]) def stmt(self, nodelist): *************** *** 440,446 **** return From(fromname, [('*', None)], lineno=nodelist[0][2]) else: ! node = nodelist[3 + (nodelist[3][0] == token.LPAR)] return From(fromname, self.com_import_as_names(node), lineno=nodelist[0][2]) --- 461,472 ---- return From(fromname, [('*', None)], lineno=nodelist[0][2]) else: ! if nodelist[3][0] == token.LPAR: ! node = nodelist[4] ! else: ! node = nodelist[3] ! if node[-1][0] == token.COMMA: ! self.syntaxerror("trailing comma not allowed without surrounding parentheses", node) return From(fromname, self.com_import_as_names(node), lineno=nodelist[0][2]) *************** *** 700,706 **** def atom_lsqb(self, nodelist): if nodelist[1][0] == token.RSQB: ! return List(()) return self.com_list_constructor(nodelist[1]) def atom_lbrace(self, nodelist): --- 726,732 ---- def atom_lsqb(self, nodelist): if nodelist[1][0] == token.RSQB: ! return List([]) return self.com_list_constructor(nodelist[1]) def atom_lbrace(self, nodelist): *************** *** 776,782 **** names = [] defaults = [] flags = 0 - i = 0 while i < len(nodelist): node = nodelist[i] --- 802,807 ---- *************** *** 784,790 **** if node[0] == token.STAR: node = nodelist[i+1] if node[0] == token.NAME: ! names.append(node[1]) flags = flags | CO_VARARGS i = i + 3 --- 809,819 ---- if node[0] == token.STAR: node = nodelist[i+1] if node[0] == token.NAME: ! name = node[1] ! if name in names: ! self.syntaxerror("duplicate argument '%s' in function definition" % ! name, node) ! names.append(name) flags = flags | CO_VARARGS i = i + 3 *************** *** 795,822 **** node = nodelist[i+1] else: raise ValueError, "unexpected token: %s" % t ! names.append(node[1]) flags = flags | CO_VARKEYWORDS break # fpdef: NAME | '(' fplist ')' ! names.append(self.com_fpdef(node)) i = i + 1 if i >= len(nodelist): break ! if nodelist[i][0] == token.EQUAL: defaults.append(self.com_node(nodelist[i + 1])) i = i + 2 elif len(defaults): ! # XXX This should be a syntax error. ! # Treat "(a=1, b)" as "(a=1, b=None)" ! defaults.append(Const(None)) i = i + 1 return names, defaults, flags def com_fpdef(self, node): --- 824,861 ---- node = nodelist[i+1] else: raise ValueError, "unexpected token: %s" % t ! name = node[1] ! if name in names: ! self.syntaxerror("duplicate argument '%s' in function definition" % ! name, node) ! names.append(name) flags = flags | CO_VARKEYWORDS break # fpdef: NAME | '(' fplist ')' ! name = self.com_fpdef(node) ! if name in names: ! self.syntaxerror("duplicate argument '%s' in function definition" % ! name, node) ! names.append(name) i = i + 1 if i >= len(nodelist): + if len(defaults): + self.syntaxerror("non-default argument follows default argument",node) break ! if nodelist[i][0] == token.EQUAL: defaults.append(self.com_node(nodelist[i + 1])) i = i + 2 elif len(defaults): ! self.syntaxerror("non-default argument follows default argument",node) i = i + 1 + if "None" in names: + self.syntaxerror( "Invalid syntax. Assignment to None.", node) return names, defaults, flags def com_fpdef(self, node): *************** *** 925,933 **** Names, slices, and attributes are the only allowable nodes. """ l = self.com_node(node) if l.__class__ in (Name, Slice, Subscript, Getattr): return l ! raise SyntaxError, "can't assign to %s" % l.__class__.__name__ def com_assign(self, node, assigning): # return a node suitable for use as an "lvalue" --- 964,977 ---- Names, slices, and attributes are the only allowable nodes. """ l = self.com_node(node) + if isinstance(l, Name): + if l.name == "__debug__": + self.syntaxerror( "can not assign to __debug__", node ) + if l.name == "None": + self.none_assignment_error( OP_ASSIGN, node ) if l.__class__ in (Name, Slice, Subscript, Getattr): return l ! self.syntaxerror( "can't assign to %s" % l.__class__.__name__, node) def com_assign(self, node, assigning): # return a node suitable for use as an "lvalue" *************** *** 940,956 **** node = node[1] elif t in _assign_types: if len(node) > 2: ! raise SyntaxError, "can't assign to operator" node = node[1] elif t == symbol.power: if node[1][0] != symbol.atom: ! raise SyntaxError, "can't assign to operator" if len(node) > 2: primary = self.com_node(node[1]) for i in range(2, len(node)-1): ch = node[i] if ch[0] == token.DOUBLESTAR: ! raise SyntaxError, "can't assign to operator" primary = self.com_apply_trailer(primary, ch) return self.com_assign_trailer(primary, node[-1], assigning) --- 984,1000 ---- node = node[1] elif t in _assign_types: if len(node) > 2: ! self.syntaxerror( "can't assign to operator", node) node = node[1] elif t == symbol.power: if node[1][0] != symbol.atom: ! self.syntaxerror( "can't assign to operator", node) if len(node) > 2: primary = self.com_node(node[1]) for i in range(2, len(node)-1): ch = node[i] if ch[0] == token.DOUBLESTAR: ! self.syntaxerror( "can't assign to operator", node) primary = self.com_apply_trailer(primary, ch) return self.com_assign_trailer(primary, node[-1], assigning) *************** *** 960,980 **** if t == token.LPAR: node = node[2] if node[0] == token.RPAR: ! raise SyntaxError, "can't assign to ()" elif t == token.LSQB: node = node[2] if node[0] == token.RSQB: ! raise SyntaxError, "can't assign to []" return self.com_assign_list(node, assigning) elif t == token.NAME: return self.com_assign_name(node[1], assigning) else: ! raise SyntaxError, "can't assign to literal" else: ! raise SyntaxError, "bad assignment" def com_assign_tuple(self, node, assigning): assigns = [] for i in range(1, len(node), 2): assigns.append(self.com_assign(node[i], assigning)) return AssTuple(assigns, lineno=extractLineNo(node)) --- 1004,1031 ---- if t == token.LPAR: node = node[2] if node[0] == token.RPAR: ! self.syntaxerror( "can't assign to ()", node) elif t == token.LSQB: node = node[2] if node[0] == token.RSQB: ! self.syntaxerror( "can't assign to []", node) return self.com_assign_list(node, assigning) elif t == token.NAME: + if node[1][1] == "__debug__": + self.syntaxerror( "can not assign to __debug__", node ) + if node[1][1] == "None": + self.none_assignment_error(assigning, node) return self.com_assign_name(node[1], assigning) else: ! self.syntaxerror( "can't assign to literal", node) else: ! self.syntaxerror( "bad assignment", node) def com_assign_tuple(self, node, assigning): assigns = [] + if len(node)>=3: + if node[2][0] == symbol.gen_for: + self.syntaxerror("assign to generator expression not possible", node) for i in range(1, len(node), 2): assigns.append(self.com_assign(node[i], assigning)) return AssTuple(assigns, lineno=extractLineNo(node)) *************** *** 984,990 **** for i in range(1, len(node), 2): if i + 1 < len(node): if node[i + 1][0] == symbol.list_for: ! raise SyntaxError, "can't assign to list comprehension" assert node[i + 1][0] == token.COMMA, node[i + 1] assigns.append(self.com_assign(node[i], assigning)) return AssList(assigns, lineno=extractLineNo(node)) --- 1035,1041 ---- for i in range(1, len(node), 2): if i + 1 < len(node): if node[i + 1][0] == symbol.list_for: ! self.syntaxerror( "can't assign to list comprehension", node) assert node[i + 1][0] == token.COMMA, node[i + 1] assigns.append(self.com_assign(node[i], assigning)) return AssList(assigns, lineno=extractLineNo(node)) *************** *** 999,1008 **** if t == token.LSQB: return self.com_subscriptlist(primary, node[2], assigning) if t == token.LPAR: ! raise SyntaxError, "can't assign to function call" ! raise SyntaxError, "unknown trailer type: %s" % t def com_assign_attr(self, primary, node, assigning): return AssAttr(primary, node[1], assigning, lineno=node[-1]) def com_binary(self, constructor, nodelist): --- 1050,1064 ---- if t == token.LSQB: return self.com_subscriptlist(primary, node[2], assigning) if t == token.LPAR: ! if assigning==OP_DELETE: ! self.syntaxerror( "can't delete function call", node) ! else: ! self.syntaxerror( "can't assign to function call", node) ! self.syntaxerror( "unknown trailer type: %s" % t, node) def com_assign_attr(self, primary, node, assigning): + if node[1]=="None": + self.none_assignment_error(assigning, node) return AssAttr(primary, node[1], assigning, lineno=node[-1]) def com_binary(self, constructor, nodelist): *************** *** 1076,1084 **** else: node = self.com_list_iter(node[3]) else: ! raise SyntaxError, \ ! ("unexpected list comprehension element: %s %d" ! % (node, lineno)) return ListComp(expr, fors, lineno=lineno) def com_list_iter(self, node): --- 1132,1140 ---- else: node = self.com_list_iter(node[3]) else: ! self.syntaxerror( ! "unexpected list comprehension element: %s %d" ! % (node, lineno), node) return ListComp(expr, fors, lineno=lineno) def com_list_iter(self, node): *************** *** 1120,1128 **** else: node = self.com_gen_iter(node[3]) else: ! raise SyntaxError, \ ! ("unexpected generator expression element: %s %d" ! % (node, lineno)) fors[0].is_outmost = True return GenExpr(GenExprInner(expr, fors), lineno=lineno) --- 1176,1184 ---- else: node = self.com_gen_iter(node[3]) else: ! self.syntaxerror( ! "unexpected generator expression element: %s %d" ! % (node, lineno), node) fors[0].is_outmost = True return GenExpr(GenExprInner(expr, fors), lineno=lineno) *************** *** 1147,1157 **** if t == token.LSQB: return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) ! raise SyntaxError, 'unknown node type: %s' % t def com_select_member(self, primaryNode, nodelist): if nodelist[0] != token.NAME: ! raise SyntaxError, "member must be a name" return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) def com_call_function(self, primaryNode, nodelist): --- 1203,1213 ---- if t == token.LSQB: return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) ! self.syntaxerror( 'unknown node type: %s' % t, nodelist[1]) def com_select_member(self, primaryNode, nodelist): if nodelist[0] != token.NAME: ! self.syntaxerror( "member must be a name", nodelist[0]) return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) def com_call_function(self, primaryNode, nodelist): *************** *** 1170,1176 **** and len(node) == 3 and node[2][0] == symbol.gen_for: # allow f(x for x in y), but reject f(x for x in y, 1) # should use f((x for x in y), 1) instead of f(x for x in y, 1) ! raise SyntaxError, 'generator expression needs parenthesis' args.append(result) else: --- 1226,1232 ---- and len(node) == 3 and node[2][0] == symbol.gen_for: # allow f(x for x in y), but reject f(x for x in y, 1) # should use f((x for x in y), 1) instead of f(x for x in y, 1) ! self.syntaxerror( 'generator expression needs parenthesis', node) args.append(result) else: *************** *** 1186,1199 **** i = i + 3 if tok[0]==token.STAR: if star_node is not None: ! raise SyntaxError, 'already have the varargs indentifier' star_node = self.com_node(ch) elif tok[0]==token.DOUBLESTAR: if dstar_node is not None: ! raise SyntaxError, 'already have the kwargs indentifier' dstar_node = self.com_node(ch) else: ! raise SyntaxError, 'unknown node type: %s' % tok return CallFunc(primaryNode, args, star_node, dstar_node, lineno=extractLineNo(nodelist)) --- 1242,1255 ---- i = i + 3 if tok[0]==token.STAR: if star_node is not None: ! self.syntaxerror( 'already have the varargs indentifier', tok ) star_node = self.com_node(ch) elif tok[0]==token.DOUBLESTAR: if dstar_node is not None: ! self.syntaxerror( 'already have the kwargs indentifier', tok ) dstar_node = self.com_node(ch) else: ! self.syntaxerror( 'unknown node type: %s' % tok, tok ) return CallFunc(primaryNode, args, star_node, dstar_node, lineno=extractLineNo(nodelist)) *************** *** 1203,1216 **** return 0, self.com_generator_expression(test, nodelist[2]) if len(nodelist) == 2: if kw: ! raise SyntaxError, "non-keyword arg after keyword arg" return 0, self.com_node(nodelist[1]) result = self.com_node(nodelist[3]) n = nodelist[1] while len(n) == 2 and n[0] != token.NAME: n = n[1] if n[0] != token.NAME: ! raise SyntaxError, "keyword can't be an expression (%s)"%n[0] node = Keyword(n[1], result, lineno=n[2]) return 1, node --- 1259,1272 ---- return 0, self.com_generator_expression(test, nodelist[2]) if len(nodelist) == 2: if kw: ! self.syntaxerror( "non-keyword arg after keyword arg", nodelist ) return 0, self.com_node(nodelist[1]) result = self.com_node(nodelist[3]) n = nodelist[1] while len(n) == 2 and n[0] != token.NAME: n = n[1] if n[0] != token.NAME: ! self.syntaxerror( "keyword can't be an expression (%s)"%n[0], n) node = Keyword(n[1], result, lineno=n[2]) return 1, node