diff -r 336137a359ae Lib/ast.py --- a/Lib/ast.py Fri Mar 11 10:27:14 2011 -0500 +++ b/Lib/ast.py Mon Mar 14 23:22:44 2011 -0400 @@ -49,10 +49,8 @@ if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert(node): - if isinstance(node, (Str, Bytes)): - return node.s - elif isinstance(node, Num): - return node.n + if isinstance(node, Lit): + return node.v elif isinstance(node, Tuple): return tuple(map(_convert, node.elts)) elif isinstance(node, List): @@ -67,7 +65,7 @@ return _safe_names[node.id] elif isinstance(node, UnaryOp) and \ isinstance(node.op, (UAdd, USub)) and \ - isinstance(node.operand, (Num, UnaryOp, BinOp)): + isinstance(node.operand, (Lit, UnaryOp, BinOp)): operand = _convert(node.operand) if isinstance(node.op, UAdd): return + operand @@ -75,8 +73,8 @@ return - operand elif isinstance(node, BinOp) and \ isinstance(node.op, (Add, Sub)) and \ - isinstance(node.right, (Num, UnaryOp, BinOp)) and \ - isinstance(node.left, (Num, UnaryOp, BinOp)): + isinstance(node.right, (Lit, UnaryOp, BinOp)) and \ + isinstance(node.left, (Lit, UnaryOp, BinOp)): left = _convert(node.left) right = _convert(node.right) if isinstance(node.op, Add): @@ -199,12 +197,10 @@ """ if not isinstance(node, (FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - if clean: - import inspect - return inspect.cleandoc(node.body[0].value.s) - return node.body[0].value.s + if clean: + import inspect + return inspect.cleandoc(node.docstring) + return node.docstring def walk(node): @@ -277,7 +273,7 @@ def visit_Name(self, node): return copy_location(Subscript( value=Name(id='data', ctx=Load()), - slice=Index(value=Str(s=node.id)), + slice=Index(value=Lit(v=node.id)), ctx=node.ctx ), node) diff -r 336137a359ae PC/os2emx/python33.def --- a/PC/os2emx/python33.def Fri Mar 11 10:27:14 2011 -0500 +++ b/PC/os2emx/python33.def Mon Mar 14 23:22:44 2011 -0400 @@ -719,14 +719,12 @@ "Compare" "Call" "Repr" - "Num" - "Str" + "Lit" "Attribute" "Subscript" "Name" "List" "Tuple" - "Ellipsis" "Slice" "ExtSlice" "Index" diff -r 336137a359ae Parser/Python.asdl --- a/Parser/Python.asdl Fri Mar 11 10:27:14 2011 -0500 +++ b/Parser/Python.asdl Mon Mar 14 23:22:44 2011 -0400 @@ -2,7 +2,7 @@ module Python version "$Revision$" { - mod = Module(stmt* body) + mod = Module(stmt* body, string? docstring) | Interactive(stmt* body) | Expression(expr body) @@ -10,14 +10,16 @@ | Suite(stmt* body) stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) + stmt* body, expr* decorator_list, expr? returns, + string? docstring) | ClassDef(identifier name, expr* bases, keyword* keywords, expr? starargs, expr? kwargs, stmt* body, - expr* decorator_list) + expr* decorator_list, + string? docstring) | Return(expr? value) | Delete(expr* targets) @@ -66,11 +68,7 @@ | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? + | Lit(object v) -- a literal as a PyObject. -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) diff -r 336137a359ae Parser/asdl_c.py --- a/Parser/asdl_c.py Fri Mar 11 10:27:14 2011 -0500 +++ b/Parser/asdl_c.py Mon Mar 14 23:22:44 2011 -0400 @@ -291,6 +291,10 @@ emit("%s(%s)" % (name, argstr)) emit("{") emit("%s p;" % ctype, 1) + # XXX: special hack for Lit. Lit value can be None and it + # should be stored as Py_None, not as NULL. + if name.value == 'Lit': + emit('if (!v) v = Py_None;', 1) for argtype, argname, opt in args: if not opt and argtype != "int": emit("if (!%s) {" % argname, 1) @@ -501,17 +505,21 @@ self.emit("goto failed;", depth+2) self.emit("}", depth+1) self.emit("len = PyList_GET_SIZE(tmp);", depth+1) + self.emit("if (len == 0)", depth+1) + self.emit("%s = NULL;" % field.name, depth+2) + self.emit("else {", depth+1) if self.isSimpleType(field): - self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1) + self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+2) else: - self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1) - self.emit("if (%s == NULL) goto failed;" % field.name, depth+1) - self.emit("for (i = 0; i < len; i++) {", depth+1) - self.emit("%s value;" % ctype, depth+2) + self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+2) + self.emit("if (%s == NULL) goto failed;" % field.name, depth+2) + self.emit("for (i = 0; i < len; i++) {", depth+2) + self.emit("%s value;" % ctype, depth+3) self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" % - field.type, depth+2, reflow=False) - self.emit("if (res != 0) goto failed;", depth+2) - self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2) + field.type, depth+3, reflow=False) + self.emit("if (res != 0) goto failed;", depth+3) + self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+3) + self.emit("}", depth+2) self.emit("}", depth+1) else: self.emit("res = obj2ast_%s(tmp, &%s, arena);" % diff -r 336137a359ae Python/ast.c --- a/Python/ast.c Fri Mar 11 10:27:14 2011 -0500 +++ b/Python/ast.c Mon Mar 14 23:22:44 2011 -0400 @@ -24,7 +24,9 @@ static asdl_seq *seq_for_testlist(struct compiling *, const node *); static expr_ty ast_for_expr(struct compiling *, const node *); static stmt_ty ast_for_stmt(struct compiling *, const node *); -static asdl_seq *ast_for_suite(struct compiling *, const node *); +static asdl_seq *ast_for_body(struct compiling *c, const node *n, + string *docstring); +static string docstring_from_stmts(asdl_seq *stmts); static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty); static expr_ty ast_for_testlist(struct compiling *, const node *); @@ -275,7 +277,7 @@ } } } - return Module(stmts, arena); + return Module(stmts, docstring_from_stmts(stmts), arena); case eval_input: { expr_ty testlist_ast; @@ -374,6 +376,12 @@ NULL, }; +static PyObject* FORBIDDEN_VALUES[] = { + Py_None, + Py_True, + Py_False +}; + static int forbidden_name(identifier name, const node *n, int full_checks) { @@ -394,6 +402,24 @@ return 0; } +static int +make_name_literal(expr_ty e) +{ + int i; + identifier name; + + assert(e->kind == Name_kind); + name = e->v.Name.id; + for (i = 0; FORBIDDEN[i]; i++) { + if (PyUnicode_CompareWithASCIIString(name, FORBIDDEN[i]) == 0) { + e->kind = Lit_kind; + e->v.Lit.v = FORBIDDEN_VALUES[i]; + return 1; + } + } + return 0; +} + /* Set the context ctx for expr_ty e, recursively traversing e. Only sets context for expr kinds that "can appear in assignment context" @@ -481,13 +507,14 @@ break; case Dict_kind: case Set_kind: - case Num_kind: - case Str_kind: + case Lit_kind: + /* Tweak error message */ + if (e->v.Lit.v == Py_None || + e->v.Lit.v == Py_True || + e->v.Lit.v == Py_False) + return ast_error(n, "assignment to keyword"); expr_name = "literal"; break; - case Ellipsis_kind: - expr_name = "Ellipsis"; - break; case Compare_kind: expr_name = "comparison"; break; @@ -1026,6 +1053,7 @@ arguments_ty args; asdl_seq *body; expr_ty returns = NULL; + string docstring; int name_i = 1; REQ(n, funcdef); @@ -1044,12 +1072,12 @@ return NULL; name_i += 2; } - body = ast_for_suite(c, CHILD(n, name_i + 3)); + body = ast_for_body(c, CHILD(n, name_i + 3), &docstring); if (!body) return NULL; - return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n), - n->n_col_offset, c->c_arena); + return FunctionDef(name, args, body, decorator_seq, returns, + docstring, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -1357,10 +1385,13 @@ case NAME: { /* All names start in Load context, but may later be changed. */ + expr_ty e; PyObject *name = NEW_IDENTIFIER(ch); if (!name) return NULL; - return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); + e = Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); + make_name_literal(e); + return e; } case STRING: { PyObject *str = parsestrplus(c, n, &bytesmode); @@ -1386,10 +1417,7 @@ return NULL; } PyArena_AddPyObject(c->c_arena, str); - if (bytesmode) - return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena); - else - return Str(str, LINENO(n), n->n_col_offset, c->c_arena); + return Lit(str, LINENO(n), n->n_col_offset, c->c_arena); } case NUMBER: { PyObject *pynum = parsenumber(c, STR(ch)); @@ -1397,10 +1425,10 @@ return NULL; PyArena_AddPyObject(c->c_arena, pynum); - return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); + return Lit(pynum, LINENO(n), n->n_col_offset, c->c_arena); } case ELLIPSIS: /* Ellipsis */ - return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena); + return Lit(Py_Ellipsis, LINENO(n), n->n_col_offset, c->c_arena); case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); @@ -2633,6 +2661,40 @@ return seq; } +static string +docstring_from_stmts(asdl_seq *stmts) +{ + /* XXX not very efficient, but simple */ + if (stmts && stmts->size) { + stmt_ty s; + + s = (stmt_ty)asdl_seq_GET(stmts, 0); + if (s->kind == Expr_kind && + s->v.Expr.value->kind == Lit_kind && + PyUnicode_CheckExact(s->v.Expr.value->v.Lit.v)) { + /* If first statement is a literal string, it's the + doc string. */ + string doc = s->v.Expr.value->v.Lit.v; + memmove(&asdl_seq_GET(stmts, 0), &asdl_seq_GET(stmts, 1), + (stmts->size - 1) * sizeof(void*)); + stmts->size--; + return doc; + } + } + + return NULL; +} + +static asdl_seq * +ast_for_body(struct compiling *c, const node *n, string *docstring) +{ + asdl_seq *stmts; + + stmts = ast_for_suite(c, n); + *docstring = docstring_from_stmts(stmts); + return stmts; +} + static stmt_ty ast_for_if_stmt(struct compiling *c, const node *n) { @@ -3031,12 +3093,13 @@ /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */ PyObject *classname; asdl_seq *s; + string docstring; expr_ty call; REQ(n, classdef); if (NCH(n) == 4) { /* class NAME ':' suite */ - s = ast_for_suite(c, CHILD(n, 3)); + s = ast_for_body(c, CHILD(n, 3), &docstring); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); @@ -3045,11 +3108,11 @@ if (forbidden_name(classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, c->c_arena); + docstring, LINENO(n), n->n_col_offset, c->c_arena); } if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ - s = ast_for_suite(c, CHILD(n,5)); + s = ast_for_body(c, CHILD(n,5), &docstring); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); @@ -3058,7 +3121,7 @@ if (forbidden_name(classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, c->c_arena); + docstring, LINENO(n), n->n_col_offset, c->c_arena); } /* class NAME '(' arglist ')' ':' suite */ @@ -3074,7 +3137,7 @@ if (!call) return NULL; } - s = ast_for_suite(c, CHILD(n, 6)); + s = ast_for_body(c, CHILD(n, 6), &docstring); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); @@ -3085,7 +3148,8 @@ return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, call->v.Call.starargs, call->v.Call.kwargs, s, - decorator_seq, LINENO(n), n->n_col_offset, c->c_arena); + decorator_seq, docstring, LINENO(n), n->n_col_offset, + c->c_arena); } static stmt_ty diff -r 336137a359ae Python/future.c --- a/Python/future.c Fri Mar 11 10:27:14 2011 -0500 +++ b/Python/future.c Mon Mar 14 23:22:44 2011 -0400 @@ -58,7 +58,7 @@ static int future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename) { - int i, found_docstring = 0, done = 0, prev_line = 0; + int i, done = 0, prev_line = 0; static PyObject *future; if (!future) { @@ -107,12 +107,8 @@ else done = 1; } - else if (s->kind == Expr_kind && !found_docstring) { - expr_ty e = s->v.Expr.value; - if (e->kind != Str_kind) - done = 1; - else - found_docstring = 1; + else if (s->kind == Expr_kind) { + done = 1; } else done = 1; diff -r 336137a359ae Python/symtable.c --- a/Python/symtable.c Fri Mar 11 10:27:14 2011 -0500 +++ b/Python/symtable.c Mon Mar 14 23:22:44 2011 -0400 @@ -1397,10 +1397,7 @@ if (e->v.Call.kwargs) VISIT(st, expr, e->v.Call.kwargs); break; - case Num_kind: - case Str_kind: - case Bytes_kind: - case Ellipsis_kind: + case Lit_kind: /* Nothing to do here. */ break; /* The following exprs can be assignment targets. */