Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(8)

Unified Diff: Python/ast.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 4 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « Parser/tokenizer.h ('k') | Python/pythonrun.c » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -606,7 +606,7 @@ ast_error(struct compiling *c, const nod
grammar:
stmt: simple_stmt | compound_stmt
- simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+ simple_stmt: small_stmt (';' small_stmt)* [';'] PYTOK_NEWLINE
A simple_stmt can contain multiple small_stmt elements joined
by semicolons. If the arg is a simple_stmt, the number of
@@ -621,7 +621,7 @@ num_stmts(const node *n)
switch (TYPE(n)) {
case single_input:
- if (TYPE(CHILD(n, 0)) == NEWLINE)
+ if (TYPE(CHILD(n, 0)) == PYTOK_NEWLINE)
return 0;
else
return num_stmts(CHILD(n, 0));
@@ -703,7 +703,7 @@ PyAST_FromNodeObject(const node *n, PyCo
goto out;
for (i = 0; i < NCH(n) - 1; i++) {
ch = CHILD(n, i);
- if (TYPE(ch) == NEWLINE)
+ if (TYPE(ch) == PYTOK_NEWLINE)
continue;
REQ(ch, stmt);
num = num_stmts(ch);
@@ -737,7 +737,7 @@ PyAST_FromNodeObject(const node *n, PyCo
break;
}
case single_input:
- if (TYPE(CHILD(n, 0)) == NEWLINE) {
+ if (TYPE(CHILD(n, 0)) == PYTOK_NEWLINE) {
stmts = _Py_asdl_seq_new(1, arena);
if (!stmts)
goto out;
@@ -763,7 +763,7 @@ PyAST_FromNodeObject(const node *n, PyCo
/* Only a simple_stmt can contain multiple statements. */
REQ(n, simple_stmt);
for (i = 0; i < NCH(n); i += 2) {
- if (TYPE(CHILD(n, i)) == NEWLINE)
+ if (TYPE(CHILD(n, i)) == PYTOK_NEWLINE)
break;
s = ast_for_stmt(&c, CHILD(n, i));
if (!s)
@@ -811,29 +811,29 @@ static operator_ty
get_operator(const node *n)
{
switch (TYPE(n)) {
- case VBAR:
+ case PYTOK_VBAR:
return BitOr;
- case CIRCUMFLEX:
+ case PYTOK_CIRCUMFLEX:
return BitXor;
- case AMPER:
+ case PYTOK_AMPER:
return BitAnd;
- case LEFTSHIFT:
+ case PYTOK_LEFTSHIFT:
return LShift;
- case RIGHTSHIFT:
+ case PYTOK_RIGHTSHIFT:
return RShift;
- case PLUS:
+ case PYTOK_PLUS:
return Add;
- case MINUS:
+ case PYTOK_MINUS:
return Sub;
- case STAR:
+ case PYTOK_STAR:
return Mult;
- case AT:
+ case PYTOK_AT:
return MatMult;
- case SLASH:
+ case PYTOK_SLASH:
return Div;
- case DOUBLESLASH:
+ case PYTOK_DOUBLESLASH:
return FloorDiv;
- case PERCENT:
+ case PYTOK_PERCENT:
return Mod;
default:
return (operator_ty)0;
@@ -1052,19 +1052,19 @@ ast_for_comp_op(struct compiling *c, con
if (NCH(n) == 1) {
n = CHILD(n, 0);
switch (TYPE(n)) {
- case LESS:
+ case PYTOK_LESS:
return Lt;
- case GREATER:
+ case PYTOK_GREATER:
return Gt;
- case EQEQUAL: /* == */
+ case PYTOK_EQEQUAL: /* == */
return Eq;
- case LESSEQUAL:
+ case PYTOK_LESSEQUAL:
return LtE;
- case GREATEREQUAL:
+ case PYTOK_GREATEREQUAL:
return GtE;
- case NOTEQUAL:
+ case PYTOK_NOTEQUAL:
return NotEq;
- case NAME:
+ case PYTOK_NAME:
if (strcmp(STR(n), "in") == 0)
return In;
if (strcmp(STR(n), "is") == 0)
@@ -1078,7 +1078,7 @@ ast_for_comp_op(struct compiling *c, con
else if (NCH(n) == 2) {
/* handle "not in" and "is not" */
switch (TYPE(CHILD(n, 0))) {
- case NAME:
+ case PYTOK_NAME:
if (strcmp(STR(CHILD(n, 1)), "in") == 0)
return NotIn;
if (strcmp(STR(CHILD(n, 0)), "is") == 0)
@@ -1139,7 +1139,7 @@ ast_for_arg(struct compiling *c, const n
if (forbidden_name(c, name, ch, 0))
return NULL;
- if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) {
+ if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == PYTOK_COLON) {
annotation = ast_for_expr(c, CHILD(n, 2));
if (!annotation)
return NULL;
@@ -1180,7 +1180,7 @@ handle_keywordonly_args(struct compiling
switch (TYPE(ch)) {
case vfpdef:
case tfpdef:
- if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
+ if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == PYTOK_EQUAL) {
expression = ast_for_expr(c, CHILD(n, i + 2));
if (!expression)
goto error;
@@ -1191,7 +1191,7 @@ handle_keywordonly_args(struct compiling
asdl_seq_SET(kwdefaults, j, NULL);
}
if (NCH(ch) == 3) {
- /* ch is NAME ':' test */
+ /* ch is PYTOK_NAME ':' test */
annotation = ast_for_expr(c, CHILD(ch, 2));
if (!annotation)
goto error;
@@ -1213,7 +1213,7 @@ handle_keywordonly_args(struct compiling
asdl_seq_SET(kwonlyargs, j++, arg);
i += 2; /* the name and the comma */
break;
- case DOUBLESTAR:
+ case PYTOK_DOUBLESTAR:
return i;
default:
ast_error(c, ch, "unexpected node");
@@ -1238,12 +1238,12 @@ ast_for_arguments(struct compiling *c, c
('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef]
| '**' tfpdef)
| tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
- tfpdef: NAME [':' test]
+ tfpdef: PYTOK_NAME [':' test]
varargslist: ((vfpdef ['=' test] ',')*
('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef]
| '**' vfpdef)
| vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
- vfpdef: NAME
+ vfpdef: PYTOK_NAME
*/
int i, j, k, nposargs = 0, nkwonlyargs = 0;
int nposdefaults = 0, found_default = 0;
@@ -1265,7 +1265,7 @@ ast_for_arguments(struct compiling *c, c
*/
for (i = 0; i < NCH(n); i++) {
ch = CHILD(n, i);
- if (TYPE(ch) == STAR) {
+ if (TYPE(ch) == PYTOK_STAR) {
/* skip star */
i++;
if (i < NCH(n) && /* skip argument following star */
@@ -1275,15 +1275,15 @@ ast_for_arguments(struct compiling *c, c
}
break;
}
- if (TYPE(ch) == DOUBLESTAR) break;
+ if (TYPE(ch) == PYTOK_DOUBLESTAR) break;
if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++;
- if (TYPE(ch) == EQUAL) nposdefaults++;
+ if (TYPE(ch) == PYTOK_EQUAL) nposdefaults++;
}
/* count the number of keyword only args &
defaults for keyword only args */
for ( ; i < NCH(n); ++i) {
ch = CHILD(n, i);
- if (TYPE(ch) == DOUBLESTAR) break;
+ if (TYPE(ch) == PYTOK_DOUBLESTAR) break;
if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++;
}
posargs = (nposargs ? _Py_asdl_seq_new(nposargs, c->c_arena) : NULL);
@@ -1310,8 +1310,8 @@ ast_for_arguments(struct compiling *c, c
return NULL;
}
- /* tfpdef: NAME [':' test]
- vfpdef: NAME
+ /* tfpdef: PYTOK_NAME [':' test]
+ vfpdef: PYTOK_NAME
*/
i = 0;
j = 0; /* index for defaults */
@@ -1322,9 +1322,9 @@ ast_for_arguments(struct compiling *c, c
case tfpdef:
case vfpdef:
/* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
- anything other than EQUAL or a comma? */
+ anything other than PYTOK_EQUAL or a comma? */
/* XXX Should NCH(n) check be made a separate check? */
- if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
+ if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == PYTOK_EQUAL) {
expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));
if (!expression)
return NULL;
@@ -1344,14 +1344,14 @@ ast_for_arguments(struct compiling *c, c
asdl_seq_SET(posargs, k++, arg);
i += 2; /* the name and the comma */
break;
- case STAR:
+ case PYTOK_STAR:
if (i+1 >= NCH(n)) {
ast_error(c, CHILD(n, i),
"named arguments must follow bare *");
return NULL;
}
- ch = CHILD(n, i+1); /* tfpdef or COMMA */
- if (TYPE(ch) == COMMA) {
+ ch = CHILD(n, i+1); /* tfpdef or PYTOK_COMMA */
+ if (TYPE(ch) == PYTOK_COMMA) {
int res = 0;
i += 2; /* now follows keyword only arguments */
res = handle_keywordonly_args(c, n, i,
@@ -1375,7 +1375,7 @@ ast_for_arguments(struct compiling *c, c
}
}
break;
- case DOUBLESTAR:
+ case PYTOK_DOUBLESTAR:
ch = CHILD(n, i+1); /* tfpdef */
assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);
kwarg = ast_for_arg(c, ch);
@@ -1428,13 +1428,13 @@ ast_for_dotted_name(struct compiling *c,
static expr_ty
ast_for_decorator(struct compiling *c, const node *n)
{
- /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */
+ /* decorator: '@' dotted_name [ '(' [arglist] ')' ] PYTOK_NEWLINE */
expr_ty d = NULL;
expr_ty name_expr;
REQ(n, decorator);
- REQ(CHILD(n, 0), AT);
- REQ(RCHILD(n, -1), NEWLINE);
+ REQ(CHILD(n, 0), PYTOK_AT);
+ REQ(RCHILD(n, -1), PYTOK_NEWLINE);
name_expr = ast_for_dotted_name(c, CHILD(n, 1));
if (!name_expr)
@@ -1485,7 +1485,7 @@ ast_for_decorators(struct compiling *c,
static stmt_ty
ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
- /* funcdef: 'def' NAME parameters ['->' test] ':' suite */
+ /* funcdef: 'def' PYTOK_NAME parameters ['->' test] ':' suite */
identifier name;
arguments_ty args;
asdl_seq *body;
@@ -1502,7 +1502,7 @@ ast_for_funcdef(struct compiling *c, con
args = ast_for_arguments(c, CHILD(n, name_i + 1));
if (!args)
return NULL;
- if (TYPE(CHILD(n, name_i+2)) == RARROW) {
+ if (TYPE(CHILD(n, name_i+2)) == PYTOK_RARROW) {
returns = ast_for_expr(c, CHILD(n, name_i + 3));
if (!returns)
return NULL;
@@ -1768,7 +1768,7 @@ ast_for_dictcomp(struct compiling *c, co
asdl_seq *comps;
assert(NCH(n) > 3);
- REQ(CHILD(n, 1), COLON);
+ REQ(CHILD(n, 1), PYTOK_COLON);
key = ast_for_expr(c, CHILD(n, 0));
if (!key)
@@ -1810,14 +1810,14 @@ static expr_ty
ast_for_atom(struct compiling *c, const node *n)
{
/* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
- | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+
+ | '{' [dictmaker|testlist_comp] '}' | PYTOK_NAME | PYTOK_NUMBER | PYTOK_STRING+
| '...' | 'None' | 'True' | 'False'
*/
node *ch = CHILD(n, 0);
int bytesmode = 0;
switch (TYPE(ch)) {
- case NAME: {
+ case PYTOK_NAME: {
PyObject *name;
const char *s = STR(ch);
size_t len = strlen(s);
@@ -1835,7 +1835,7 @@ ast_for_atom(struct compiling *c, const
/* All names start in Load context, but may later be changed. */
return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
- case STRING: {
+ case PYTOK_STRING: {
PyObject *str = parsestrplus(c, n, &bytesmode);
if (!str) {
const char *errtype = NULL;
@@ -1871,7 +1871,7 @@ ast_for_atom(struct compiling *c, const
else
return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
}
- case NUMBER: {
+ case PYTOK_NUMBER: {
PyObject *pynum = parsenumber(c, STR(ch));
if (!pynum)
return NULL;
@@ -1882,12 +1882,12 @@ ast_for_atom(struct compiling *c, const
}
return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
}
- case ELLIPSIS: /* Ellipsis */
+ case PYTOK_ELLIPSIS: /* Ellipsis */
return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
- case LPAR: /* some parenthesized expressions */
+ case PYTOK_LPAR: /* some parenthesized expressions */
ch = CHILD(n, 1);
- if (TYPE(ch) == RPAR)
+ if (TYPE(ch) == PYTOK_RPAR)
return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
if (TYPE(ch) == yield_expr)
@@ -1898,14 +1898,14 @@ ast_for_atom(struct compiling *c, const
return ast_for_genexp(c, ch);
return ast_for_testlist(c, ch);
- case LSQB: /* list (or list comprehension) */
+ case PYTOK_LSQB: /* list (or list comprehension) */
ch = CHILD(n, 1);
- if (TYPE(ch) == RSQB)
+ if (TYPE(ch) == PYTOK_RSQB)
return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
REQ(ch, testlist_comp);
- if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
+ if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == PYTOK_COMMA) {
asdl_seq *elts = seq_for_testlist(c, ch);
if (!elts)
return NULL;
@@ -1914,17 +1914,17 @@ ast_for_atom(struct compiling *c, const
}
else
return ast_for_listcomp(c, ch);
- case LBRACE: {
+ case PYTOK_LBRACE: {
/* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
* test (gen_for | (',' test)* [',']) */
int i, size;
asdl_seq *keys, *values;
ch = CHILD(n, 1);
- if (TYPE(ch) == RBRACE) {
+ if (TYPE(ch) == PYTOK_RBRACE) {
/* it's an empty dict */
return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
- } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
+ } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == PYTOK_COMMA) {
/* it's a simple set */
asdl_seq *elts;
size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
@@ -2009,7 +2009,7 @@ ast_for_slice(struct compiling *c, const
}
/* If there's an upper bound it's in the second or third position. */
- if (TYPE(ch) == COLON) {
+ if (TYPE(ch) == PYTOK_COLON) {
if (NCH(n) > 1) {
node *n2 = CHILD(n, 1);
@@ -2099,19 +2099,19 @@ ast_for_binop(struct compiling *c, const
static expr_ty
ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
{
- /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+ /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' PYTOK_NAME
subscriptlist: subscript (',' subscript)* [',']
subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
*/
REQ(n, trailer);
- if (TYPE(CHILD(n, 0)) == LPAR) {
+ if (TYPE(CHILD(n, 0)) == PYTOK_LPAR) {
if (NCH(n) == 2)
return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n),
n->n_col_offset, c->c_arena);
else
return ast_for_call(c, CHILD(n, 1), left_expr);
}
- else if (TYPE(CHILD(n, 0)) == DOT) {
+ else if (TYPE(CHILD(n, 0)) == PYTOK_DOT) {
PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1));
if (!attr_id)
return NULL;
@@ -2119,8 +2119,8 @@ ast_for_trailer(struct compiling *c, con
LINENO(n), n->n_col_offset, c->c_arena);
}
else {
- REQ(CHILD(n, 0), LSQB);
- REQ(CHILD(n, 2), RSQB);
+ REQ(CHILD(n, 0), PYTOK_LSQB);
+ REQ(CHILD(n, 2), PYTOK_RSQB);
n = CHILD(n, 1);
if (NCH(n) == 1) {
slice_ty slc = ast_for_slice(c, CHILD(n, 0));
@@ -2182,13 +2182,13 @@ ast_for_factor(struct compiling *c, cons
return NULL;
switch (TYPE(CHILD(n, 0))) {
- case PLUS:
+ case PYTOK_PLUS:
return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset,
c->c_arena);
- case MINUS:
+ case PYTOK_MINUS:
return UnaryOp(USub, expression, LINENO(n), n->n_col_offset,
c->c_arena);
- case TILDE:
+ case PYTOK_TILDE:
return UnaryOp(Invert, expression, LINENO(n),
n->n_col_offset, c->c_arena);
}
@@ -2526,13 +2526,13 @@ ast_for_call(struct compiling *c, const
asdl_seq_SET(keywords, nkeywords++, kw);
}
}
- else if (TYPE(ch) == STAR) {
+ else if (TYPE(ch) == PYTOK_STAR) {
vararg = ast_for_expr(c, CHILD(n, i+1));
if (!vararg)
return NULL;
i++;
}
- else if (TYPE(ch) == DOUBLESTAR) {
+ else if (TYPE(ch) == PYTOK_DOUBLESTAR) {
kwarg = ast_for_expr(c, CHILD(n, i+1));
if (!kwarg)
return NULL;
@@ -2631,7 +2631,7 @@ ast_for_expr_stmt(struct compiling *c, c
expr_ty expression;
/* a normal assignment */
- REQ(CHILD(n, 1), EQUAL);
+ REQ(CHILD(n, 1), PYTOK_EQUAL);
targets = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena);
if (!targets)
return NULL;
@@ -2767,9 +2767,9 @@ static alias_ty
alias_for_import_name(struct compiling *c, const node *n, int store)
{
/*
- import_as_name: NAME ['as' NAME]
- dotted_as_name: dotted_name ['as' NAME]
- dotted_name: NAME ('.' NAME)*
+ import_as_name: PYTOK_NAME ['as' PYTOK_NAME]
+ dotted_as_name: dotted_name ['as' PYTOK_NAME]
+ dotted_name: PYTOK_NAME ('.' PYTOK_NAME)*
*/
identifier str, name;
@@ -2865,7 +2865,7 @@ alias_for_import_name(struct compiling *
return alias(str, NULL, c->c_arena);
}
break;
- case STAR:
+ case PYTOK_STAR:
str = PyUnicode_InternFromString("*");
if (PyArena_AddPyObject(c->c_arena, str) < 0) {
Py_DECREF(str);
@@ -2929,23 +2929,23 @@ ast_for_import_stmt(struct compiling *c,
return NULL;
idx++;
break;
- } else if (TYPE(CHILD(n, idx)) == ELLIPSIS) {
- /* three consecutive dots are tokenized as one ELLIPSIS */
+ } else if (TYPE(CHILD(n, idx)) == PYTOK_ELLIPSIS) {
+ /* three consecutive dots are tokenized as one PYTOK_ELLIPSIS */
ndots += 3;
continue;
- } else if (TYPE(CHILD(n, idx)) != DOT) {
+ } else if (TYPE(CHILD(n, idx)) != PYTOK_DOT) {
break;
}
ndots++;
}
idx++; /* skip over the 'import' keyword */
switch (TYPE(CHILD(n, idx))) {
- case STAR:
+ case PYTOK_STAR:
/* from ... import * */
n = CHILD(n, idx);
n_children = 1;
break;
- case LPAR:
+ case PYTOK_LPAR:
/* from ... import (x, y, z) */
n = CHILD(n, idx + 1);
n_children = NCH(n);
@@ -2970,7 +2970,7 @@ ast_for_import_stmt(struct compiling *c,
return NULL;
/* handle "from ... import *" special b/c there's no children */
- if (TYPE(n) == STAR) {
+ if (TYPE(n) == PYTOK_STAR) {
alias_ty import_alias = alias_for_import_name(c, n, 1);
if (!import_alias)
return NULL;
@@ -2998,7 +2998,7 @@ ast_for_import_stmt(struct compiling *c,
static stmt_ty
ast_for_global_stmt(struct compiling *c, const node *n)
{
- /* global_stmt: 'global' NAME (',' NAME)* */
+ /* global_stmt: 'global' PYTOK_NAME (',' PYTOK_NAME)* */
identifier name;
asdl_seq *s;
int i;
@@ -3019,7 +3019,7 @@ ast_for_global_stmt(struct compiling *c,
static stmt_ty
ast_for_nonlocal_stmt(struct compiling *c, const node *n)
{
- /* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* */
+ /* nonlocal_stmt: 'nonlocal' PYTOK_NAME (',' PYTOK_NAME)* */
identifier name;
asdl_seq *s;
int i;
@@ -3069,7 +3069,7 @@ ast_for_assert_stmt(struct compiling *c,
static asdl_seq *
ast_for_suite(struct compiling *c, const node *n)
{
- /* suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT */
+ /* suite: simple_stmt | PYTOK_NEWLINE PYTOK_INDENT stmt+ PYTOK_DEDENT */
asdl_seq *seq;
stmt_ty s;
int i, total, num, end, pos = 0;
@@ -3083,11 +3083,11 @@ ast_for_suite(struct compiling *c, const
return NULL;
if (TYPE(CHILD(n, 0)) == simple_stmt) {
n = CHILD(n, 0);
- /* simple_stmt always ends with a NEWLINE,
- and may have a trailing SEMI
+ /* simple_stmt always ends with a PYTOK_NEWLINE,
+ and may have a trailing PYTOK_SEMI
*/
end = NCH(n) - 1;
- if (TYPE(CHILD(n, end - 1)) == SEMI)
+ if (TYPE(CHILD(n, end - 1)) == PYTOK_SEMI)
end--;
/* loop by 2 to skip semi-colons */
for (i = 0; i < end; i += 2) {
@@ -3187,7 +3187,7 @@ ast_for_if_stmt(struct compiling *c, con
n_elif = NCH(n) - 4;
/* must reference the child n_elif+1 since 'else' token is third,
not fourth, child from the end. */
- if (TYPE(CHILD(n, (n_elif + 1))) == NAME
+ if (TYPE(CHILD(n, (n_elif + 1))) == PYTOK_NAME
&& STR(CHILD(n, (n_elif + 1)))[2] == 's') {
has_else = 1;
n_elif -= 3;
@@ -3399,9 +3399,9 @@ ast_for_try_stmt(struct compiling *c, co
if (body == NULL)
return NULL;
- if (TYPE(CHILD(n, nch - 3)) == NAME) {
+ if (TYPE(CHILD(n, nch - 3)) == PYTOK_NAME) {
if (strcmp(STR(CHILD(n, nch - 3)), "finally") == 0) {
- if (nch >= 9 && TYPE(CHILD(n, nch - 6)) == NAME) {
+ if (nch >= 9 && TYPE(CHILD(n, nch - 6)) == PYTOK_NAME) {
/* we can assume it's an "else",
because nch >= 9 for try-else-finally and
it would otherwise have a type of except_clause */
@@ -3504,14 +3504,14 @@ ast_for_with_stmt(struct compiling *c, c
static stmt_ty
ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
- /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */
+ /* classdef: 'class' PYTOK_NAME ['(' arglist ')'] ':' suite */
PyObject *classname;
asdl_seq *s;
expr_ty call;
REQ(n, classdef);
- if (NCH(n) == 4) { /* class NAME ':' suite */
+ if (NCH(n) == 4) { /* class PYTOK_NAME ':' suite */
s = ast_for_suite(c, CHILD(n, 3));
if (!s)
return NULL;
@@ -3524,7 +3524,7 @@ ast_for_classdef(struct compiling *c, co
LINENO(n), n->n_col_offset, c->c_arena);
}
- if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
+ if (TYPE(CHILD(n, 3)) == PYTOK_RPAR) { /* class PYTOK_NAME '(' ')' ':' suite */
s = ast_for_suite(c, CHILD(n,5));
if (!s)
return NULL;
@@ -3537,7 +3537,7 @@ ast_for_classdef(struct compiling *c, co
LINENO(n), n->n_col_offset, c->c_arena);
}
- /* class NAME '(' arglist ')' ':' suite */
+ /* class PYTOK_NAME '(' arglist ')' ':' suite */
/* build up a fake Call node so we can extract its pieces */
{
PyObject *dummy_name;
@@ -3844,7 +3844,7 @@ parsestr(struct compiling *c, const node
need_encoding ? c->c_encoding : NULL);
}
-/* Build a Python string object out of a STRING+ atom. This takes care of
+/* Build a Python string object out of a PYTOK_STRING+ atom. This takes care of
* compile-time literal catenation, calling parsestr() on each piece, and
* pasting the intermediate results together.
*/
@@ -3853,7 +3853,7 @@ parsestrplus(struct compiling *c, const
{
PyObject *v;
int i;
- REQ(CHILD(n, 0), STRING);
+ REQ(CHILD(n, 0), PYTOK_STRING);
v = parsestr(c, CHILD(n, 0), bytesmode);
if (v != NULL) {
/* String literal concatenation */
« no previous file with comments | « Parser/tokenizer.h ('k') | Python/pythonrun.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+