Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(2)

Delta Between Two Patch Sets: Python/ast.c

Issue 3353: make built-in tokenizer available via Python C API
Left Patch Set: Created 4 years, 10 months ago
Right Patch Set: Created 4 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « Parser/tokenizer.h ('k') | Python/pythonrun.c » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 /* 1 /*
2 * This file includes functions to transform a concrete syntax tree (CST) to 2 * This file includes functions to transform a concrete syntax tree (CST) to
3 * an abstract syntax tree (AST). The main function is PyAST_FromNode(). 3 * an abstract syntax tree (AST). The main function is PyAST_FromNode().
4 * 4 *
5 */ 5 */
6 #include "Python.h" 6 #include "Python.h"
7 #include "Python-ast.h" 7 #include "Python-ast.h"
8 #include "node.h" 8 #include "node.h"
9 #include "ast.h" 9 #include "ast.h"
10 #include "token.h" 10 #include "token.h"
(...skipping 588 matching lines...) Expand 10 before | Expand all | Expand 10 after
599 return 0; 599 return 0;
600 } 600 }
601 601
602 /* num_stmts() returns number of contained statements. 602 /* num_stmts() returns number of contained statements.
603 603
604 Use this routine to determine how big a sequence is needed for 604 Use this routine to determine how big a sequence is needed for
605 the statements in a parse tree. Its raison d'etre is this bit of 605 the statements in a parse tree. Its raison d'etre is this bit of
606 grammar: 606 grammar:
607 607
608 stmt: simple_stmt | compound_stmt 608 stmt: simple_stmt | compound_stmt
609 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 609 simple_stmt: small_stmt (';' small_stmt)* [';'] PYTOK_NEWLINE
610 610
611 A simple_stmt can contain multiple small_stmt elements joined 611 A simple_stmt can contain multiple small_stmt elements joined
612 by semicolons. If the arg is a simple_stmt, the number of 612 by semicolons. If the arg is a simple_stmt, the number of
613 small_stmt elements is returned. 613 small_stmt elements is returned.
614 */ 614 */
615 615
616 static int 616 static int
617 num_stmts(const node *n) 617 num_stmts(const node *n)
618 { 618 {
619 int i, l; 619 int i, l;
620 node *ch; 620 node *ch;
621 621
622 switch (TYPE(n)) { 622 switch (TYPE(n)) {
623 case single_input: 623 case single_input:
624 if (TYPE(CHILD(n, 0)) == NEWLINE) 624 if (TYPE(CHILD(n, 0)) == PYTOK_NEWLINE)
625 return 0; 625 return 0;
626 else 626 else
627 return num_stmts(CHILD(n, 0)); 627 return num_stmts(CHILD(n, 0));
628 case file_input: 628 case file_input:
629 l = 0; 629 l = 0;
630 for (i = 0; i < NCH(n); i++) { 630 for (i = 0; i < NCH(n); i++) {
631 ch = CHILD(n, i); 631 ch = CHILD(n, i);
632 if (TYPE(ch) == stmt) 632 if (TYPE(ch) == stmt)
633 l += num_stmts(ch); 633 l += num_stmts(ch);
634 } 634 }
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
696 } 696 }
697 697
698 k = 0; 698 k = 0;
699 switch (TYPE(n)) { 699 switch (TYPE(n)) {
700 case file_input: 700 case file_input:
701 stmts = _Py_asdl_seq_new(num_stmts(n), arena); 701 stmts = _Py_asdl_seq_new(num_stmts(n), arena);
702 if (!stmts) 702 if (!stmts)
703 goto out; 703 goto out;
704 for (i = 0; i < NCH(n) - 1; i++) { 704 for (i = 0; i < NCH(n) - 1; i++) {
705 ch = CHILD(n, i); 705 ch = CHILD(n, i);
706 if (TYPE(ch) == NEWLINE) 706 if (TYPE(ch) == PYTOK_NEWLINE)
707 continue; 707 continue;
708 REQ(ch, stmt); 708 REQ(ch, stmt);
709 num = num_stmts(ch); 709 num = num_stmts(ch);
710 if (num == 1) { 710 if (num == 1) {
711 s = ast_for_stmt(&c, ch); 711 s = ast_for_stmt(&c, ch);
712 if (!s) 712 if (!s)
713 goto out; 713 goto out;
714 asdl_seq_SET(stmts, k++, s); 714 asdl_seq_SET(stmts, k++, s);
715 } 715 }
716 else { 716 else {
(...skipping 13 matching lines...) Expand all
730 expr_ty testlist_ast; 730 expr_ty testlist_ast;
731 731
732 /* XXX Why not comp_for here? */ 732 /* XXX Why not comp_for here? */
733 testlist_ast = ast_for_testlist(&c, CHILD(n, 0)); 733 testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
734 if (!testlist_ast) 734 if (!testlist_ast)
735 goto out; 735 goto out;
736 res = Expression(testlist_ast, arena); 736 res = Expression(testlist_ast, arena);
737 break; 737 break;
738 } 738 }
739 case single_input: 739 case single_input:
740 if (TYPE(CHILD(n, 0)) == NEWLINE) { 740 if (TYPE(CHILD(n, 0)) == PYTOK_NEWLINE) {
741 stmts = _Py_asdl_seq_new(1, arena); 741 stmts = _Py_asdl_seq_new(1, arena);
742 if (!stmts) 742 if (!stmts)
743 goto out; 743 goto out;
744 asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, 744 asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,
745 arena)); 745 arena));
746 if (!asdl_seq_GET(stmts, 0)) 746 if (!asdl_seq_GET(stmts, 0))
747 goto out; 747 goto out;
748 res = Interactive(stmts, arena); 748 res = Interactive(stmts, arena);
749 } 749 }
750 else { 750 else {
751 n = CHILD(n, 0); 751 n = CHILD(n, 0);
752 num = num_stmts(n); 752 num = num_stmts(n);
753 stmts = _Py_asdl_seq_new(num, arena); 753 stmts = _Py_asdl_seq_new(num, arena);
754 if (!stmts) 754 if (!stmts)
755 goto out; 755 goto out;
756 if (num == 1) { 756 if (num == 1) {
757 s = ast_for_stmt(&c, n); 757 s = ast_for_stmt(&c, n);
758 if (!s) 758 if (!s)
759 goto out; 759 goto out;
760 asdl_seq_SET(stmts, 0, s); 760 asdl_seq_SET(stmts, 0, s);
761 } 761 }
762 else { 762 else {
763 /* Only a simple_stmt can contain multiple statements. */ 763 /* Only a simple_stmt can contain multiple statements. */
764 REQ(n, simple_stmt); 764 REQ(n, simple_stmt);
765 for (i = 0; i < NCH(n); i += 2) { 765 for (i = 0; i < NCH(n); i += 2) {
766 if (TYPE(CHILD(n, i)) == NEWLINE) 766 if (TYPE(CHILD(n, i)) == PYTOK_NEWLINE)
767 break; 767 break;
768 s = ast_for_stmt(&c, CHILD(n, i)); 768 s = ast_for_stmt(&c, CHILD(n, i));
769 if (!s) 769 if (!s)
770 goto out; 770 goto out;
771 asdl_seq_SET(stmts, i / 2, s); 771 asdl_seq_SET(stmts, i / 2, s);
772 } 772 }
773 } 773 }
774 774
775 res = Interactive(stmts, arena); 775 res = Interactive(stmts, arena);
776 } 776 }
(...skipping 27 matching lines...) Expand all
804 804
805 } 805 }
806 806
807 /* Return the AST repr. of the operator represented as syntax (|, ^, etc.) 807 /* Return the AST repr. of the operator represented as syntax (|, ^, etc.)
808 */ 808 */
809 809
810 static operator_ty 810 static operator_ty
811 get_operator(const node *n) 811 get_operator(const node *n)
812 { 812 {
813 switch (TYPE(n)) { 813 switch (TYPE(n)) {
814 case VBAR: 814 case PYTOK_VBAR:
815 return BitOr; 815 return BitOr;
816 case CIRCUMFLEX: 816 case PYTOK_CIRCUMFLEX:
817 return BitXor; 817 return BitXor;
818 case AMPER: 818 case PYTOK_AMPER:
819 return BitAnd; 819 return BitAnd;
820 case LEFTSHIFT: 820 case PYTOK_LEFTSHIFT:
821 return LShift; 821 return LShift;
822 case RIGHTSHIFT: 822 case PYTOK_RIGHTSHIFT:
823 return RShift; 823 return RShift;
824 case PLUS: 824 case PYTOK_PLUS:
825 return Add; 825 return Add;
826 case MINUS: 826 case PYTOK_MINUS:
827 return Sub; 827 return Sub;
828 case STAR: 828 case PYTOK_STAR:
829 return Mult; 829 return Mult;
830 case AT: 830 case PYTOK_AT:
831 return MatMult; 831 return MatMult;
832 case SLASH: 832 case PYTOK_SLASH:
833 return Div; 833 return Div;
834 case DOUBLESLASH: 834 case PYTOK_DOUBLESLASH:
835 return FloorDiv; 835 return FloorDiv;
836 case PERCENT: 836 case PYTOK_PERCENT:
837 return Mod; 837 return Mod;
838 default: 838 default:
839 return (operator_ty)0; 839 return (operator_ty)0;
840 } 840 }
841 } 841 }
842 842
843 static const char* FORBIDDEN[] = { 843 static const char* FORBIDDEN[] = {
844 "None", 844 "None",
845 "True", 845 "True",
846 "False", 846 "False",
(...skipping 198 matching lines...) Expand 10 before | Expand all | Expand 10 after
1045 static cmpop_ty 1045 static cmpop_ty
1046 ast_for_comp_op(struct compiling *c, const node *n) 1046 ast_for_comp_op(struct compiling *c, const node *n)
1047 { 1047 {
1048 /* comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is' 1048 /* comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'
1049 |'is' 'not' 1049 |'is' 'not'
1050 */ 1050 */
1051 REQ(n, comp_op); 1051 REQ(n, comp_op);
1052 if (NCH(n) == 1) { 1052 if (NCH(n) == 1) {
1053 n = CHILD(n, 0); 1053 n = CHILD(n, 0);
1054 switch (TYPE(n)) { 1054 switch (TYPE(n)) {
1055 case LESS: 1055 case PYTOK_LESS:
1056 return Lt; 1056 return Lt;
1057 case GREATER: 1057 case PYTOK_GREATER:
1058 return Gt; 1058 return Gt;
1059 case EQEQUAL: /* == */ 1059 case PYTOK_EQEQUAL: /* == */
1060 return Eq; 1060 return Eq;
1061 case LESSEQUAL: 1061 case PYTOK_LESSEQUAL:
1062 return LtE; 1062 return LtE;
1063 case GREATEREQUAL: 1063 case PYTOK_GREATEREQUAL:
1064 return GtE; 1064 return GtE;
1065 case NOTEQUAL: 1065 case PYTOK_NOTEQUAL:
1066 return NotEq; 1066 return NotEq;
1067 case NAME: 1067 case PYTOK_NAME:
1068 if (strcmp(STR(n), "in") == 0) 1068 if (strcmp(STR(n), "in") == 0)
1069 return In; 1069 return In;
1070 if (strcmp(STR(n), "is") == 0) 1070 if (strcmp(STR(n), "is") == 0)
1071 return Is; 1071 return Is;
1072 default: 1072 default:
1073 PyErr_Format(PyExc_SystemError, "invalid comp_op: %s", 1073 PyErr_Format(PyExc_SystemError, "invalid comp_op: %s",
1074 STR(n)); 1074 STR(n));
1075 return (cmpop_ty)0; 1075 return (cmpop_ty)0;
1076 } 1076 }
1077 } 1077 }
1078 else if (NCH(n) == 2) { 1078 else if (NCH(n) == 2) {
1079 /* handle "not in" and "is not" */ 1079 /* handle "not in" and "is not" */
1080 switch (TYPE(CHILD(n, 0))) { 1080 switch (TYPE(CHILD(n, 0))) {
1081 case NAME: 1081 case PYTOK_NAME:
1082 if (strcmp(STR(CHILD(n, 1)), "in") == 0) 1082 if (strcmp(STR(CHILD(n, 1)), "in") == 0)
1083 return NotIn; 1083 return NotIn;
1084 if (strcmp(STR(CHILD(n, 0)), "is") == 0) 1084 if (strcmp(STR(CHILD(n, 0)), "is") == 0)
1085 return IsNot; 1085 return IsNot;
1086 default: 1086 default:
1087 PyErr_Format(PyExc_SystemError, "invalid comp_op: %s %s", 1087 PyErr_Format(PyExc_SystemError, "invalid comp_op: %s %s",
1088 STR(CHILD(n, 0)), STR(CHILD(n, 1))); 1088 STR(CHILD(n, 0)), STR(CHILD(n, 1)));
1089 return (cmpop_ty)0; 1089 return (cmpop_ty)0;
1090 } 1090 }
1091 } 1091 }
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
1132 arg_ty ret; 1132 arg_ty ret;
1133 1133
1134 assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef); 1134 assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef);
1135 ch = CHILD(n, 0); 1135 ch = CHILD(n, 0);
1136 name = NEW_IDENTIFIER(ch); 1136 name = NEW_IDENTIFIER(ch);
1137 if (!name) 1137 if (!name)
1138 return NULL; 1138 return NULL;
1139 if (forbidden_name(c, name, ch, 0)) 1139 if (forbidden_name(c, name, ch, 0))
1140 return NULL; 1140 return NULL;
1141 1141
1142 if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) { 1142 if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == PYTOK_COLON) {
1143 annotation = ast_for_expr(c, CHILD(n, 2)); 1143 annotation = ast_for_expr(c, CHILD(n, 2));
1144 if (!annotation) 1144 if (!annotation)
1145 return NULL; 1145 return NULL;
1146 } 1146 }
1147 1147
1148 ret = arg(name, annotation, c->c_arena); 1148 ret = arg(name, annotation, c->c_arena);
1149 if (!ret) 1149 if (!ret)
1150 return NULL; 1150 return NULL;
1151 ret->lineno = LINENO(n); 1151 ret->lineno = LINENO(n);
1152 ret->col_offset = n->n_col_offset; 1152 ret->col_offset = n->n_col_offset;
(...skipping 20 matching lines...) Expand all
1173 if (kwonlyargs == NULL) { 1173 if (kwonlyargs == NULL) {
1174 ast_error(c, CHILD(n, start), "named arguments must follow bare *"); 1174 ast_error(c, CHILD(n, start), "named arguments must follow bare *");
1175 return -1; 1175 return -1;
1176 } 1176 }
1177 assert(kwdefaults != NULL); 1177 assert(kwdefaults != NULL);
1178 while (i < NCH(n)) { 1178 while (i < NCH(n)) {
1179 ch = CHILD(n, i); 1179 ch = CHILD(n, i);
1180 switch (TYPE(ch)) { 1180 switch (TYPE(ch)) {
1181 case vfpdef: 1181 case vfpdef:
1182 case tfpdef: 1182 case tfpdef:
1183 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { 1183 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == PYTOK_EQUAL) {
1184 expression = ast_for_expr(c, CHILD(n, i + 2)); 1184 expression = ast_for_expr(c, CHILD(n, i + 2));
1185 if (!expression) 1185 if (!expression)
1186 goto error; 1186 goto error;
1187 asdl_seq_SET(kwdefaults, j, expression); 1187 asdl_seq_SET(kwdefaults, j, expression);
1188 i += 2; /* '=' and test */ 1188 i += 2; /* '=' and test */
1189 } 1189 }
1190 else { /* setting NULL if no default value exists */ 1190 else { /* setting NULL if no default value exists */
1191 asdl_seq_SET(kwdefaults, j, NULL); 1191 asdl_seq_SET(kwdefaults, j, NULL);
1192 } 1192 }
1193 if (NCH(ch) == 3) { 1193 if (NCH(ch) == 3) {
1194 /* ch is NAME ':' test */ 1194 /* ch is PYTOK_NAME ':' test */
1195 annotation = ast_for_expr(c, CHILD(ch, 2)); 1195 annotation = ast_for_expr(c, CHILD(ch, 2));
1196 if (!annotation) 1196 if (!annotation)
1197 goto error; 1197 goto error;
1198 } 1198 }
1199 else { 1199 else {
1200 annotation = NULL; 1200 annotation = NULL;
1201 } 1201 }
1202 ch = CHILD(ch, 0); 1202 ch = CHILD(ch, 0);
1203 argname = NEW_IDENTIFIER(ch); 1203 argname = NEW_IDENTIFIER(ch);
1204 if (!argname) 1204 if (!argname)
1205 goto error; 1205 goto error;
1206 if (forbidden_name(c, argname, ch, 0)) 1206 if (forbidden_name(c, argname, ch, 0))
1207 goto error; 1207 goto error;
1208 arg = arg(argname, annotation, c->c_arena); 1208 arg = arg(argname, annotation, c->c_arena);
1209 if (!arg) 1209 if (!arg)
1210 goto error; 1210 goto error;
1211 arg->lineno = LINENO(ch); 1211 arg->lineno = LINENO(ch);
1212 arg->col_offset = ch->n_col_offset; 1212 arg->col_offset = ch->n_col_offset;
1213 asdl_seq_SET(kwonlyargs, j++, arg); 1213 asdl_seq_SET(kwonlyargs, j++, arg);
1214 i += 2; /* the name and the comma */ 1214 i += 2; /* the name and the comma */
1215 break; 1215 break;
1216 case DOUBLESTAR: 1216 case PYTOK_DOUBLESTAR:
1217 return i; 1217 return i;
1218 default: 1218 default:
1219 ast_error(c, ch, "unexpected node"); 1219 ast_error(c, ch, "unexpected node");
1220 goto error; 1220 goto error;
1221 } 1221 }
1222 } 1222 }
1223 return i; 1223 return i;
1224 error: 1224 error:
1225 return -1; 1225 return -1;
1226 } 1226 }
1227 1227
1228 /* Create AST for argument list. */ 1228 /* Create AST for argument list. */
1229 1229
1230 static arguments_ty 1230 static arguments_ty
1231 ast_for_arguments(struct compiling *c, const node *n) 1231 ast_for_arguments(struct compiling *c, const node *n)
1232 { 1232 {
1233 /* This function handles both typedargslist (function definition) 1233 /* This function handles both typedargslist (function definition)
1234 and varargslist (lambda definition). 1234 and varargslist (lambda definition).
1235 1235
1236 parameters: '(' [typedargslist] ')' 1236 parameters: '(' [typedargslist] ')'
1237 typedargslist: ((tfpdef ['=' test] ',')* 1237 typedargslist: ((tfpdef ['=' test] ',')*
1238 ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] 1238 ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef]
1239 | '**' tfpdef) 1239 | '**' tfpdef)
1240 | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) 1240 | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
1241 tfpdef: NAME [':' test] 1241 tfpdef: PYTOK_NAME [':' test]
1242 varargslist: ((vfpdef ['=' test] ',')* 1242 varargslist: ((vfpdef ['=' test] ',')*
1243 ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] 1243 ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef]
1244 | '**' vfpdef) 1244 | '**' vfpdef)
1245 | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) 1245 | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
1246 vfpdef: NAME 1246 vfpdef: PYTOK_NAME
1247 */ 1247 */
1248 int i, j, k, nposargs = 0, nkwonlyargs = 0; 1248 int i, j, k, nposargs = 0, nkwonlyargs = 0;
1249 int nposdefaults = 0, found_default = 0; 1249 int nposdefaults = 0, found_default = 0;
1250 asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults; 1250 asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults;
1251 arg_ty vararg = NULL, kwarg = NULL; 1251 arg_ty vararg = NULL, kwarg = NULL;
1252 arg_ty arg; 1252 arg_ty arg;
1253 node *ch; 1253 node *ch;
1254 1254
1255 if (TYPE(n) == parameters) { 1255 if (TYPE(n) == parameters) {
1256 if (NCH(n) == 2) /* () as argument list */ 1256 if (NCH(n) == 2) /* () as argument list */
1257 return arguments(NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena); 1257 return arguments(NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena);
1258 n = CHILD(n, 1); 1258 n = CHILD(n, 1);
1259 } 1259 }
1260 assert(TYPE(n) == typedargslist || TYPE(n) == varargslist); 1260 assert(TYPE(n) == typedargslist || TYPE(n) == varargslist);
1261 1261
1262 /* First count the number of positional args & defaults. The 1262 /* First count the number of positional args & defaults. The
1263 variable i is the loop index for this for loop and the next. 1263 variable i is the loop index for this for loop and the next.
1264 The next loop picks up where the first leaves off. 1264 The next loop picks up where the first leaves off.
1265 */ 1265 */
1266 for (i = 0; i < NCH(n); i++) { 1266 for (i = 0; i < NCH(n); i++) {
1267 ch = CHILD(n, i); 1267 ch = CHILD(n, i);
1268 if (TYPE(ch) == STAR) { 1268 if (TYPE(ch) == PYTOK_STAR) {
1269 /* skip star */ 1269 /* skip star */
1270 i++; 1270 i++;
1271 if (i < NCH(n) && /* skip argument following star */ 1271 if (i < NCH(n) && /* skip argument following star */
1272 (TYPE(CHILD(n, i)) == tfpdef || 1272 (TYPE(CHILD(n, i)) == tfpdef ||
1273 TYPE(CHILD(n, i)) == vfpdef)) { 1273 TYPE(CHILD(n, i)) == vfpdef)) {
1274 i++; 1274 i++;
1275 } 1275 }
1276 break; 1276 break;
1277 } 1277 }
1278 if (TYPE(ch) == DOUBLESTAR) break; 1278 if (TYPE(ch) == PYTOK_DOUBLESTAR) break;
1279 if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++; 1279 if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++;
1280 if (TYPE(ch) == EQUAL) nposdefaults++; 1280 if (TYPE(ch) == PYTOK_EQUAL) nposdefaults++;
1281 } 1281 }
1282 /* count the number of keyword only args & 1282 /* count the number of keyword only args &
1283 defaults for keyword only args */ 1283 defaults for keyword only args */
1284 for ( ; i < NCH(n); ++i) { 1284 for ( ; i < NCH(n); ++i) {
1285 ch = CHILD(n, i); 1285 ch = CHILD(n, i);
1286 if (TYPE(ch) == DOUBLESTAR) break; 1286 if (TYPE(ch) == PYTOK_DOUBLESTAR) break;
1287 if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++; 1287 if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++;
1288 } 1288 }
1289 posargs = (nposargs ? _Py_asdl_seq_new(nposargs, c->c_arena) : NULL); 1289 posargs = (nposargs ? _Py_asdl_seq_new(nposargs, c->c_arena) : NULL);
1290 if (!posargs && nposargs) 1290 if (!posargs && nposargs)
1291 return NULL; 1291 return NULL;
1292 kwonlyargs = (nkwonlyargs ? 1292 kwonlyargs = (nkwonlyargs ?
1293 _Py_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL); 1293 _Py_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
1294 if (!kwonlyargs && nkwonlyargs) 1294 if (!kwonlyargs && nkwonlyargs)
1295 return NULL; 1295 return NULL;
1296 posdefaults = (nposdefaults ? 1296 posdefaults = (nposdefaults ?
1297 _Py_asdl_seq_new(nposdefaults, c->c_arena) : NULL); 1297 _Py_asdl_seq_new(nposdefaults, c->c_arena) : NULL);
1298 if (!posdefaults && nposdefaults) 1298 if (!posdefaults && nposdefaults)
1299 return NULL; 1299 return NULL;
1300 /* The length of kwonlyargs and kwdefaults are same 1300 /* The length of kwonlyargs and kwdefaults are same
1301 since we set NULL as default for keyword only argument w/o default 1301 since we set NULL as default for keyword only argument w/o default
1302 - we have sequence data structure, but no dictionary */ 1302 - we have sequence data structure, but no dictionary */
1303 kwdefaults = (nkwonlyargs ? 1303 kwdefaults = (nkwonlyargs ?
1304 _Py_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL); 1304 _Py_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
1305 if (!kwdefaults && nkwonlyargs) 1305 if (!kwdefaults && nkwonlyargs)
1306 return NULL; 1306 return NULL;
1307 1307
1308 if (nposargs + nkwonlyargs > 255) { 1308 if (nposargs + nkwonlyargs > 255) {
1309 ast_error(c, n, "more than 255 arguments"); 1309 ast_error(c, n, "more than 255 arguments");
1310 return NULL; 1310 return NULL;
1311 } 1311 }
1312 1312
1313 /* tfpdef: NAME [':' test] 1313 /* tfpdef: PYTOK_NAME [':' test]
1314 vfpdef: NAME 1314 vfpdef: PYTOK_NAME
1315 */ 1315 */
1316 i = 0; 1316 i = 0;
1317 j = 0; /* index for defaults */ 1317 j = 0; /* index for defaults */
1318 k = 0; /* index for args */ 1318 k = 0; /* index for args */
1319 while (i < NCH(n)) { 1319 while (i < NCH(n)) {
1320 ch = CHILD(n, i); 1320 ch = CHILD(n, i);
1321 switch (TYPE(ch)) { 1321 switch (TYPE(ch)) {
1322 case tfpdef: 1322 case tfpdef:
1323 case vfpdef: 1323 case vfpdef:
1324 /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is 1324 /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
1325 anything other than EQUAL or a comma? */ 1325 anything other than PYTOK_EQUAL or a comma? */
1326 /* XXX Should NCH(n) check be made a separate check? */ 1326 /* XXX Should NCH(n) check be made a separate check? */
1327 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { 1327 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == PYTOK_EQUAL) {
1328 expr_ty expression = ast_for_expr(c, CHILD(n, i + 2)); 1328 expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));
1329 if (!expression) 1329 if (!expression)
1330 return NULL; 1330 return NULL;
1331 assert(posdefaults != NULL); 1331 assert(posdefaults != NULL);
1332 asdl_seq_SET(posdefaults, j++, expression); 1332 asdl_seq_SET(posdefaults, j++, expression);
1333 i += 2; 1333 i += 2;
1334 found_default = 1; 1334 found_default = 1;
1335 } 1335 }
1336 else if (found_default) { 1336 else if (found_default) {
1337 ast_error(c, n, 1337 ast_error(c, n,
1338 "non-default argument follows default argument"); 1338 "non-default argument follows default argument");
1339 return NULL; 1339 return NULL;
1340 } 1340 }
1341 arg = ast_for_arg(c, ch); 1341 arg = ast_for_arg(c, ch);
1342 if (!arg) 1342 if (!arg)
1343 return NULL; 1343 return NULL;
1344 asdl_seq_SET(posargs, k++, arg); 1344 asdl_seq_SET(posargs, k++, arg);
1345 i += 2; /* the name and the comma */ 1345 i += 2; /* the name and the comma */
1346 break; 1346 break;
1347 case STAR: 1347 case PYTOK_STAR:
1348 if (i+1 >= NCH(n)) { 1348 if (i+1 >= NCH(n)) {
1349 ast_error(c, CHILD(n, i), 1349 ast_error(c, CHILD(n, i),
1350 "named arguments must follow bare *"); 1350 "named arguments must follow bare *");
1351 return NULL; 1351 return NULL;
1352 } 1352 }
1353 ch = CHILD(n, i+1); /* tfpdef or COMMA */ 1353 ch = CHILD(n, i+1); /* tfpdef or PYTOK_COMMA */
1354 if (TYPE(ch) == COMMA) { 1354 if (TYPE(ch) == PYTOK_COMMA) {
1355 int res = 0; 1355 int res = 0;
1356 i += 2; /* now follows keyword only arguments */ 1356 i += 2; /* now follows keyword only arguments */
1357 res = handle_keywordonly_args(c, n, i, 1357 res = handle_keywordonly_args(c, n, i,
1358 kwonlyargs, kwdefaults); 1358 kwonlyargs, kwdefaults);
1359 if (res == -1) return NULL; 1359 if (res == -1) return NULL;
1360 i = res; /* res has new position to process */ 1360 i = res; /* res has new position to process */
1361 } 1361 }
1362 else { 1362 else {
1363 vararg = ast_for_arg(c, ch); 1363 vararg = ast_for_arg(c, ch);
1364 if (!vararg) 1364 if (!vararg)
1365 return NULL; 1365 return NULL;
1366 1366
1367 i += 3; 1367 i += 3;
1368 if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef 1368 if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef
1369 || TYPE(CHILD(n, i)) == vfpdef)) { 1369 || TYPE(CHILD(n, i)) == vfpdef)) {
1370 int res = 0; 1370 int res = 0;
1371 res = handle_keywordonly_args(c, n, i, 1371 res = handle_keywordonly_args(c, n, i,
1372 kwonlyargs, kwdefaults); 1372 kwonlyargs, kwdefaults);
1373 if (res == -1) return NULL; 1373 if (res == -1) return NULL;
1374 i = res; /* res has new position to process */ 1374 i = res; /* res has new position to process */
1375 } 1375 }
1376 } 1376 }
1377 break; 1377 break;
1378 case DOUBLESTAR: 1378 case PYTOK_DOUBLESTAR:
1379 ch = CHILD(n, i+1); /* tfpdef */ 1379 ch = CHILD(n, i+1); /* tfpdef */
1380 assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef); 1380 assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);
1381 kwarg = ast_for_arg(c, ch); 1381 kwarg = ast_for_arg(c, ch);
1382 if (!kwarg) 1382 if (!kwarg)
1383 return NULL; 1383 return NULL;
1384 i += 3; 1384 i += 3;
1385 break; 1385 break;
1386 default: 1386 default:
1387 PyErr_Format(PyExc_SystemError, 1387 PyErr_Format(PyExc_SystemError,
1388 "unexpected node in varargslist: %d @ %d", 1388 "unexpected node in varargslist: %d @ %d",
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after
1421 if (!e) 1421 if (!e)
1422 return NULL; 1422 return NULL;
1423 } 1423 }
1424 1424
1425 return e; 1425 return e;
1426 } 1426 }
1427 1427
1428 static expr_ty 1428 static expr_ty
1429 ast_for_decorator(struct compiling *c, const node *n) 1429 ast_for_decorator(struct compiling *c, const node *n)
1430 { 1430 {
1431 /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */ 1431 /* decorator: '@' dotted_name [ '(' [arglist] ')' ] PYTOK_NEWLINE */
1432 expr_ty d = NULL; 1432 expr_ty d = NULL;
1433 expr_ty name_expr; 1433 expr_ty name_expr;
1434 1434
1435 REQ(n, decorator); 1435 REQ(n, decorator);
1436 REQ(CHILD(n, 0), AT); 1436 REQ(CHILD(n, 0), PYTOK_AT);
1437 REQ(RCHILD(n, -1), NEWLINE); 1437 REQ(RCHILD(n, -1), PYTOK_NEWLINE);
1438 1438
1439 name_expr = ast_for_dotted_name(c, CHILD(n, 1)); 1439 name_expr = ast_for_dotted_name(c, CHILD(n, 1));
1440 if (!name_expr) 1440 if (!name_expr)
1441 return NULL; 1441 return NULL;
1442 1442
1443 if (NCH(n) == 3) { /* No arguments */ 1443 if (NCH(n) == 3) { /* No arguments */
1444 d = name_expr; 1444 d = name_expr;
1445 name_expr = NULL; 1445 name_expr = NULL;
1446 } 1446 }
1447 else if (NCH(n) == 5) { /* Call with no arguments */ 1447 else if (NCH(n) == 5) { /* Call with no arguments */
(...skipping 30 matching lines...) Expand all
1478 if (!d) 1478 if (!d)
1479 return NULL; 1479 return NULL;
1480 asdl_seq_SET(decorator_seq, i, d); 1480 asdl_seq_SET(decorator_seq, i, d);
1481 } 1481 }
1482 return decorator_seq; 1482 return decorator_seq;
1483 } 1483 }
1484 1484
1485 static stmt_ty 1485 static stmt_ty
1486 ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) 1486 ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
1487 { 1487 {
1488 /* funcdef: 'def' NAME parameters ['->' test] ':' suite */ 1488 /* funcdef: 'def' PYTOK_NAME parameters ['->' test] ':' suite */
1489 identifier name; 1489 identifier name;
1490 arguments_ty args; 1490 arguments_ty args;
1491 asdl_seq *body; 1491 asdl_seq *body;
1492 expr_ty returns = NULL; 1492 expr_ty returns = NULL;
1493 int name_i = 1; 1493 int name_i = 1;
1494 1494
1495 REQ(n, funcdef); 1495 REQ(n, funcdef);
1496 1496
1497 name = NEW_IDENTIFIER(CHILD(n, name_i)); 1497 name = NEW_IDENTIFIER(CHILD(n, name_i));
1498 if (!name) 1498 if (!name)
1499 return NULL; 1499 return NULL;
1500 if (forbidden_name(c, name, CHILD(n, name_i), 0)) 1500 if (forbidden_name(c, name, CHILD(n, name_i), 0))
1501 return NULL; 1501 return NULL;
1502 args = ast_for_arguments(c, CHILD(n, name_i + 1)); 1502 args = ast_for_arguments(c, CHILD(n, name_i + 1));
1503 if (!args) 1503 if (!args)
1504 return NULL; 1504 return NULL;
1505 if (TYPE(CHILD(n, name_i+2)) == RARROW) { 1505 if (TYPE(CHILD(n, name_i+2)) == PYTOK_RARROW) {
1506 returns = ast_for_expr(c, CHILD(n, name_i + 3)); 1506 returns = ast_for_expr(c, CHILD(n, name_i + 3));
1507 if (!returns) 1507 if (!returns)
1508 return NULL; 1508 return NULL;
1509 name_i += 2; 1509 name_i += 2;
1510 } 1510 }
1511 body = ast_for_suite(c, CHILD(n, name_i + 3)); 1511 body = ast_for_suite(c, CHILD(n, name_i + 3));
1512 if (!body) 1512 if (!body)
1513 return NULL; 1513 return NULL;
1514 1514
1515 return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n), 1515 return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n),
(...skipping 245 matching lines...) Expand 10 before | Expand all | Expand 10 after
1761 return NULL; 1761 return NULL;
1762 } 1762 }
1763 1763
1764 static expr_ty 1764 static expr_ty
1765 ast_for_dictcomp(struct compiling *c, const node *n) 1765 ast_for_dictcomp(struct compiling *c, const node *n)
1766 { 1766 {
1767 expr_ty key, value; 1767 expr_ty key, value;
1768 asdl_seq *comps; 1768 asdl_seq *comps;
1769 1769
1770 assert(NCH(n) > 3); 1770 assert(NCH(n) > 3);
1771 REQ(CHILD(n, 1), COLON); 1771 REQ(CHILD(n, 1), PYTOK_COLON);
1772 1772
1773 key = ast_for_expr(c, CHILD(n, 0)); 1773 key = ast_for_expr(c, CHILD(n, 0));
1774 if (!key) 1774 if (!key)
1775 return NULL; 1775 return NULL;
1776 value = ast_for_expr(c, CHILD(n, 2)); 1776 value = ast_for_expr(c, CHILD(n, 2));
1777 if (!value) 1777 if (!value)
1778 return NULL; 1778 return NULL;
1779 1779
1780 comps = ast_for_comprehension(c, CHILD(n, 3)); 1780 comps = ast_for_comprehension(c, CHILD(n, 3));
1781 if (!comps) 1781 if (!comps)
(...skipping 21 matching lines...) Expand all
1803 { 1803 {
1804 assert(TYPE(n) == (dictorsetmaker)); 1804 assert(TYPE(n) == (dictorsetmaker));
1805 return ast_for_itercomp(c, n, COMP_SETCOMP); 1805 return ast_for_itercomp(c, n, COMP_SETCOMP);
1806 } 1806 }
1807 1807
1808 1808
1809 static expr_ty 1809 static expr_ty
1810 ast_for_atom(struct compiling *c, const node *n) 1810 ast_for_atom(struct compiling *c, const node *n)
1811 { 1811 {
1812 /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' 1812 /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
1813 | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+ 1813 | '{' [dictmaker|testlist_comp] '}' | PYTOK_NAME | PYTOK_NUMBER | PYTOK_S TRING+
1814 | '...' | 'None' | 'True' | 'False' 1814 | '...' | 'None' | 'True' | 'False'
1815 */ 1815 */
1816 node *ch = CHILD(n, 0); 1816 node *ch = CHILD(n, 0);
1817 int bytesmode = 0; 1817 int bytesmode = 0;
1818 1818
1819 switch (TYPE(ch)) { 1819 switch (TYPE(ch)) {
1820 case NAME: { 1820 case PYTOK_NAME: {
1821 PyObject *name; 1821 PyObject *name;
1822 const char *s = STR(ch); 1822 const char *s = STR(ch);
1823 size_t len = strlen(s); 1823 size_t len = strlen(s);
1824 if (len >= 4 && len <= 5) { 1824 if (len >= 4 && len <= 5) {
1825 if (!strcmp(s, "None")) 1825 if (!strcmp(s, "None"))
1826 return NameConstant(Py_None, LINENO(n), n->n_col_offset, c->c_ar ena); 1826 return NameConstant(Py_None, LINENO(n), n->n_col_offset, c->c_ar ena);
1827 if (!strcmp(s, "True")) 1827 if (!strcmp(s, "True"))
1828 return NameConstant(Py_True, LINENO(n), n->n_col_offset, c->c_ar ena); 1828 return NameConstant(Py_True, LINENO(n), n->n_col_offset, c->c_ar ena);
1829 if (!strcmp(s, "False")) 1829 if (!strcmp(s, "False"))
1830 return NameConstant(Py_False, LINENO(n), n->n_col_offset, c->c_a rena); 1830 return NameConstant(Py_False, LINENO(n), n->n_col_offset, c->c_a rena);
1831 } 1831 }
1832 name = new_identifier(s, c); 1832 name = new_identifier(s, c);
1833 if (!name) 1833 if (!name)
1834 return NULL; 1834 return NULL;
1835 /* All names start in Load context, but may later be changed. */ 1835 /* All names start in Load context, but may later be changed. */
1836 return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); 1836 return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
1837 } 1837 }
1838 case STRING: { 1838 case PYTOK_STRING: {
1839 PyObject *str = parsestrplus(c, n, &bytesmode); 1839 PyObject *str = parsestrplus(c, n, &bytesmode);
1840 if (!str) { 1840 if (!str) {
1841 const char *errtype = NULL; 1841 const char *errtype = NULL;
1842 if (PyErr_ExceptionMatches(PyExc_UnicodeError)) 1842 if (PyErr_ExceptionMatches(PyExc_UnicodeError))
1843 errtype = "unicode error"; 1843 errtype = "unicode error";
1844 else if (PyErr_ExceptionMatches(PyExc_ValueError)) 1844 else if (PyErr_ExceptionMatches(PyExc_ValueError))
1845 errtype = "value error"; 1845 errtype = "value error";
1846 if (errtype) { 1846 if (errtype) {
1847 char buf[128]; 1847 char buf[128];
1848 PyObject *type, *value, *tback, *errstr; 1848 PyObject *type, *value, *tback, *errstr;
(...skipping 15 matching lines...) Expand all
1864 } 1864 }
1865 if (PyArena_AddPyObject(c->c_arena, str) < 0) { 1865 if (PyArena_AddPyObject(c->c_arena, str) < 0) {
1866 Py_DECREF(str); 1866 Py_DECREF(str);
1867 return NULL; 1867 return NULL;
1868 } 1868 }
1869 if (bytesmode) 1869 if (bytesmode)
1870 return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena); 1870 return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena);
1871 else 1871 else
1872 return Str(str, LINENO(n), n->n_col_offset, c->c_arena); 1872 return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
1873 } 1873 }
1874 case NUMBER: { 1874 case PYTOK_NUMBER: {
1875 PyObject *pynum = parsenumber(c, STR(ch)); 1875 PyObject *pynum = parsenumber(c, STR(ch));
1876 if (!pynum) 1876 if (!pynum)
1877 return NULL; 1877 return NULL;
1878 1878
1879 if (PyArena_AddPyObject(c->c_arena, pynum) < 0) { 1879 if (PyArena_AddPyObject(c->c_arena, pynum) < 0) {
1880 Py_DECREF(pynum); 1880 Py_DECREF(pynum);
1881 return NULL; 1881 return NULL;
1882 } 1882 }
1883 return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); 1883 return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
1884 } 1884 }
1885 case ELLIPSIS: /* Ellipsis */ 1885 case PYTOK_ELLIPSIS: /* Ellipsis */
1886 return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena); 1886 return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
1887 case LPAR: /* some parenthesized expressions */ 1887 case PYTOK_LPAR: /* some parenthesized expressions */
1888 ch = CHILD(n, 1); 1888 ch = CHILD(n, 1);
1889 1889
1890 if (TYPE(ch) == RPAR) 1890 if (TYPE(ch) == PYTOK_RPAR)
1891 return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); 1891 return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
1892 1892
1893 if (TYPE(ch) == yield_expr) 1893 if (TYPE(ch) == yield_expr)
1894 return ast_for_expr(c, ch); 1894 return ast_for_expr(c, ch);
1895 1895
1896 /* testlist_comp: test ( comp_for | (',' test)* [','] ) */ 1896 /* testlist_comp: test ( comp_for | (',' test)* [','] ) */
1897 if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for)) 1897 if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for))
1898 return ast_for_genexp(c, ch); 1898 return ast_for_genexp(c, ch);
1899 1899
1900 return ast_for_testlist(c, ch); 1900 return ast_for_testlist(c, ch);
1901 case LSQB: /* list (or list comprehension) */ 1901 case PYTOK_LSQB: /* list (or list comprehension) */
1902 ch = CHILD(n, 1); 1902 ch = CHILD(n, 1);
1903 1903
1904 if (TYPE(ch) == RSQB) 1904 if (TYPE(ch) == PYTOK_RSQB)
1905 return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); 1905 return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
1906 1906
1907 REQ(ch, testlist_comp); 1907 REQ(ch, testlist_comp);
1908 if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { 1908 if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == PYTOK_COMMA) {
1909 asdl_seq *elts = seq_for_testlist(c, ch); 1909 asdl_seq *elts = seq_for_testlist(c, ch);
1910 if (!elts) 1910 if (!elts)
1911 return NULL; 1911 return NULL;
1912 1912
1913 return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena); 1913 return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
1914 } 1914 }
1915 else 1915 else
1916 return ast_for_listcomp(c, ch); 1916 return ast_for_listcomp(c, ch);
1917 case LBRACE: { 1917 case PYTOK_LBRACE: {
1918 /* dictorsetmaker: test ':' test (',' test ':' test)* [','] | 1918 /* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
1919 * test (gen_for | (',' test)* [',']) */ 1919 * test (gen_for | (',' test)* [',']) */
1920 int i, size; 1920 int i, size;
1921 asdl_seq *keys, *values; 1921 asdl_seq *keys, *values;
1922 1922
1923 ch = CHILD(n, 1); 1923 ch = CHILD(n, 1);
1924 if (TYPE(ch) == RBRACE) { 1924 if (TYPE(ch) == PYTOK_RBRACE) {
1925 /* it's an empty dict */ 1925 /* it's an empty dict */
1926 return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); 1926 return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
1927 } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { 1927 } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == PYTOK_COMMA) {
1928 /* it's a simple set */ 1928 /* it's a simple set */
1929 asdl_seq *elts; 1929 asdl_seq *elts;
1930 size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */ 1930 size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
1931 elts = _Py_asdl_seq_new(size, c->c_arena); 1931 elts = _Py_asdl_seq_new(size, c->c_arena);
1932 if (!elts) 1932 if (!elts)
1933 return NULL; 1933 return NULL;
1934 for (i = 0; i < NCH(ch); i += 2) { 1934 for (i = 0; i < NCH(ch); i += 2) {
1935 expr_ty expression; 1935 expr_ty expression;
1936 expression = ast_for_expr(c, CHILD(ch, i)); 1936 expression = ast_for_expr(c, CHILD(ch, i));
1937 if (!expression) 1937 if (!expression)
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after
2002 return Index(step, c->c_arena); 2002 return Index(step, c->c_arena);
2003 } 2003 }
2004 2004
2005 if (TYPE(ch) == test) { 2005 if (TYPE(ch) == test) {
2006 lower = ast_for_expr(c, ch); 2006 lower = ast_for_expr(c, ch);
2007 if (!lower) 2007 if (!lower)
2008 return NULL; 2008 return NULL;
2009 } 2009 }
2010 2010
2011 /* If there's an upper bound it's in the second or third position. */ 2011 /* If there's an upper bound it's in the second or third position. */
2012 if (TYPE(ch) == COLON) { 2012 if (TYPE(ch) == PYTOK_COLON) {
2013 if (NCH(n) > 1) { 2013 if (NCH(n) > 1) {
2014 node *n2 = CHILD(n, 1); 2014 node *n2 = CHILD(n, 1);
2015 2015
2016 if (TYPE(n2) == test) { 2016 if (TYPE(n2) == test) {
2017 upper = ast_for_expr(c, n2); 2017 upper = ast_for_expr(c, n2);
2018 if (!upper) 2018 if (!upper)
2019 return NULL; 2019 return NULL;
2020 } 2020 }
2021 } 2021 }
2022 } else if (NCH(n) > 2) { 2022 } else if (NCH(n) > 2) {
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
2092 if (!tmp_result) 2092 if (!tmp_result)
2093 return NULL; 2093 return NULL;
2094 result = tmp_result; 2094 result = tmp_result;
2095 } 2095 }
2096 return result; 2096 return result;
2097 } 2097 }
2098 2098
2099 static expr_ty 2099 static expr_ty
2100 ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) 2100 ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
2101 { 2101 {
2102 /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 2102 /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' PYTOK_NAME
2103 subscriptlist: subscript (',' subscript)* [','] 2103 subscriptlist: subscript (',' subscript)* [',']
2104 subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] 2104 subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
2105 */ 2105 */
2106 REQ(n, trailer); 2106 REQ(n, trailer);
2107 if (TYPE(CHILD(n, 0)) == LPAR) { 2107 if (TYPE(CHILD(n, 0)) == PYTOK_LPAR) {
2108 if (NCH(n) == 2) 2108 if (NCH(n) == 2)
2109 return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), 2109 return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n),
2110 n->n_col_offset, c->c_arena); 2110 n->n_col_offset, c->c_arena);
2111 else 2111 else
2112 return ast_for_call(c, CHILD(n, 1), left_expr); 2112 return ast_for_call(c, CHILD(n, 1), left_expr);
2113 } 2113 }
2114 else if (TYPE(CHILD(n, 0)) == DOT) { 2114 else if (TYPE(CHILD(n, 0)) == PYTOK_DOT) {
2115 PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1)); 2115 PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1));
2116 if (!attr_id) 2116 if (!attr_id)
2117 return NULL; 2117 return NULL;
2118 return Attribute(left_expr, attr_id, Load, 2118 return Attribute(left_expr, attr_id, Load,
2119 LINENO(n), n->n_col_offset, c->c_arena); 2119 LINENO(n), n->n_col_offset, c->c_arena);
2120 } 2120 }
2121 else { 2121 else {
2122 REQ(CHILD(n, 0), LSQB); 2122 REQ(CHILD(n, 0), PYTOK_LSQB);
2123 REQ(CHILD(n, 2), RSQB); 2123 REQ(CHILD(n, 2), PYTOK_RSQB);
2124 n = CHILD(n, 1); 2124 n = CHILD(n, 1);
2125 if (NCH(n) == 1) { 2125 if (NCH(n) == 1) {
2126 slice_ty slc = ast_for_slice(c, CHILD(n, 0)); 2126 slice_ty slc = ast_for_slice(c, CHILD(n, 0));
2127 if (!slc) 2127 if (!slc)
2128 return NULL; 2128 return NULL;
2129 return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, 2129 return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset,
2130 c->c_arena); 2130 c->c_arena);
2131 } 2131 }
2132 else { 2132 else {
2133 /* The grammar is ambiguous here. The ambiguity is resolved 2133 /* The grammar is ambiguous here. The ambiguity is resolved
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after
2175 static expr_ty 2175 static expr_ty
2176 ast_for_factor(struct compiling *c, const node *n) 2176 ast_for_factor(struct compiling *c, const node *n)
2177 { 2177 {
2178 expr_ty expression; 2178 expr_ty expression;
2179 2179
2180 expression = ast_for_expr(c, CHILD(n, 1)); 2180 expression = ast_for_expr(c, CHILD(n, 1));
2181 if (!expression) 2181 if (!expression)
2182 return NULL; 2182 return NULL;
2183 2183
2184 switch (TYPE(CHILD(n, 0))) { 2184 switch (TYPE(CHILD(n, 0))) {
2185 case PLUS: 2185 case PYTOK_PLUS:
2186 return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset, 2186 return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset,
2187 c->c_arena); 2187 c->c_arena);
2188 case MINUS: 2188 case PYTOK_MINUS:
2189 return UnaryOp(USub, expression, LINENO(n), n->n_col_offset, 2189 return UnaryOp(USub, expression, LINENO(n), n->n_col_offset,
2190 c->c_arena); 2190 c->c_arena);
2191 case TILDE: 2191 case PYTOK_TILDE:
2192 return UnaryOp(Invert, expression, LINENO(n), 2192 return UnaryOp(Invert, expression, LINENO(n),
2193 n->n_col_offset, c->c_arena); 2193 n->n_col_offset, c->c_arena);
2194 } 2194 }
2195 PyErr_Format(PyExc_SystemError, "unhandled factor: %d", 2195 PyErr_Format(PyExc_SystemError, "unhandled factor: %d",
2196 TYPE(CHILD(n, 0))); 2196 TYPE(CHILD(n, 0)));
2197 return NULL; 2197 return NULL;
2198 } 2198 }
2199 2199
2200 static expr_ty 2200 static expr_ty
2201 ast_for_power(struct compiling *c, const node *n) 2201 ast_for_power(struct compiling *c, const node *n)
(...skipping 317 matching lines...) Expand 10 before | Expand all | Expand 10 after
2519 } 2519 }
2520 e = ast_for_expr(c, CHILD(ch, 2)); 2520 e = ast_for_expr(c, CHILD(ch, 2));
2521 if (!e) 2521 if (!e)
2522 return NULL; 2522 return NULL;
2523 kw = keyword(key, e, c->c_arena); 2523 kw = keyword(key, e, c->c_arena);
2524 if (!kw) 2524 if (!kw)
2525 return NULL; 2525 return NULL;
2526 asdl_seq_SET(keywords, nkeywords++, kw); 2526 asdl_seq_SET(keywords, nkeywords++, kw);
2527 } 2527 }
2528 } 2528 }
2529 else if (TYPE(ch) == STAR) { 2529 else if (TYPE(ch) == PYTOK_STAR) {
2530 vararg = ast_for_expr(c, CHILD(n, i+1)); 2530 vararg = ast_for_expr(c, CHILD(n, i+1));
2531 if (!vararg) 2531 if (!vararg)
2532 return NULL; 2532 return NULL;
2533 i++; 2533 i++;
2534 } 2534 }
2535 else if (TYPE(ch) == DOUBLESTAR) { 2535 else if (TYPE(ch) == PYTOK_DOUBLESTAR) {
2536 kwarg = ast_for_expr(c, CHILD(n, i+1)); 2536 kwarg = ast_for_expr(c, CHILD(n, i+1));
2537 if (!kwarg) 2537 if (!kwarg)
2538 return NULL; 2538 return NULL;
2539 i++; 2539 i++;
2540 } 2540 }
2541 } 2541 }
2542 2542
2543 return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_off set, c->c_arena); 2543 return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_off set, c->c_arena);
2544 } 2544 }
2545 2545
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
2624 2624
2625 return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena); 2625 return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
2626 } 2626 }
2627 else { 2627 else {
2628 int i; 2628 int i;
2629 asdl_seq *targets; 2629 asdl_seq *targets;
2630 node *value; 2630 node *value;
2631 expr_ty expression; 2631 expr_ty expression;
2632 2632
2633 /* a normal assignment */ 2633 /* a normal assignment */
2634 REQ(CHILD(n, 1), EQUAL); 2634 REQ(CHILD(n, 1), PYTOK_EQUAL);
2635 targets = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena); 2635 targets = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena);
2636 if (!targets) 2636 if (!targets)
2637 return NULL; 2637 return NULL;
2638 for (i = 0; i < NCH(n) - 2; i += 2) { 2638 for (i = 0; i < NCH(n) - 2; i += 2) {
2639 expr_ty e; 2639 expr_ty e;
2640 node *ch = CHILD(n, i); 2640 node *ch = CHILD(n, i);
2641 if (TYPE(ch) == yield_expr) { 2641 if (TYPE(ch) == yield_expr) {
2642 ast_error(c, ch, "assignment to yield expression not possible"); 2642 ast_error(c, ch, "assignment to yield expression not possible");
2643 return NULL; 2643 return NULL;
2644 } 2644 }
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after
2760 } 2760 }
2761 2761
2762 PyErr_SetString(PyExc_SystemError, "unhandled flow statement"); 2762 PyErr_SetString(PyExc_SystemError, "unhandled flow statement");
2763 return NULL; 2763 return NULL;
2764 } 2764 }
2765 2765
2766 static alias_ty 2766 static alias_ty
2767 alias_for_import_name(struct compiling *c, const node *n, int store) 2767 alias_for_import_name(struct compiling *c, const node *n, int store)
2768 { 2768 {
2769 /* 2769 /*
2770 import_as_name: NAME ['as' NAME] 2770 import_as_name: PYTOK_NAME ['as' PYTOK_NAME]
2771 dotted_as_name: dotted_name ['as' NAME] 2771 dotted_as_name: dotted_name ['as' PYTOK_NAME]
2772 dotted_name: NAME ('.' NAME)* 2772 dotted_name: PYTOK_NAME ('.' PYTOK_NAME)*
2773 */ 2773 */
2774 identifier str, name; 2774 identifier str, name;
2775 2775
2776 loop: 2776 loop:
2777 switch (TYPE(n)) { 2777 switch (TYPE(n)) {
2778 case import_as_name: { 2778 case import_as_name: {
2779 node *name_node = CHILD(n, 0); 2779 node *name_node = CHILD(n, 0);
2780 str = NULL; 2780 str = NULL;
2781 name = NEW_IDENTIFIER(name_node); 2781 name = NEW_IDENTIFIER(name_node);
2782 if (!name) 2782 if (!name)
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after
2858 return NULL; 2858 return NULL;
2859 str = uni; 2859 str = uni;
2860 PyUnicode_InternInPlace(&str); 2860 PyUnicode_InternInPlace(&str);
2861 if (PyArena_AddPyObject(c->c_arena, str) < 0) { 2861 if (PyArena_AddPyObject(c->c_arena, str) < 0) {
2862 Py_DECREF(str); 2862 Py_DECREF(str);
2863 return NULL; 2863 return NULL;
2864 } 2864 }
2865 return alias(str, NULL, c->c_arena); 2865 return alias(str, NULL, c->c_arena);
2866 } 2866 }
2867 break; 2867 break;
2868 case STAR: 2868 case PYTOK_STAR:
2869 str = PyUnicode_InternFromString("*"); 2869 str = PyUnicode_InternFromString("*");
2870 if (PyArena_AddPyObject(c->c_arena, str) < 0) { 2870 if (PyArena_AddPyObject(c->c_arena, str) < 0) {
2871 Py_DECREF(str); 2871 Py_DECREF(str);
2872 return NULL; 2872 return NULL;
2873 } 2873 }
2874 return alias(str, NULL, c->c_arena); 2874 return alias(str, NULL, c->c_arena);
2875 default: 2875 default:
2876 PyErr_Format(PyExc_SystemError, 2876 PyErr_Format(PyExc_SystemError,
2877 "unexpected import name: %d", TYPE(n)); 2877 "unexpected import name: %d", TYPE(n));
2878 return NULL; 2878 return NULL;
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
2922 2922
2923 /* Count the number of dots (for relative imports) and check for the 2923 /* Count the number of dots (for relative imports) and check for the
2924 optional module name */ 2924 optional module name */
2925 for (idx = 1; idx < NCH(n); idx++) { 2925 for (idx = 1; idx < NCH(n); idx++) {
2926 if (TYPE(CHILD(n, idx)) == dotted_name) { 2926 if (TYPE(CHILD(n, idx)) == dotted_name) {
2927 mod = alias_for_import_name(c, CHILD(n, idx), 0); 2927 mod = alias_for_import_name(c, CHILD(n, idx), 0);
2928 if (!mod) 2928 if (!mod)
2929 return NULL; 2929 return NULL;
2930 idx++; 2930 idx++;
2931 break; 2931 break;
2932 } else if (TYPE(CHILD(n, idx)) == ELLIPSIS) { 2932 } else if (TYPE(CHILD(n, idx)) == PYTOK_ELLIPSIS) {
2933 /* three consecutive dots are tokenized as one ELLIPSIS */ 2933 /* three consecutive dots are tokenized as one PYTOK_ELLIPSIS */
2934 ndots += 3; 2934 ndots += 3;
2935 continue; 2935 continue;
2936 } else if (TYPE(CHILD(n, idx)) != DOT) { 2936 } else if (TYPE(CHILD(n, idx)) != PYTOK_DOT) {
2937 break; 2937 break;
2938 } 2938 }
2939 ndots++; 2939 ndots++;
2940 } 2940 }
2941 idx++; /* skip over the 'import' keyword */ 2941 idx++; /* skip over the 'import' keyword */
2942 switch (TYPE(CHILD(n, idx))) { 2942 switch (TYPE(CHILD(n, idx))) {
2943 case STAR: 2943 case PYTOK_STAR:
2944 /* from ... import * */ 2944 /* from ... import * */
2945 n = CHILD(n, idx); 2945 n = CHILD(n, idx);
2946 n_children = 1; 2946 n_children = 1;
2947 break; 2947 break;
2948 case LPAR: 2948 case PYTOK_LPAR:
2949 /* from ... import (x, y, z) */ 2949 /* from ... import (x, y, z) */
2950 n = CHILD(n, idx + 1); 2950 n = CHILD(n, idx + 1);
2951 n_children = NCH(n); 2951 n_children = NCH(n);
2952 break; 2952 break;
2953 case import_as_names: 2953 case import_as_names:
2954 /* from ... import x, y, z */ 2954 /* from ... import x, y, z */
2955 n = CHILD(n, idx); 2955 n = CHILD(n, idx);
2956 n_children = NCH(n); 2956 n_children = NCH(n);
2957 if (n_children % 2 == 0) { 2957 if (n_children % 2 == 0) {
2958 ast_error(c, n, "trailing comma not allowed without" 2958 ast_error(c, n, "trailing comma not allowed without"
2959 " surrounding parentheses"); 2959 " surrounding parentheses");
2960 return NULL; 2960 return NULL;
2961 } 2961 }
2962 break; 2962 break;
2963 default: 2963 default:
2964 ast_error(c, n, "Unexpected node-type in from-import"); 2964 ast_error(c, n, "Unexpected node-type in from-import");
2965 return NULL; 2965 return NULL;
2966 } 2966 }
2967 2967
2968 aliases = _Py_asdl_seq_new((n_children + 1) / 2, c->c_arena); 2968 aliases = _Py_asdl_seq_new((n_children + 1) / 2, c->c_arena);
2969 if (!aliases) 2969 if (!aliases)
2970 return NULL; 2970 return NULL;
2971 2971
2972 /* handle "from ... import *" special b/c there's no children */ 2972 /* handle "from ... import *" special b/c there's no children */
2973 if (TYPE(n) == STAR) { 2973 if (TYPE(n) == PYTOK_STAR) {
2974 alias_ty import_alias = alias_for_import_name(c, n, 1); 2974 alias_ty import_alias = alias_for_import_name(c, n, 1);
2975 if (!import_alias) 2975 if (!import_alias)
2976 return NULL; 2976 return NULL;
2977 asdl_seq_SET(aliases, 0, import_alias); 2977 asdl_seq_SET(aliases, 0, import_alias);
2978 } 2978 }
2979 else { 2979 else {
2980 for (i = 0; i < NCH(n); i += 2) { 2980 for (i = 0; i < NCH(n); i += 2) {
2981 alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1) ; 2981 alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1) ;
2982 if (!import_alias) 2982 if (!import_alias)
2983 return NULL; 2983 return NULL;
2984 asdl_seq_SET(aliases, i / 2, import_alias); 2984 asdl_seq_SET(aliases, i / 2, import_alias);
2985 } 2985 }
2986 } 2986 }
2987 if (mod != NULL) 2987 if (mod != NULL)
2988 modname = mod->name; 2988 modname = mod->name;
2989 return ImportFrom(modname, aliases, ndots, lineno, col_offset, 2989 return ImportFrom(modname, aliases, ndots, lineno, col_offset,
2990 c->c_arena); 2990 c->c_arena);
2991 } 2991 }
2992 PyErr_Format(PyExc_SystemError, 2992 PyErr_Format(PyExc_SystemError,
2993 "unknown import statement: starts with command '%s'", 2993 "unknown import statement: starts with command '%s'",
2994 STR(CHILD(n, 0))); 2994 STR(CHILD(n, 0)));
2995 return NULL; 2995 return NULL;
2996 } 2996 }
2997 2997
2998 static stmt_ty 2998 static stmt_ty
2999 ast_for_global_stmt(struct compiling *c, const node *n) 2999 ast_for_global_stmt(struct compiling *c, const node *n)
3000 { 3000 {
3001 /* global_stmt: 'global' NAME (',' NAME)* */ 3001 /* global_stmt: 'global' PYTOK_NAME (',' PYTOK_NAME)* */
3002 identifier name; 3002 identifier name;
3003 asdl_seq *s; 3003 asdl_seq *s;
3004 int i; 3004 int i;
3005 3005
3006 REQ(n, global_stmt); 3006 REQ(n, global_stmt);
3007 s = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena); 3007 s = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena);
3008 if (!s) 3008 if (!s)
3009 return NULL; 3009 return NULL;
3010 for (i = 1; i < NCH(n); i += 2) { 3010 for (i = 1; i < NCH(n); i += 2) {
3011 name = NEW_IDENTIFIER(CHILD(n, i)); 3011 name = NEW_IDENTIFIER(CHILD(n, i));
3012 if (!name) 3012 if (!name)
3013 return NULL; 3013 return NULL;
3014 asdl_seq_SET(s, i / 2, name); 3014 asdl_seq_SET(s, i / 2, name);
3015 } 3015 }
3016 return Global(s, LINENO(n), n->n_col_offset, c->c_arena); 3016 return Global(s, LINENO(n), n->n_col_offset, c->c_arena);
3017 } 3017 }
3018 3018
3019 static stmt_ty 3019 static stmt_ty
3020 ast_for_nonlocal_stmt(struct compiling *c, const node *n) 3020 ast_for_nonlocal_stmt(struct compiling *c, const node *n)
3021 { 3021 {
3022 /* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* */ 3022 /* nonlocal_stmt: 'nonlocal' PYTOK_NAME (',' PYTOK_NAME)* */
3023 identifier name; 3023 identifier name;
3024 asdl_seq *s; 3024 asdl_seq *s;
3025 int i; 3025 int i;
3026 3026
3027 REQ(n, nonlocal_stmt); 3027 REQ(n, nonlocal_stmt);
3028 s = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena); 3028 s = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena);
3029 if (!s) 3029 if (!s)
3030 return NULL; 3030 return NULL;
3031 for (i = 1; i < NCH(n); i += 2) { 3031 for (i = 1; i < NCH(n); i += 2) {
3032 name = NEW_IDENTIFIER(CHILD(n, i)); 3032 name = NEW_IDENTIFIER(CHILD(n, i));
(...skipping 29 matching lines...) Expand all
3062 } 3062 }
3063 PyErr_Format(PyExc_SystemError, 3063 PyErr_Format(PyExc_SystemError,
3064 "improper number of parts to 'assert' statement: %d", 3064 "improper number of parts to 'assert' statement: %d",
3065 NCH(n)); 3065 NCH(n));
3066 return NULL; 3066 return NULL;
3067 } 3067 }
3068 3068
3069 static asdl_seq * 3069 static asdl_seq *
3070 ast_for_suite(struct compiling *c, const node *n) 3070 ast_for_suite(struct compiling *c, const node *n)
3071 { 3071 {
3072 /* suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT */ 3072 /* suite: simple_stmt | PYTOK_NEWLINE PYTOK_INDENT stmt+ PYTOK_DEDENT */
3073 asdl_seq *seq; 3073 asdl_seq *seq;
3074 stmt_ty s; 3074 stmt_ty s;
3075 int i, total, num, end, pos = 0; 3075 int i, total, num, end, pos = 0;
3076 node *ch; 3076 node *ch;
3077 3077
3078 REQ(n, suite); 3078 REQ(n, suite);
3079 3079
3080 total = num_stmts(n); 3080 total = num_stmts(n);
3081 seq = _Py_asdl_seq_new(total, c->c_arena); 3081 seq = _Py_asdl_seq_new(total, c->c_arena);
3082 if (!seq) 3082 if (!seq)
3083 return NULL; 3083 return NULL;
3084 if (TYPE(CHILD(n, 0)) == simple_stmt) { 3084 if (TYPE(CHILD(n, 0)) == simple_stmt) {
3085 n = CHILD(n, 0); 3085 n = CHILD(n, 0);
3086 /* simple_stmt always ends with a NEWLINE, 3086 /* simple_stmt always ends with a PYTOK_NEWLINE,
3087 and may have a trailing SEMI 3087 and may have a trailing PYTOK_SEMI
3088 */ 3088 */
3089 end = NCH(n) - 1; 3089 end = NCH(n) - 1;
3090 if (TYPE(CHILD(n, end - 1)) == SEMI) 3090 if (TYPE(CHILD(n, end - 1)) == PYTOK_SEMI)
3091 end--; 3091 end--;
3092 /* loop by 2 to skip semi-colons */ 3092 /* loop by 2 to skip semi-colons */
3093 for (i = 0; i < end; i += 2) { 3093 for (i = 0; i < end; i += 2) {
3094 ch = CHILD(n, i); 3094 ch = CHILD(n, i);
3095 s = ast_for_stmt(c, ch); 3095 s = ast_for_stmt(c, ch);
3096 if (!s) 3096 if (!s)
3097 return NULL; 3097 return NULL;
3098 asdl_seq_SET(seq, pos++, s); 3098 asdl_seq_SET(seq, pos++, s);
3099 } 3099 }
3100 } 3100 }
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after
3180 c->c_arena); 3180 c->c_arena);
3181 } 3181 }
3182 else if (s[2] == 'i') { 3182 else if (s[2] == 'i') {
3183 int i, n_elif, has_else = 0; 3183 int i, n_elif, has_else = 0;
3184 expr_ty expression; 3184 expr_ty expression;
3185 asdl_seq *suite_seq; 3185 asdl_seq *suite_seq;
3186 asdl_seq *orelse = NULL; 3186 asdl_seq *orelse = NULL;
3187 n_elif = NCH(n) - 4; 3187 n_elif = NCH(n) - 4;
3188 /* must reference the child n_elif+1 since 'else' token is third, 3188 /* must reference the child n_elif+1 since 'else' token is third,
3189 not fourth, child from the end. */ 3189 not fourth, child from the end. */
3190 if (TYPE(CHILD(n, (n_elif + 1))) == NAME 3190 if (TYPE(CHILD(n, (n_elif + 1))) == PYTOK_NAME
3191 && STR(CHILD(n, (n_elif + 1)))[2] == 's') { 3191 && STR(CHILD(n, (n_elif + 1)))[2] == 's') {
3192 has_else = 1; 3192 has_else = 1;
3193 n_elif -= 3; 3193 n_elif -= 3;
3194 } 3194 }
3195 n_elif /= 4; 3195 n_elif /= 4;
3196 3196
3197 if (has_else) { 3197 if (has_else) {
3198 asdl_seq *suite_seq2; 3198 asdl_seq *suite_seq2;
3199 3199
3200 orelse = _Py_asdl_seq_new(1, c->c_arena); 3200 orelse = _Py_asdl_seq_new(1, c->c_arena);
(...skipping 191 matching lines...) Expand 10 before | Expand all | Expand 10 after
3392 const int nch = NCH(n); 3392 const int nch = NCH(n);
3393 int n_except = (nch - 3)/3; 3393 int n_except = (nch - 3)/3;
3394 asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL; 3394 asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL;
3395 3395
3396 REQ(n, try_stmt); 3396 REQ(n, try_stmt);
3397 3397
3398 body = ast_for_suite(c, CHILD(n, 2)); 3398 body = ast_for_suite(c, CHILD(n, 2));
3399 if (body == NULL) 3399 if (body == NULL)
3400 return NULL; 3400 return NULL;
3401 3401
3402 if (TYPE(CHILD(n, nch - 3)) == NAME) { 3402 if (TYPE(CHILD(n, nch - 3)) == PYTOK_NAME) {
3403 if (strcmp(STR(CHILD(n, nch - 3)), "finally") == 0) { 3403 if (strcmp(STR(CHILD(n, nch - 3)), "finally") == 0) {
3404 if (nch >= 9 && TYPE(CHILD(n, nch - 6)) == NAME) { 3404 if (nch >= 9 && TYPE(CHILD(n, nch - 6)) == PYTOK_NAME) {
3405 /* we can assume it's an "else", 3405 /* we can assume it's an "else",
3406 because nch >= 9 for try-else-finally and 3406 because nch >= 9 for try-else-finally and
3407 it would otherwise have a type of except_clause */ 3407 it would otherwise have a type of except_clause */
3408 orelse = ast_for_suite(c, CHILD(n, nch - 4)); 3408 orelse = ast_for_suite(c, CHILD(n, nch - 4));
3409 if (orelse == NULL) 3409 if (orelse == NULL)
3410 return NULL; 3410 return NULL;
3411 n_except--; 3411 n_except--;
3412 } 3412 }
3413 3413
3414 finally = ast_for_suite(c, CHILD(n, nch - 1)); 3414 finally = ast_for_suite(c, CHILD(n, nch - 1));
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after
3497 body = ast_for_suite(c, CHILD(n, NCH(n) - 1)); 3497 body = ast_for_suite(c, CHILD(n, NCH(n) - 1));
3498 if (!body) 3498 if (!body)
3499 return NULL; 3499 return NULL;
3500 3500
3501 return With(items, body, LINENO(n), n->n_col_offset, c->c_arena); 3501 return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
3502 } 3502 }
3503 3503
3504 static stmt_ty 3504 static stmt_ty
3505 ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) 3505 ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
3506 { 3506 {
3507 /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */ 3507 /* classdef: 'class' PYTOK_NAME ['(' arglist ')'] ':' suite */
3508 PyObject *classname; 3508 PyObject *classname;
3509 asdl_seq *s; 3509 asdl_seq *s;
3510 expr_ty call; 3510 expr_ty call;
3511 3511
3512 REQ(n, classdef); 3512 REQ(n, classdef);
3513 3513
3514 if (NCH(n) == 4) { /* class NAME ':' suite */ 3514 if (NCH(n) == 4) { /* class PYTOK_NAME ':' suite */
3515 s = ast_for_suite(c, CHILD(n, 3)); 3515 s = ast_for_suite(c, CHILD(n, 3));
3516 if (!s) 3516 if (!s)
3517 return NULL; 3517 return NULL;
3518 classname = NEW_IDENTIFIER(CHILD(n, 1)); 3518 classname = NEW_IDENTIFIER(CHILD(n, 1));
3519 if (!classname) 3519 if (!classname)
3520 return NULL; 3520 return NULL;
3521 if (forbidden_name(c, classname, CHILD(n, 3), 0)) 3521 if (forbidden_name(c, classname, CHILD(n, 3), 0))
3522 return NULL; 3522 return NULL;
3523 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, 3523 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
3524 LINENO(n), n->n_col_offset, c->c_arena); 3524 LINENO(n), n->n_col_offset, c->c_arena);
3525 } 3525 }
3526 3526
3527 if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ 3527 if (TYPE(CHILD(n, 3)) == PYTOK_RPAR) { /* class PYTOK_NAME '(' ')' ':' suite */
3528 s = ast_for_suite(c, CHILD(n,5)); 3528 s = ast_for_suite(c, CHILD(n,5));
3529 if (!s) 3529 if (!s)
3530 return NULL; 3530 return NULL;
3531 classname = NEW_IDENTIFIER(CHILD(n, 1)); 3531 classname = NEW_IDENTIFIER(CHILD(n, 1));
3532 if (!classname) 3532 if (!classname)
3533 return NULL; 3533 return NULL;
3534 if (forbidden_name(c, classname, CHILD(n, 3), 0)) 3534 if (forbidden_name(c, classname, CHILD(n, 3), 0))
3535 return NULL; 3535 return NULL;
3536 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, 3536 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
3537 LINENO(n), n->n_col_offset, c->c_arena); 3537 LINENO(n), n->n_col_offset, c->c_arena);
3538 } 3538 }
3539 3539
3540 /* class NAME '(' arglist ')' ':' suite */ 3540 /* class PYTOK_NAME '(' arglist ')' ':' suite */
3541 /* build up a fake Call node so we can extract its pieces */ 3541 /* build up a fake Call node so we can extract its pieces */
3542 { 3542 {
3543 PyObject *dummy_name; 3543 PyObject *dummy_name;
3544 expr_ty dummy; 3544 expr_ty dummy;
3545 dummy_name = NEW_IDENTIFIER(CHILD(n, 1)); 3545 dummy_name = NEW_IDENTIFIER(CHILD(n, 1));
3546 if (!dummy_name) 3546 if (!dummy_name)
3547 return NULL; 3547 return NULL;
3548 dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena); 3548 dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena);
3549 call = ast_for_call(c, CHILD(n, 3), dummy); 3549 call = ast_for_call(c, CHILD(n, 3), dummy);
3550 if (!call) 3550 if (!call)
(...skipping 286 matching lines...) Expand 10 before | Expand all | Expand 10 after
3837 } else if (strcmp(c->c_encoding, "utf-8") == 0) { 3837 } else if (strcmp(c->c_encoding, "utf-8") == 0) {
3838 return PyUnicode_FromStringAndSize(s, len); 3838 return PyUnicode_FromStringAndSize(s, len);
3839 } else { 3839 } else {
3840 return PyUnicode_DecodeLatin1(s, len, NULL); 3840 return PyUnicode_DecodeLatin1(s, len, NULL);
3841 } 3841 }
3842 } 3842 }
3843 return PyBytes_DecodeEscape(s, len, NULL, 1, 3843 return PyBytes_DecodeEscape(s, len, NULL, 1,
3844 need_encoding ? c->c_encoding : NULL); 3844 need_encoding ? c->c_encoding : NULL);
3845 } 3845 }
3846 3846
3847 /* Build a Python string object out of a STRING+ atom. This takes care of 3847 /* Build a Python string object out of a PYTOK_STRING+ atom. This takes care of
3848 * compile-time literal catenation, calling parsestr() on each piece, and 3848 * compile-time literal catenation, calling parsestr() on each piece, and
3849 * pasting the intermediate results together. 3849 * pasting the intermediate results together.
3850 */ 3850 */
3851 static PyObject * 3851 static PyObject *
3852 parsestrplus(struct compiling *c, const node *n, int *bytesmode) 3852 parsestrplus(struct compiling *c, const node *n, int *bytesmode)
3853 { 3853 {
3854 PyObject *v; 3854 PyObject *v;
3855 int i; 3855 int i;
3856 REQ(CHILD(n, 0), STRING); 3856 REQ(CHILD(n, 0), PYTOK_STRING);
3857 v = parsestr(c, CHILD(n, 0), bytesmode); 3857 v = parsestr(c, CHILD(n, 0), bytesmode);
3858 if (v != NULL) { 3858 if (v != NULL) {
3859 /* String literal concatenation */ 3859 /* String literal concatenation */
3860 for (i = 1; i < NCH(n); i++) { 3860 for (i = 1; i < NCH(n); i++) {
3861 PyObject *s; 3861 PyObject *s;
3862 int subbm = 0; 3862 int subbm = 0;
3863 s = parsestr(c, CHILD(n, i), &subbm); 3863 s = parsestr(c, CHILD(n, i), &subbm);
3864 if (s == NULL) 3864 if (s == NULL)
3865 goto onError; 3865 goto onError;
3866 if (*bytesmode != subbm) { 3866 if (*bytesmode != subbm) {
(...skipping 15 matching lines...) Expand all
3882 goto onError; 3882 goto onError;
3883 } 3883 }
3884 } 3884 }
3885 } 3885 }
3886 return v; 3886 return v;
3887 3887
3888 onError: 3888 onError:
3889 Py_XDECREF(v); 3889 Py_XDECREF(v);
3890 return NULL; 3890 return NULL;
3891 } 3891 }
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+