Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1)

Side by Side Diff: Modules/parsermodule.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 4 years, 11 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Makefile.pre.in ('k') | Parser/acceler.c » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* parsermodule.c 1 /* parsermodule.c
2 * 2 *
3 * Copyright 1995-1996 by Fred L. Drake, Jr. and Virginia Polytechnic 3 * Copyright 1995-1996 by Fred L. Drake, Jr. and Virginia Polytechnic
4 * Institute and State University, Blacksburg, Virginia, USA. 4 * Institute and State University, Blacksburg, Virginia, USA.
5 * Portions copyright 1991-1995 by Stichting Mathematisch Centrum, 5 * Portions copyright 1991-1995 by Stichting Mathematisch Centrum,
6 * Amsterdam, The Netherlands. Copying is permitted under the terms 6 * Amsterdam, The Netherlands. Copying is permitted under the terms
7 * associated with the main Python distribution, with the additional 7 * associated with the main Python distribution, with the additional
8 * restriction that this additional notice be included and maintained 8 * restriction that this additional notice be included and maintained
9 * on all distributed copies. 9 * on all distributed copies.
10 * 10 *
(...skipping 15 matching lines...) Expand all
26 */ 26 */
27 27
28 #include "Python.h" /* general Python API */ 28 #include "Python.h" /* general Python API */
29 #include "Python-ast.h" /* mod_ty */ 29 #include "Python-ast.h" /* mod_ty */
30 #include "graminit.h" /* symbols defined in the grammar */ 30 #include "graminit.h" /* symbols defined in the grammar */
31 #include "node.h" /* internal parser structure */ 31 #include "node.h" /* internal parser structure */
32 #include "errcode.h" /* error codes for PyNode_*() */ 32 #include "errcode.h" /* error codes for PyNode_*() */
33 #include "token.h" /* token definitions */ 33 #include "token.h" /* token definitions */
34 #include "grammar.h" 34 #include "grammar.h"
35 #include "parsetok.h" 35 #include "parsetok.h"
36 /* ISTERMINAL() / ISNONTERMINAL() */ 36 /* PYTOK_ISTERMINAL() / PYTOK_ISNONTERMI NAL() */
37 #undef Yield 37 #undef Yield
38 #include "ast.h" 38 #include "ast.h"
39 39
40 extern grammar _PyParser_Grammar; /* From graminit.c */ 40 extern grammar _PyParser_Grammar; /* From graminit.c */
41 41
42 #ifdef lint 42 #ifdef lint
43 #include <note.h> 43 #include <note.h>
44 #else 44 #else
45 #define NOTE(x) 45 #define NOTE(x)
46 #endif 46 #endif
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after
83 int lineno, /* include line numbers? */ 83 int lineno, /* include line numbers? */
84 int col_offset) /* include column offsets? */ 84 int col_offset) /* include column offsets? */
85 { 85 {
86 PyObject *result = NULL, *w; 86 PyObject *result = NULL, *w;
87 87
88 if (n == NULL) { 88 if (n == NULL) {
89 Py_INCREF(Py_None); 89 Py_INCREF(Py_None);
90 return Py_None; 90 return Py_None;
91 } 91 }
92 92
93 if (ISNONTERMINAL(TYPE(n))) { 93 if (PYTOK_ISNONTERMINAL(TYPE(n))) {
94 int i; 94 int i;
95 95
96 result = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl)); 96 result = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));
97 if (result == NULL) 97 if (result == NULL)
98 goto error; 98 goto error;
99 99
100 w = PyLong_FromLong(TYPE(n)); 100 w = PyLong_FromLong(TYPE(n));
101 if (w == NULL) 101 if (w == NULL)
102 goto error; 102 goto error;
103 (void) addelem(result, 0, w); 103 (void) addelem(result, 0, w);
104 104
105 for (i = 0; i < NCH(n); i++) { 105 for (i = 0; i < NCH(n); i++) {
106 w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset); 106 w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);
107 if (w == NULL) 107 if (w == NULL)
108 goto error; 108 goto error;
109 (void) addelem(result, i+1, w); 109 (void) addelem(result, i+1, w);
110 } 110 }
111 111
112 if (TYPE(n) == encoding_decl) { 112 if (TYPE(n) == encoding_decl) {
113 w = PyUnicode_FromString(STR(n)); 113 w = PyUnicode_FromString(STR(n));
114 if (w == NULL) 114 if (w == NULL)
115 goto error; 115 goto error;
116 (void) addelem(result, i+1, w); 116 (void) addelem(result, i+1, w);
117 } 117 }
118 } 118 }
119 else if (ISTERMINAL(TYPE(n))) { 119 else if (PYTOK_ISTERMINAL(TYPE(n))) {
120 result = mkseq(2 + lineno + col_offset); 120 result = mkseq(2 + lineno + col_offset);
121 if (result == NULL) 121 if (result == NULL)
122 goto error; 122 goto error;
123 123
124 w = PyLong_FromLong(TYPE(n)); 124 w = PyLong_FromLong(TYPE(n));
125 if (w == NULL) 125 if (w == NULL)
126 goto error; 126 goto error;
127 (void) addelem(result, 0, w); 127 (void) addelem(result, 0, w);
128 128
129 w = PyUnicode_FromString(STR(n)); 129 w = PyUnicode_FromString(STR(n));
(...skipping 132 matching lines...) Expand 10 before | Expand all | Expand 10 after
262 parser_compare_nodes(node *left, node *right) 262 parser_compare_nodes(node *left, node *right)
263 { 263 {
264 int j; 264 int j;
265 265
266 if (TYPE(left) < TYPE(right)) 266 if (TYPE(left) < TYPE(right))
267 return (-1); 267 return (-1);
268 268
269 if (TYPE(right) < TYPE(left)) 269 if (TYPE(right) < TYPE(left))
270 return (1); 270 return (1);
271 271
272 if (ISTERMINAL(TYPE(left))) 272 if (PYTOK_ISTERMINAL(TYPE(left)))
273 return (strcmp(STR(left), STR(right))); 273 return (strcmp(STR(left), STR(right)));
274 274
275 if (NCH(left) < NCH(right)) 275 if (NCH(left) < NCH(right))
276 return (-1); 276 return (-1);
277 277
278 if (NCH(right) < NCH(left)) 278 if (NCH(right) < NCH(left))
279 return (1); 279 return (1);
280 280
281 for (j = 0; j < NCH(left); ++j) { 281 for (j = 0; j < NCH(left); ++j) {
282 int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j)); 282 int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j));
(...skipping 502 matching lines...) Expand 10 before | Expand all | Expand 10 after
785 } 785 }
786 } 786 }
787 if (!ok) { 787 if (!ok) {
788 PyObject *err = Py_BuildValue("Os", elem, 788 PyObject *err = Py_BuildValue("Os", elem,
789 "Illegal node construct."); 789 "Illegal node construct.");
790 PyErr_SetObject(parser_error, err); 790 PyErr_SetObject(parser_error, err);
791 Py_XDECREF(err); 791 Py_XDECREF(err);
792 Py_XDECREF(elem); 792 Py_XDECREF(elem);
793 return (0); 793 return (0);
794 } 794 }
795 if (ISTERMINAL(type)) { 795 if (PYTOK_ISTERMINAL(type)) {
796 Py_ssize_t len = PyObject_Size(elem); 796 Py_ssize_t len = PyObject_Size(elem);
797 PyObject *temp; 797 PyObject *temp;
798 const char *temp_str; 798 const char *temp_str;
799 799
800 if ((len != 2) && (len != 3)) { 800 if ((len != 2) && (len != 3)) {
801 err_string("terminal nodes must have 2 or 3 entries"); 801 err_string("terminal nodes must have 2 or 3 entries");
802 return 0; 802 return 0;
803 } 803 }
804 temp = PySequence_GetItem(elem, 1); 804 temp = PySequence_GetItem(elem, 1);
805 if (temp == NULL) 805 if (temp == NULL)
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
848 strn = (char *)PyObject_MALLOC(len + 1); 848 strn = (char *)PyObject_MALLOC(len + 1);
849 if (strn == NULL) { 849 if (strn == NULL) {
850 Py_DECREF(temp); 850 Py_DECREF(temp);
851 Py_XDECREF(elem); 851 Py_XDECREF(elem);
852 PyErr_NoMemory(); 852 PyErr_NoMemory();
853 return 0; 853 return 0;
854 } 854 }
855 (void) memcpy(strn, temp_str, len + 1); 855 (void) memcpy(strn, temp_str, len + 1);
856 Py_DECREF(temp); 856 Py_DECREF(temp);
857 } 857 }
858 else if (!ISNONTERMINAL(type)) { 858 else if (!PYTOK_ISNONTERMINAL(type)) {
859 /* 859 /*
860 * It has to be one or the other; this is an error. 860 * It has to be one or the other; this is an error.
861 * Raise an exception. 861 * Raise an exception.
862 */ 862 */
863 PyObject *err = Py_BuildValue("os", elem, "unknown node type."); 863 PyObject *err = Py_BuildValue("os", elem, "unknown node type.");
864 PyErr_SetObject(parser_error, err); 864 PyErr_SetObject(parser_error, err);
865 Py_XDECREF(err); 865 Py_XDECREF(err);
866 Py_XDECREF(elem); 866 Py_XDECREF(elem);
867 return (0); 867 return (0);
868 } 868 }
869 err = PyNode_AddChild(root, type, strn, *line_num, 0); 869 err = PyNode_AddChild(root, type, strn, *line_num, 0);
870 if (err == E_NOMEM) { 870 if (err == E_NOMEM) {
871 Py_XDECREF(elem); 871 Py_XDECREF(elem);
872 PyObject_FREE(strn); 872 PyObject_FREE(strn);
873 return (node *) PyErr_NoMemory(); 873 return (node *) PyErr_NoMemory();
874 } 874 }
875 if (err == E_OVERFLOW) { 875 if (err == E_OVERFLOW) {
876 Py_XDECREF(elem); 876 Py_XDECREF(elem);
877 PyObject_FREE(strn); 877 PyObject_FREE(strn);
878 PyErr_SetString(PyExc_ValueError, 878 PyErr_SetString(PyExc_ValueError,
879 "unsupported number of child nodes"); 879 "unsupported number of child nodes");
880 return NULL; 880 return NULL;
881 } 881 }
882 882
883 if (ISNONTERMINAL(type)) { 883 if (PYTOK_ISNONTERMINAL(type)) {
884 node* new_child = CHILD(root, i - 1); 884 node* new_child = CHILD(root, i - 1);
885 885
886 if (new_child != build_node_children(elem, new_child, line_num)) { 886 if (new_child != build_node_children(elem, new_child, line_num)) {
887 Py_XDECREF(elem); 887 Py_XDECREF(elem);
888 return (0); 888 return (0);
889 } 889 }
890 } 890 }
891 else if (type == NEWLINE) { /* It's true: we increment the */ 891 else if (type == PYTOK_NEWLINE) { /* It's true: we increment the */
892 ++(*line_num); /* line number *after* the newline! */ 892 ++(*line_num); /* line number *after* the newline! */
893 } 893 }
894 Py_XDECREF(elem); 894 Py_XDECREF(elem);
895 } 895 }
896 return root; 896 return root;
897 } 897 }
898 898
899 899
900 static node* 900 static node*
901 build_node_tree(PyObject *tuple) 901 build_node_tree(PyObject *tuple)
902 { 902 {
903 node* res = 0; 903 node* res = 0;
904 PyObject *temp = PySequence_GetItem(tuple, 0); 904 PyObject *temp = PySequence_GetItem(tuple, 0);
905 long num = -1; 905 long num = -1;
906 906
907 if (temp != NULL) 907 if (temp != NULL)
908 num = PyLong_AsLong(temp); 908 num = PyLong_AsLong(temp);
909 Py_XDECREF(temp); 909 Py_XDECREF(temp);
910 if (ISTERMINAL(num)) { 910 if (PYTOK_ISTERMINAL(num)) {
911 /* 911 /*
912 * The tuple is simple, but it doesn't start with a start symbol. 912 * The tuple is simple, but it doesn't start with a start symbol.
913 * Raise an exception now and be done with it. 913 * Raise an exception now and be done with it.
914 */ 914 */
915 tuple = Py_BuildValue("Os", tuple, 915 tuple = Py_BuildValue("Os", tuple,
916 "Illegal syntax-tree; cannot start with terminal symbol."); 916 "Illegal syntax-tree; cannot start with terminal symbol.");
917 PyErr_SetObject(parser_error, tuple); 917 PyErr_SetObject(parser_error, tuple);
918 Py_XDECREF(tuple); 918 Py_XDECREF(tuple);
919 } 919 }
920 else if (ISNONTERMINAL(num)) { 920 else if (PYTOK_ISNONTERMINAL(num)) {
921 /* 921 /*
922 * Not efficient, but that can be handled later. 922 * Not efficient, but that can be handled later.
923 */ 923 */
924 int line_num = 0; 924 int line_num = 0;
925 PyObject *encoding = NULL; 925 PyObject *encoding = NULL;
926 926
927 if (num == encoding_decl) { 927 if (num == encoding_decl) {
928 encoding = PySequence_GetItem(tuple, 2); 928 encoding = PySequence_GetItem(tuple, 2);
929 /* tuple isn't borrowed anymore here, need to DECREF */ 929 /* tuple isn't borrowed anymore here, need to DECREF */
930 tuple = PySequence_GetSlice(tuple, 0, 2); 930 tuple = PySequence_GetSlice(tuple, 0, 2);
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
974 974
975 return (res); 975 return (res);
976 } 976 }
977 977
978 978
979 /* 979 /*
980 * Validation routines used within the validation section: 980 * Validation routines used within the validation section:
981 */ 981 */
982 static int validate_terminal(node *terminal, int type, char *string); 982 static int validate_terminal(node *terminal, int type, char *string);
983 983
984 #define validate_ampersand(ch) validate_terminal(ch, AMPER, "&") 984 #define validate_ampersand(ch) validate_terminal(ch, PYTOK_AMPER, "&")
985 #define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^") 985 #define validate_circumflex(ch) validate_terminal(ch, PYTOK_CIRCUMFLEX, "^")
986 #define validate_colon(ch) validate_terminal(ch, COLON, ":") 986 #define validate_colon(ch) validate_terminal(ch, PYTOK_COLON, ":")
987 #define validate_comma(ch) validate_terminal(ch, COMMA, ",") 987 #define validate_comma(ch) validate_terminal(ch, PYTOK_COMMA, ",")
988 #define validate_dedent(ch) validate_terminal(ch, DEDENT, "") 988 #define validate_dedent(ch) validate_terminal(ch, PYTOK_DEDENT, "")
989 #define validate_equal(ch) validate_terminal(ch, EQUAL, "=") 989 #define validate_equal(ch) validate_terminal(ch, PYTOK_EQUAL, "=")
990 #define validate_indent(ch) validate_terminal(ch, INDENT, (char*)NULL) 990 #define validate_indent(ch) validate_terminal(ch, PYTOK_INDENT, (char*)N ULL)
991 #define validate_lparen(ch) validate_terminal(ch, LPAR, "(") 991 #define validate_lparen(ch) validate_terminal(ch, PYTOK_LPAR, "(")
992 #define validate_newline(ch) validate_terminal(ch, NEWLINE, (char*)NULL) 992 #define validate_newline(ch) validate_terminal(ch, PYTOK_NEWLINE, (char*)N ULL)
993 #define validate_rparen(ch) validate_terminal(ch, RPAR, ")") 993 #define validate_rparen(ch) validate_terminal(ch, PYTOK_RPAR, ")")
994 #define validate_semi(ch) validate_terminal(ch, SEMI, ";") 994 #define validate_semi(ch) validate_terminal(ch, PYTOK_SEMI, ";")
995 #define validate_star(ch) validate_terminal(ch, STAR, "*") 995 #define validate_star(ch) validate_terminal(ch, PYTOK_STAR, "*")
996 #define validate_vbar(ch) validate_terminal(ch, VBAR, "|") 996 #define validate_vbar(ch) validate_terminal(ch, PYTOK_VBAR, "|")
997 #define validate_doublestar(ch) validate_terminal(ch, DOUBLESTAR, "**") 997 #define validate_doublestar(ch) validate_terminal(ch, PYTOK_DOUBLESTAR, "**")
998 #define validate_dot(ch) validate_terminal(ch, DOT, ".") 998 #define validate_dot(ch) validate_terminal(ch, PYTOK_DOT, ".")
999 #define validate_at(ch) validate_terminal(ch, AT, "@") 999 #define validate_at(ch) validate_terminal(ch, PYTOK_AT, "@")
1000 #define validate_rarrow(ch) validate_terminal(ch, RARROW, "->") 1000 #define validate_rarrow(ch) validate_terminal(ch, PYTOK_RARROW, "->")
1001 #define validate_name(ch, str) validate_terminal(ch, NAME, str) 1001 #define validate_name(ch, str) validate_terminal(ch, PYTOK_NAME, str)
1002 1002
1003 #define VALIDATER(n) static int validate_##n(node *tree) 1003 #define VALIDATER(n) static int validate_##n(node *tree)
1004 1004
1005 VALIDATER(node); VALIDATER(small_stmt); 1005 VALIDATER(node); VALIDATER(small_stmt);
1006 VALIDATER(class); VALIDATER(node); 1006 VALIDATER(class); VALIDATER(node);
1007 VALIDATER(parameters); VALIDATER(suite); 1007 VALIDATER(parameters); VALIDATER(suite);
1008 VALIDATER(testlist); VALIDATER(varargslist); 1008 VALIDATER(testlist); VALIDATER(varargslist);
1009 VALIDATER(vfpdef); 1009 VALIDATER(vfpdef);
1010 VALIDATER(stmt); VALIDATER(simple_stmt); 1010 VALIDATER(stmt); VALIDATER(simple_stmt);
1011 VALIDATER(expr_stmt); VALIDATER(power); 1011 VALIDATER(expr_stmt); VALIDATER(power);
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after
1110 && vfunc(CHILD(tree, pos + 1))); 1110 && vfunc(CHILD(tree, pos + 1)));
1111 } 1111 }
1112 } 1112 }
1113 return (res); 1113 return (res);
1114 } 1114 }
1115 1115
1116 1116
1117 /* validate_class() 1117 /* validate_class()
1118 * 1118 *
1119 * classdef: 1119 * classdef:
1120 * 'class' NAME ['(' testlist ')'] ':' suite 1120 * 'class' PYTOK_NAME ['(' testlist ')'] ':' suite
1121 */ 1121 */
1122 static int 1122 static int
1123 validate_class(node *tree) 1123 validate_class(node *tree)
1124 { 1124 {
1125 int nch = NCH(tree); 1125 int nch = NCH(tree);
1126 int res = (validate_ntype(tree, classdef) && 1126 int res = (validate_ntype(tree, classdef) &&
1127 ((nch == 4) || (nch == 6) || (nch == 7))); 1127 ((nch == 4) || (nch == 6) || (nch == 7)));
1128 1128
1129 if (res) { 1129 if (res) {
1130 res = (validate_name(CHILD(tree, 0), "class") 1130 res = (validate_name(CHILD(tree, 0), "class")
1131 && validate_ntype(CHILD(tree, 1), NAME) 1131 && validate_ntype(CHILD(tree, 1), PYTOK_NAME)
1132 && validate_colon(CHILD(tree, nch - 2)) 1132 && validate_colon(CHILD(tree, nch - 2))
1133 && validate_suite(CHILD(tree, nch - 1))); 1133 && validate_suite(CHILD(tree, nch - 1)));
1134 } 1134 }
1135 else { 1135 else {
1136 (void) validate_numnodes(tree, 4, "class"); 1136 (void) validate_numnodes(tree, 4, "class");
1137 } 1137 }
1138 1138
1139 if (res) { 1139 if (res) {
1140 if (nch == 7) { 1140 if (nch == 7) {
1141 res = ((validate_lparen(CHILD(tree, 2)) && 1141 res = ((validate_lparen(CHILD(tree, 2)) &&
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after
1212 (void) validate_numnodes(tree, 2, "parameters"); 1212 (void) validate_numnodes(tree, 2, "parameters");
1213 } 1213 }
1214 return (res); 1214 return (res);
1215 } 1215 }
1216 1216
1217 1217
1218 /* validate_suite() 1218 /* validate_suite()
1219 * 1219 *
1220 * suite: 1220 * suite:
1221 * simple_stmt 1221 * simple_stmt
1222 * | NEWLINE INDENT stmt+ DEDENT 1222 * | PYTOK_NEWLINE PYTOK_INDENT stmt+ PYTOK_DEDENT
1223 */ 1223 */
1224 static int 1224 static int
1225 validate_suite(node *tree) 1225 validate_suite(node *tree)
1226 { 1226 {
1227 int nch = NCH(tree); 1227 int nch = NCH(tree);
1228 int res = (validate_ntype(tree, suite) && ((nch == 1) || (nch >= 4))); 1228 int res = (validate_ntype(tree, suite) && ((nch == 1) || (nch >= 4)));
1229 1229
1230 if (res && (nch == 1)) 1230 if (res && (nch == 1))
1231 res = validate_simple_stmt(CHILD(tree, 0)); 1231 res = validate_simple_stmt(CHILD(tree, 0));
1232 else if (res) { 1232 else if (res) {
1233 /* NEWLINE INDENT stmt+ DEDENT */ 1233 /* PYTOK_NEWLINE PYTOK_INDENT stmt+ PYTOK_DEDENT */
1234 res = (validate_newline(CHILD(tree, 0)) 1234 res = (validate_newline(CHILD(tree, 0))
1235 && validate_indent(CHILD(tree, 1)) 1235 && validate_indent(CHILD(tree, 1))
1236 && validate_stmt(CHILD(tree, 2)) 1236 && validate_stmt(CHILD(tree, 2))
1237 && validate_dedent(CHILD(tree, nch - 1))); 1237 && validate_dedent(CHILD(tree, nch - 1)));
1238 1238
1239 if (res && (nch > 4)) { 1239 if (res && (nch > 4)) {
1240 int i = 3; 1240 int i = 3;
1241 --nch; /* forget the DEDENT */ 1241 --nch; /* forget the PYTOK_DEDENT */
1242 for ( ; res && (i < nch); ++i) 1242 for ( ; res && (i < nch); ++i)
1243 res = validate_stmt(CHILD(tree, i)); 1243 res = validate_stmt(CHILD(tree, i));
1244 } 1244 }
1245 else if (nch < 4) 1245 else if (nch < 4)
1246 res = validate_numnodes(tree, 4, "suite"); 1246 res = validate_numnodes(tree, 4, "suite");
1247 } 1247 }
1248 return (res); 1248 return (res);
1249 } 1249 }
1250 1250
1251 1251
1252 static int 1252 static int
1253 validate_testlist(node *tree) 1253 validate_testlist(node *tree)
1254 { 1254 {
1255 return (validate_repeating_list(tree, testlist, 1255 return (validate_repeating_list(tree, testlist,
1256 validate_test, "testlist")); 1256 validate_test, "testlist"));
1257 } 1257 }
1258 1258
1259 static int 1259 static int
1260 validate_testlist_star_expr(node *tl) 1260 validate_testlist_star_expr(node *tl)
1261 { 1261 {
1262 return (validate_repeating_list(tl, testlist_star_expr, validate_test_or_sta r_expr, 1262 return (validate_repeating_list(tl, testlist_star_expr, validate_test_or_sta r_expr,
1263 "testlist")); 1263 "testlist"));
1264 } 1264 }
1265 1265
1266 1266
1267 /* validate either vfpdef or tfpdef. 1267 /* validate either vfpdef or tfpdef.
1268 * vfpdef: NAME 1268 * vfpdef: PYTOK_NAME
1269 * tfpdef: NAME [':' test] 1269 * tfpdef: PYTOK_NAME [':' test]
1270 */ 1270 */
1271 static int 1271 static int
1272 validate_vfpdef(node *tree) 1272 validate_vfpdef(node *tree)
1273 { 1273 {
1274 int nch = NCH(tree); 1274 int nch = NCH(tree);
1275 if (TYPE(tree) == vfpdef) { 1275 if (TYPE(tree) == vfpdef) {
1276 return nch == 1 && validate_name(CHILD(tree, 0), NULL); 1276 return nch == 1 && validate_name(CHILD(tree, 0), NULL);
1277 } 1277 }
1278 else if (TYPE(tree) == tfpdef) { 1278 else if (TYPE(tree) == tfpdef) {
1279 if (nch == 1) { 1279 if (nch == 1) {
1280 return validate_name(CHILD(tree, 0), NULL); 1280 return validate_name(CHILD(tree, 0), NULL);
1281 } 1281 }
1282 else if (nch == 3) { 1282 else if (nch == 3) {
1283 return validate_name(CHILD(tree, 0), NULL) && 1283 return validate_name(CHILD(tree, 0), NULL) &&
1284 validate_colon(CHILD(tree, 1)) && 1284 validate_colon(CHILD(tree, 1)) &&
1285 validate_test(CHILD(tree, 2)); 1285 validate_test(CHILD(tree, 2));
1286 } 1286 }
1287 } 1287 }
1288 return 0; 1288 return 0;
1289 } 1289 }
1290 1290
1291 /* '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef 1291 /* '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
1292 * ..or tfpdef in place of vfpdef. vfpdef: NAME; tfpdef: NAME [':' test] 1292 * ..or tfpdef in place of vfpdef. vfpdef: PYTOK_NAME; tfpdef: PYTOK_NAME [':' t est]
1293 */ 1293 */
1294 static int 1294 static int
1295 validate_varargslist_trailer(node *tree, int start) 1295 validate_varargslist_trailer(node *tree, int start)
1296 { 1296 {
1297 int nch = NCH(tree); 1297 int nch = NCH(tree);
1298 int res = 0; 1298 int res = 0;
1299 1299
1300 if (nch <= start) { 1300 if (nch <= start) {
1301 err_string("expected variable argument trailer for varargslist"); 1301 err_string("expected variable argument trailer for varargslist");
1302 return 0; 1302 return 0;
1303 } 1303 }
1304 if (TYPE(CHILD(tree, start)) == STAR) { 1304 if (TYPE(CHILD(tree, start)) == PYTOK_STAR) {
1305 /* 1305 /*
1306 * '*' [vfpdef] 1306 * '*' [vfpdef]
1307 */ 1307 */
1308 res = validate_star(CHILD(tree, start++)); 1308 res = validate_star(CHILD(tree, start++));
1309 if (res && start < nch && (TYPE(CHILD(tree, start)) == vfpdef || 1309 if (res && start < nch && (TYPE(CHILD(tree, start)) == vfpdef ||
1310 TYPE(CHILD(tree, start)) == tfpdef)) 1310 TYPE(CHILD(tree, start)) == tfpdef))
1311 res = validate_vfpdef(CHILD(tree, start++)); 1311 res = validate_vfpdef(CHILD(tree, start++));
1312 /* 1312 /*
1313 * (',' vfpdef ['=' test])* 1313 * (',' vfpdef ['=' test])*
1314 */ 1314 */
1315 while (res && start + 1 < nch && ( 1315 while (res && start + 1 < nch && (
1316 TYPE(CHILD(tree, start + 1)) == vfpdef || 1316 TYPE(CHILD(tree, start + 1)) == vfpdef ||
1317 TYPE(CHILD(tree, start + 1)) == tfpdef)) { 1317 TYPE(CHILD(tree, start + 1)) == tfpdef)) {
1318 res = (validate_comma(CHILD(tree, start++)) 1318 res = (validate_comma(CHILD(tree, start++))
1319 && validate_vfpdef(CHILD(tree, start++))); 1319 && validate_vfpdef(CHILD(tree, start++)));
1320 if (res && start + 1 < nch && TYPE(CHILD(tree, start)) == EQUAL) 1320 if (res && start + 1 < nch && TYPE(CHILD(tree, start)) == PYTOK_EQUA L)
1321 res = (validate_equal(CHILD(tree, start++)) 1321 res = (validate_equal(CHILD(tree, start++))
1322 && validate_test(CHILD(tree, start++))); 1322 && validate_test(CHILD(tree, start++)));
1323 } 1323 }
1324 /* 1324 /*
1325 * [',' '**' vfpdef] 1325 * [',' '**' vfpdef]
1326 */ 1326 */
1327 if (res && start + 2 < nch && TYPE(CHILD(tree, start+1)) == DOUBLESTAR) 1327 if (res && start + 2 < nch && TYPE(CHILD(tree, start+1)) == PYTOK_DOUBLE STAR)
1328 res = (validate_comma(CHILD(tree, start++)) 1328 res = (validate_comma(CHILD(tree, start++))
1329 && validate_doublestar(CHILD(tree, start++)) 1329 && validate_doublestar(CHILD(tree, start++))
1330 && validate_vfpdef(CHILD(tree, start++))); 1330 && validate_vfpdef(CHILD(tree, start++)));
1331 } 1331 }
1332 else if (TYPE(CHILD(tree, start)) == DOUBLESTAR) { 1332 else if (TYPE(CHILD(tree, start)) == PYTOK_DOUBLESTAR) {
1333 /* 1333 /*
1334 * '**' vfpdef 1334 * '**' vfpdef
1335 */ 1335 */
1336 if (start + 1 < nch) 1336 if (start + 1 < nch)
1337 res = (validate_doublestar(CHILD(tree, start++)) 1337 res = (validate_doublestar(CHILD(tree, start++))
1338 && validate_vfpdef(CHILD(tree, start++))); 1338 && validate_vfpdef(CHILD(tree, start++)));
1339 else { 1339 else {
1340 res = 0; 1340 res = 0;
1341 err_string("expected vfpdef after ** in varargslist trailer"); 1341 err_string("expected vfpdef after ** in varargslist trailer");
1342 } 1342 }
(...skipping 12 matching lines...) Expand all
1355 1355
1356 1356
1357 /* validate_varargslist() 1357 /* validate_varargslist()
1358 * 1358 *
1359 * Validate typedargslist or varargslist. 1359 * Validate typedargslist or varargslist.
1360 * 1360 *
1361 * typedargslist: ((tfpdef ['=' test] ',')* 1361 * typedargslist: ((tfpdef ['=' test] ',')*
1362 * ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | 1362 * ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] |
1363 * '**' tfpdef) 1363 * '**' tfpdef)
1364 * | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) 1364 * | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
1365 * tfpdef: NAME [':' test] 1365 * tfpdef: PYTOK_NAME [':' test]
1366 * varargslist: ((vfpdef ['=' test] ',')* 1366 * varargslist: ((vfpdef ['=' test] ',')*
1367 * ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | 1367 * ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] |
1368 * '**' vfpdef) 1368 * '**' vfpdef)
1369 * | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) 1369 * | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
1370 * vfpdef: NAME 1370 * vfpdef: PYTOK_NAME
1371 * 1371 *
1372 */ 1372 */
1373 static int 1373 static int
1374 validate_varargslist(node *tree) 1374 validate_varargslist(node *tree)
1375 { 1375 {
1376 int nch = NCH(tree); 1376 int nch = NCH(tree);
1377 int res = (TYPE(tree) == varargslist || 1377 int res = (TYPE(tree) == varargslist ||
1378 TYPE(tree) == typedargslist) && 1378 TYPE(tree) == typedargslist) &&
1379 (nch != 0); 1379 (nch != 0);
1380 int sym; 1380 int sym;
1381 node *ch; 1381 node *ch;
1382 int i = 0; 1382 int i = 0;
1383 1383
1384 if (!res) 1384 if (!res)
1385 return 0; 1385 return 0;
1386 if (nch < 1) { 1386 if (nch < 1) {
1387 err_string("varargslist missing child nodes"); 1387 err_string("varargslist missing child nodes");
1388 return 0; 1388 return 0;
1389 } 1389 }
1390 while (i < nch) { 1390 while (i < nch) {
1391 ch = CHILD(tree, i); 1391 ch = CHILD(tree, i);
1392 sym = TYPE(ch); 1392 sym = TYPE(ch);
1393 if (sym == vfpdef || sym == tfpdef) { 1393 if (sym == vfpdef || sym == tfpdef) {
1394 /* validate (vfpdef ['=' test] ',')+ */ 1394 /* validate (vfpdef ['=' test] ',')+ */
1395 res = validate_vfpdef(ch); 1395 res = validate_vfpdef(ch);
1396 ++i; 1396 ++i;
1397 if (res && (i+2 <= nch) && TYPE(CHILD(tree, i)) == EQUAL) { 1397 if (res && (i+2 <= nch) && TYPE(CHILD(tree, i)) == PYTOK_EQUAL) {
1398 res = (validate_equal(CHILD(tree, i)) 1398 res = (validate_equal(CHILD(tree, i))
1399 && validate_test(CHILD(tree, i+1))); 1399 && validate_test(CHILD(tree, i+1)));
1400 if (res) 1400 if (res)
1401 i += 2; 1401 i += 2;
1402 } 1402 }
1403 if (res && i < nch) { 1403 if (res && i < nch) {
1404 res = validate_comma(CHILD(tree, i)); 1404 res = validate_comma(CHILD(tree, i));
1405 ++i; 1405 ++i;
1406 } 1406 }
1407 } else if (sym == DOUBLESTAR || sym == STAR) { 1407 } else if (sym == PYTOK_DOUBLESTAR || sym == PYTOK_STAR) {
1408 res = validate_varargslist_trailer(tree, i); 1408 res = validate_varargslist_trailer(tree, i);
1409 break; 1409 break;
1410 } else { 1410 } else {
1411 res = 0; 1411 res = 0;
1412 err_string("illegal formation for varargslist"); 1412 err_string("illegal formation for varargslist");
1413 } 1413 }
1414 } 1414 }
1415 return res; 1415 return res;
1416 } 1416 }
1417 1417
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after
1488 1488
1489 if (TYPE(tree) == simple_stmt) 1489 if (TYPE(tree) == simple_stmt)
1490 res = validate_simple_stmt(tree); 1490 res = validate_simple_stmt(tree);
1491 else 1491 else
1492 res = validate_compound_stmt(tree); 1492 res = validate_compound_stmt(tree);
1493 } 1493 }
1494 return (res); 1494 return (res);
1495 } 1495 }
1496 1496
1497 1497
1498 /* small_stmt (';' small_stmt)* [';'] NEWLINE 1498 /* small_stmt (';' small_stmt)* [';'] PYTOK_NEWLINE
1499 * 1499 *
1500 */ 1500 */
1501 static int 1501 static int
1502 validate_simple_stmt(node *tree) 1502 validate_simple_stmt(node *tree)
1503 { 1503 {
1504 int nch = NCH(tree); 1504 int nch = NCH(tree);
1505 int res = (validate_ntype(tree, simple_stmt) 1505 int res = (validate_ntype(tree, simple_stmt)
1506 && (nch >= 2) 1506 && (nch >= 2)
1507 && validate_small_stmt(CHILD(tree, 0)) 1507 && validate_small_stmt(CHILD(tree, 0))
1508 && validate_newline(CHILD(tree, nch - 1))); 1508 && validate_newline(CHILD(tree, nch - 1)));
1509 1509
1510 if (nch < 2) 1510 if (nch < 2)
1511 res = validate_numnodes(tree, 2, "simple_stmt"); 1511 res = validate_numnodes(tree, 2, "simple_stmt");
1512 --nch; /* forget the NEWLINE */ 1512 --nch; /* forget the PYTOK_NEWLINE */
1513 if (res && is_even(nch)) 1513 if (res && is_even(nch))
1514 res = validate_semi(CHILD(tree, --nch)); 1514 res = validate_semi(CHILD(tree, --nch));
1515 if (res && (nch > 2)) { 1515 if (res && (nch > 2)) {
1516 int i; 1516 int i;
1517 1517
1518 for (i = 1; res && (i < nch); i += 2) 1518 for (i = 1; res && (i < nch); i += 2)
1519 res = (validate_semi(CHILD(tree, i)) 1519 res = (validate_semi(CHILD(tree, i))
1520 && validate_small_stmt(CHILD(tree, i + 1))); 1520 && validate_small_stmt(CHILD(tree, i + 1)));
1521 } 1521 }
1522 return (res); 1522 return (res);
(...skipping 241 matching lines...) Expand 10 before | Expand all | Expand 10 after
1764 ok = (validate_name(CHILD(tree, 0), NULL) 1764 ok = (validate_name(CHILD(tree, 0), NULL)
1765 && validate_name(CHILD(tree, 1), "as") 1765 && validate_name(CHILD(tree, 1), "as")
1766 && validate_name(CHILD(tree, 2), NULL)); 1766 && validate_name(CHILD(tree, 2), NULL));
1767 else 1767 else
1768 ok = validate_numnodes(tree, 3, "import_as_name"); 1768 ok = validate_numnodes(tree, 3, "import_as_name");
1769 } 1769 }
1770 return ok; 1770 return ok;
1771 } 1771 }
1772 1772
1773 1773
1774 /* dotted_name: NAME ("." NAME)* 1774 /* dotted_name: PYTOK_NAME ("." PYTOK_NAME)*
1775 */ 1775 */
1776 static int 1776 static int
1777 validate_dotted_name(node *tree) 1777 validate_dotted_name(node *tree)
1778 { 1778 {
1779 int nch = NCH(tree); 1779 int nch = NCH(tree);
1780 int res = (validate_ntype(tree, dotted_name) 1780 int res = (validate_ntype(tree, dotted_name)
1781 && is_odd(nch) 1781 && is_odd(nch)
1782 && validate_name(CHILD(tree, 0), NULL)); 1782 && validate_name(CHILD(tree, 0), NULL));
1783 int i; 1783 int i;
1784 1784
1785 for (i = 1; res && (i < nch); i += 2) { 1785 for (i = 1; res && (i < nch); i += 2) {
1786 res = (validate_dot(CHILD(tree, i)) 1786 res = (validate_dot(CHILD(tree, i))
1787 && validate_name(CHILD(tree, i+1), NULL)); 1787 && validate_name(CHILD(tree, i+1), NULL));
1788 } 1788 }
1789 return res; 1789 return res;
1790 } 1790 }
1791 1791
1792 1792
1793 /* dotted_as_name: dotted_name [NAME NAME] 1793 /* dotted_as_name: dotted_name [PYTOK_NAME PYTOK_NAME]
1794 */ 1794 */
1795 static int 1795 static int
1796 validate_dotted_as_name(node *tree) 1796 validate_dotted_as_name(node *tree)
1797 { 1797 {
1798 int nch = NCH(tree); 1798 int nch = NCH(tree);
1799 int res = validate_ntype(tree, dotted_as_name); 1799 int res = validate_ntype(tree, dotted_as_name);
1800 1800
1801 if (res) { 1801 if (res) {
1802 if (nch == 1) 1802 if (nch == 1)
1803 res = validate_dotted_name(CHILD(tree, 0)); 1803 res = validate_dotted_name(CHILD(tree, 0));
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after
1855 } 1855 }
1856 1856
1857 /* Helper function to count the number of leading dots (or ellipsis tokens) in 1857 /* Helper function to count the number of leading dots (or ellipsis tokens) in
1858 * 'from ...module import name' 1858 * 'from ...module import name'
1859 */ 1859 */
1860 static int 1860 static int
1861 count_from_dots(node *tree) 1861 count_from_dots(node *tree)
1862 { 1862 {
1863 int i; 1863 int i;
1864 for (i = 1; i < NCH(tree); i++) 1864 for (i = 1; i < NCH(tree); i++)
1865 if (TYPE(CHILD(tree, i)) != DOT && TYPE(CHILD(tree, i)) != ELLIPSIS) 1865 if (TYPE(CHILD(tree, i)) != PYTOK_DOT && TYPE(CHILD(tree, i)) != PYTOK_E LLIPSIS)
1866 break; 1866 break;
1867 return i - 1; 1867 return i - 1;
1868 } 1868 }
1869 1869
1870 /* import_from: ('from' ('.'* dotted_name | '.'+) 1870 /* import_from: ('from' ('.'* dotted_name | '.'+)
1871 * 'import' ('*' | '(' import_as_names ')' | import_as_names)) 1871 * 'import' ('*' | '(' import_as_names ')' | import_as_names))
1872 */ 1872 */
1873 static int 1873 static int
1874 validate_import_from(node *tree) 1874 validate_import_from(node *tree)
1875 { 1875 {
1876 int nch = NCH(tree); 1876 int nch = NCH(tree);
1877 int ndots = count_from_dots(tree); 1877 int ndots = count_from_dots(tree);
1878 int havename = (TYPE(CHILD(tree, ndots + 1)) == dotted_name); 1878 int havename = (TYPE(CHILD(tree, ndots + 1)) == dotted_name);
1879 int offset = ndots + havename; 1879 int offset = ndots + havename;
1880 int res = validate_ntype(tree, import_from) 1880 int res = validate_ntype(tree, import_from)
1881 && (offset >= 1) 1881 && (offset >= 1)
1882 && (nch >= 3 + offset) 1882 && (nch >= 3 + offset)
1883 && validate_name(CHILD(tree, 0), "from") 1883 && validate_name(CHILD(tree, 0), "from")
1884 && (!havename || validate_dotted_name(CHILD(tree, ndots + 1))) 1884 && (!havename || validate_dotted_name(CHILD(tree, ndots + 1)))
1885 && validate_name(CHILD(tree, offset + 1), "import"); 1885 && validate_name(CHILD(tree, offset + 1), "import");
1886 1886
1887 if (res && TYPE(CHILD(tree, offset + 2)) == LPAR) 1887 if (res && TYPE(CHILD(tree, offset + 2)) == PYTOK_LPAR)
1888 res = ((nch == offset + 5) 1888 res = ((nch == offset + 5)
1889 && validate_lparen(CHILD(tree, offset + 2)) 1889 && validate_lparen(CHILD(tree, offset + 2))
1890 && validate_import_as_names(CHILD(tree, offset + 3)) 1890 && validate_import_as_names(CHILD(tree, offset + 3))
1891 && validate_rparen(CHILD(tree, offset + 4))); 1891 && validate_rparen(CHILD(tree, offset + 4)));
1892 else if (res && TYPE(CHILD(tree, offset + 2)) != STAR) 1892 else if (res && TYPE(CHILD(tree, offset + 2)) != PYTOK_STAR)
1893 res = validate_import_as_names(CHILD(tree, offset + 2)); 1893 res = validate_import_as_names(CHILD(tree, offset + 2));
1894 return (res); 1894 return (res);
1895 } 1895 }
1896 1896
1897 1897
1898 /* import_stmt: import_name | import_from */ 1898 /* import_stmt: import_name | import_from */
1899 static int 1899 static int
1900 validate_import_stmt(node *tree) 1900 validate_import_stmt(node *tree)
1901 { 1901 {
1902 int nch = NCH(tree); 1902 int nch = NCH(tree);
(...skipping 14 matching lines...) Expand all
1917 PyErr_Format(parser_error, 1917 PyErr_Format(parser_error,
1918 "Unrecognized child node of import_stmt: %d.", 1918 "Unrecognized child node of import_stmt: %d.",
1919 TYPE(CHILD(tree, 0))); 1919 TYPE(CHILD(tree, 0)));
1920 } 1920 }
1921 return (res); 1921 return (res);
1922 } 1922 }
1923 1923
1924 1924
1925 /* global_stmt: 1925 /* global_stmt:
1926 * 1926 *
1927 * 'global' NAME (',' NAME)* 1927 * 'global' PYTOK_NAME (',' PYTOK_NAME)*
1928 */ 1928 */
1929 static int 1929 static int
1930 validate_global_stmt(node *tree) 1930 validate_global_stmt(node *tree)
1931 { 1931 {
1932 int j; 1932 int j;
1933 int nch = NCH(tree); 1933 int nch = NCH(tree);
1934 int res = (validate_ntype(tree, global_stmt) 1934 int res = (validate_ntype(tree, global_stmt)
1935 && is_even(nch) && (nch >= 2)); 1935 && is_even(nch) && (nch >= 2));
1936 1936
1937 if (!res && !PyErr_Occurred()) 1937 if (!res && !PyErr_Occurred())
1938 err_string("illegal global statement"); 1938 err_string("illegal global statement");
1939 1939
1940 if (res) 1940 if (res)
1941 res = (validate_name(CHILD(tree, 0), "global") 1941 res = (validate_name(CHILD(tree, 0), "global")
1942 && validate_ntype(CHILD(tree, 1), NAME)); 1942 && validate_ntype(CHILD(tree, 1), PYTOK_NAME));
1943 for (j = 2; res && (j < nch); j += 2) 1943 for (j = 2; res && (j < nch); j += 2)
1944 res = (validate_comma(CHILD(tree, j)) 1944 res = (validate_comma(CHILD(tree, j))
1945 && validate_ntype(CHILD(tree, j + 1), NAME)); 1945 && validate_ntype(CHILD(tree, j + 1), PYTOK_NAME));
1946 1946
1947 return (res); 1947 return (res);
1948 } 1948 }
1949 1949
1950 /* nonlocal_stmt: 1950 /* nonlocal_stmt:
1951 * 1951 *
1952 * 'nonlocal' NAME (',' NAME)* 1952 * 'nonlocal' PYTOK_NAME (',' PYTOK_NAME)*
1953 */ 1953 */
1954 static int 1954 static int
1955 validate_nonlocal_stmt(node *tree) 1955 validate_nonlocal_stmt(node *tree)
1956 { 1956 {
1957 int j; 1957 int j;
1958 int nch = NCH(tree); 1958 int nch = NCH(tree);
1959 int res = (validate_ntype(tree, nonlocal_stmt) 1959 int res = (validate_ntype(tree, nonlocal_stmt)
1960 && is_even(nch) && (nch >= 2)); 1960 && is_even(nch) && (nch >= 2));
1961 1961
1962 if (!res && !PyErr_Occurred()) 1962 if (!res && !PyErr_Occurred())
1963 err_string("illegal nonlocal statement"); 1963 err_string("illegal nonlocal statement");
1964 1964
1965 if (res) 1965 if (res)
1966 res = (validate_name(CHILD(tree, 0), "nonlocal") 1966 res = (validate_name(CHILD(tree, 0), "nonlocal")
1967 && validate_ntype(CHILD(tree, 1), NAME)); 1967 && validate_ntype(CHILD(tree, 1), PYTOK_NAME));
1968 for (j = 2; res && (j < nch); j += 2) 1968 for (j = 2; res && (j < nch); j += 2)
1969 res = (validate_comma(CHILD(tree, j)) 1969 res = (validate_comma(CHILD(tree, j))
1970 && validate_ntype(CHILD(tree, j + 1), NAME)); 1970 && validate_ntype(CHILD(tree, j + 1), PYTOK_NAME));
1971 1971
1972 return res; 1972 return res;
1973 } 1973 }
1974 1974
1975 /* assert_stmt: 1975 /* assert_stmt:
1976 * 1976 *
1977 * 'assert' test [',' test] 1977 * 'assert' test [',' test]
1978 */ 1978 */
1979 static int 1979 static int
1980 validate_assert_stmt(node *tree) 1980 validate_assert_stmt(node *tree)
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
2059 && validate_suite(CHILD(tree, nch - 1))); 2059 && validate_suite(CHILD(tree, nch - 1)));
2060 else if (!PyErr_Occurred()) { 2060 else if (!PyErr_Occurred()) {
2061 const char* name = "except"; 2061 const char* name = "except";
2062 if (TYPE(CHILD(tree, nch - 3)) != except_clause) 2062 if (TYPE(CHILD(tree, nch - 3)) != except_clause)
2063 name = STR(CHILD(tree, nch - 3)); 2063 name = STR(CHILD(tree, nch - 3));
2064 2064
2065 PyErr_Format(parser_error, 2065 PyErr_Format(parser_error,
2066 "Illegal number of children for try/%s node.", name); 2066 "Illegal number of children for try/%s node.", name);
2067 } 2067 }
2068 /* Handle try/finally statement */ 2068 /* Handle try/finally statement */
2069 if (res && (TYPE(CHILD(tree, pos)) == NAME) && 2069 if (res && (TYPE(CHILD(tree, pos)) == PYTOK_NAME) &&
2070 (strcmp(STR(CHILD(tree, pos)), "finally") == 0)) { 2070 (strcmp(STR(CHILD(tree, pos)), "finally") == 0)) {
2071 res = (validate_numnodes(tree, 6, "try/finally") 2071 res = (validate_numnodes(tree, 6, "try/finally")
2072 && validate_colon(CHILD(tree, 4)) 2072 && validate_colon(CHILD(tree, 4))
2073 && validate_suite(CHILD(tree, 5))); 2073 && validate_suite(CHILD(tree, 5)));
2074 return (res); 2074 return (res);
2075 } 2075 }
2076 /* try/except statement: skip past except_clause sections */ 2076 /* try/except statement: skip past except_clause sections */
2077 while (res && pos < nch && (TYPE(CHILD(tree, pos)) == except_clause)) { 2077 while (res && pos < nch && (TYPE(CHILD(tree, pos)) == except_clause)) {
2078 res = (validate_except_clause(CHILD(tree, pos)) 2078 res = (validate_except_clause(CHILD(tree, pos))
2079 && validate_colon(CHILD(tree, pos + 1)) 2079 && validate_colon(CHILD(tree, pos + 1))
2080 && validate_suite(CHILD(tree, pos + 2))); 2080 && validate_suite(CHILD(tree, pos + 2)));
2081 pos += 3; 2081 pos += 3;
2082 } 2082 }
2083 /* skip else clause */ 2083 /* skip else clause */
2084 if (res && pos < nch && (TYPE(CHILD(tree, pos)) == NAME) && 2084 if (res && pos < nch && (TYPE(CHILD(tree, pos)) == PYTOK_NAME) &&
2085 (strcmp(STR(CHILD(tree, pos)), "else") == 0)) { 2085 (strcmp(STR(CHILD(tree, pos)), "else") == 0)) {
2086 res = (validate_colon(CHILD(tree, pos + 1)) 2086 res = (validate_colon(CHILD(tree, pos + 1))
2087 && validate_suite(CHILD(tree, pos + 2))); 2087 && validate_suite(CHILD(tree, pos + 2)));
2088 pos += 3; 2088 pos += 3;
2089 } 2089 }
2090 if (res && pos < nch) { 2090 if (res && pos < nch) {
2091 /* last clause must be a finally */ 2091 /* last clause must be a finally */
2092 res = (validate_name(CHILD(tree, pos), "finally") 2092 res = (validate_name(CHILD(tree, pos), "finally")
2093 && validate_numnodes(tree, pos + 3, "try/except/finally") 2093 && validate_numnodes(tree, pos + 3, "try/except/finally")
2094 && validate_colon(CHILD(tree, pos + 1)) 2094 && validate_colon(CHILD(tree, pos + 1))
2095 && validate_suite(CHILD(tree, pos + 2))); 2095 && validate_suite(CHILD(tree, pos + 2)));
2096 } 2096 }
2097 return (res); 2097 return (res);
2098 } 2098 }
2099 2099
2100 2100
2101 static int 2101 static int
2102 validate_except_clause(node *tree) 2102 validate_except_clause(node *tree)
2103 { 2103 {
2104 int nch = NCH(tree); 2104 int nch = NCH(tree);
2105 int res = (validate_ntype(tree, except_clause) 2105 int res = (validate_ntype(tree, except_clause)
2106 && ((nch == 1) || (nch == 2) || (nch == 4)) 2106 && ((nch == 1) || (nch == 2) || (nch == 4))
2107 && validate_name(CHILD(tree, 0), "except")); 2107 && validate_name(CHILD(tree, 0), "except"));
2108 2108
2109 if (res && (nch > 1)) 2109 if (res && (nch > 1))
2110 res = validate_test(CHILD(tree, 1)); 2110 res = validate_test(CHILD(tree, 1));
2111 if (res && (nch == 4)) 2111 if (res && (nch == 4))
2112 res = (validate_name(CHILD(tree, 2), "as") 2112 res = (validate_name(CHILD(tree, 2), "as")
2113 && validate_ntype(CHILD(tree, 3), NAME)); 2113 && validate_ntype(CHILD(tree, 3), PYTOK_NAME));
2114 2114
2115 return (res); 2115 return (res);
2116 } 2116 }
2117 2117
2118 2118
2119 static int 2119 static int
2120 validate_test(node *tree) 2120 validate_test(node *tree)
2121 { 2121 {
2122 int nch = NCH(tree); 2122 int nch = NCH(tree);
2123 int res = validate_ntype(tree, test) && is_odd(nch); 2123 int res = validate_ntype(tree, test) && is_odd(nch);
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after
2222 validate_comp_op(node *tree) 2222 validate_comp_op(node *tree)
2223 { 2223 {
2224 int res = 0; 2224 int res = 0;
2225 int nch = NCH(tree); 2225 int nch = NCH(tree);
2226 2226
2227 if (!validate_ntype(tree, comp_op)) 2227 if (!validate_ntype(tree, comp_op))
2228 return (0); 2228 return (0);
2229 if (nch == 1) { 2229 if (nch == 1) {
2230 /* 2230 /*
2231 * Only child will be a terminal with a well-defined symbolic name 2231 * Only child will be a terminal with a well-defined symbolic name
2232 * or a NAME with a string of either 'is' or 'in' 2232 * or a PYTOK_NAME with a string of either 'is' or 'in'
2233 */ 2233 */
2234 tree = CHILD(tree, 0); 2234 tree = CHILD(tree, 0);
2235 switch (TYPE(tree)) { 2235 switch (TYPE(tree)) {
2236 case LESS: 2236 case PYTOK_LESS:
2237 case GREATER: 2237 case PYTOK_GREATER:
2238 case EQEQUAL: 2238 case PYTOK_EQEQUAL:
2239 case EQUAL: 2239 case PYTOK_EQUAL:
2240 case LESSEQUAL: 2240 case PYTOK_LESSEQUAL:
2241 case GREATEREQUAL: 2241 case PYTOK_GREATEREQUAL:
2242 case NOTEQUAL: 2242 case PYTOK_NOTEQUAL:
2243 res = 1; 2243 res = 1;
2244 break; 2244 break;
2245 case NAME: 2245 case PYTOK_NAME:
2246 res = ((strcmp(STR(tree), "in") == 0) 2246 res = ((strcmp(STR(tree), "in") == 0)
2247 || (strcmp(STR(tree), "is") == 0)); 2247 || (strcmp(STR(tree), "is") == 0));
2248 if (!res) { 2248 if (!res) {
2249 PyErr_Format(parser_error, 2249 PyErr_Format(parser_error,
2250 "illegal operator '%s'", STR(tree)); 2250 "illegal operator '%s'", STR(tree));
2251 } 2251 }
2252 break; 2252 break;
2253 default: 2253 default:
2254 err_string("illegal comparison operator type"); 2254 err_string("illegal comparison operator type");
2255 break; 2255 break;
2256 } 2256 }
2257 } 2257 }
2258 else if ((res = validate_numnodes(tree, 2, "comp_op")) != 0) { 2258 else if ((res = validate_numnodes(tree, 2, "comp_op")) != 0) {
2259 res = (validate_ntype(CHILD(tree, 0), NAME) 2259 res = (validate_ntype(CHILD(tree, 0), PYTOK_NAME)
2260 && validate_ntype(CHILD(tree, 1), NAME) 2260 && validate_ntype(CHILD(tree, 1), PYTOK_NAME)
2261 && (((strcmp(STR(CHILD(tree, 0)), "is") == 0) 2261 && (((strcmp(STR(CHILD(tree, 0)), "is") == 0)
2262 && (strcmp(STR(CHILD(tree, 1)), "not") == 0)) 2262 && (strcmp(STR(CHILD(tree, 1)), "not") == 0))
2263 || ((strcmp(STR(CHILD(tree, 0)), "not") == 0) 2263 || ((strcmp(STR(CHILD(tree, 0)), "not") == 0)
2264 && (strcmp(STR(CHILD(tree, 1)), "in") == 0)))); 2264 && (strcmp(STR(CHILD(tree, 1)), "in") == 0))));
2265 if (!res && !PyErr_Occurred()) 2265 if (!res && !PyErr_Occurred())
2266 err_string("unknown comparison operator"); 2266 err_string("unknown comparison operator");
2267 } 2267 }
2268 return (res); 2268 return (res);
2269 } 2269 }
2270 2270
2271 2271
2272 static int 2272 static int
2273 validate_star_expr(node *tree) 2273 validate_star_expr(node *tree)
2274 { 2274 {
2275 int res = validate_ntype(tree, star_expr); 2275 int res = validate_ntype(tree, star_expr);
2276 if (!res) return res; 2276 if (!res) return res;
2277 if (!validate_numnodes(tree, 2, "star_expr")) 2277 if (!validate_numnodes(tree, 2, "star_expr"))
2278 return 0; 2278 return 0;
2279 return validate_ntype(CHILD(tree, 0), STAR) && \ 2279 return validate_ntype(CHILD(tree, 0), PYTOK_STAR) && \
2280 validate_expr(CHILD(tree, 1)); 2280 validate_expr(CHILD(tree, 1));
2281 } 2281 }
2282 2282
2283 2283
2284 static int 2284 static int
2285 validate_expr(node *tree) 2285 validate_expr(node *tree)
2286 { 2286 {
2287 int j; 2287 int j;
2288 int nch = NCH(tree); 2288 int nch = NCH(tree);
2289 int res = (validate_ntype(tree, expr) 2289 int res = (validate_ntype(tree, expr)
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after
2348 } 2348 }
2349 return (res); 2349 return (res);
2350 } 2350 }
2351 2351
2352 2352
2353 static int 2353 static int
2354 validate_shift_expr(node *tree) 2354 validate_shift_expr(node *tree)
2355 { 2355 {
2356 return (validate_ntype(tree, shift_expr) 2356 return (validate_ntype(tree, shift_expr)
2357 && validate_chain_two_ops(tree, validate_arith_expr, 2357 && validate_chain_two_ops(tree, validate_arith_expr,
2358 LEFTSHIFT, RIGHTSHIFT)); 2358 PYTOK_LEFTSHIFT, PYTOK_RIGHTSHIFT));
2359 } 2359 }
2360 2360
2361 2361
2362 static int 2362 static int
2363 validate_arith_expr(node *tree) 2363 validate_arith_expr(node *tree)
2364 { 2364 {
2365 return (validate_ntype(tree, arith_expr) 2365 return (validate_ntype(tree, arith_expr)
2366 && validate_chain_two_ops(tree, validate_term, PLUS, MINUS)); 2366 && validate_chain_two_ops(tree, validate_term, PYTOK_PLUS, PYTOK_MIN US));
2367 } 2367 }
2368 2368
2369 2369
2370 static int 2370 static int
2371 validate_term(node *tree) 2371 validate_term(node *tree)
2372 { 2372 {
2373 int pos = 1; 2373 int pos = 1;
2374 int nch = NCH(tree); 2374 int nch = NCH(tree);
2375 int res = (validate_ntype(tree, term) 2375 int res = (validate_ntype(tree, term)
2376 && is_odd(nch) 2376 && is_odd(nch)
2377 && validate_factor(CHILD(tree, 0))); 2377 && validate_factor(CHILD(tree, 0)));
2378 2378
2379 for ( ; res && (pos < nch); pos += 2) 2379 for ( ; res && (pos < nch); pos += 2)
2380 res = (((TYPE(CHILD(tree, pos)) == STAR) 2380 res = (((TYPE(CHILD(tree, pos)) == PYTOK_STAR)
2381 || (TYPE(CHILD(tree, pos)) == SLASH) 2381 || (TYPE(CHILD(tree, pos)) == PYTOK_SLASH)
2382 || (TYPE(CHILD(tree, pos)) == DOUBLESLASH) 2382 || (TYPE(CHILD(tree, pos)) == PYTOK_DOUBLESLASH)
2383 || (TYPE(CHILD(tree, pos)) == PERCENT)) 2383 || (TYPE(CHILD(tree, pos)) == PYTOK_PERCENT))
2384 && validate_factor(CHILD(tree, pos + 1))); 2384 && validate_factor(CHILD(tree, pos + 1)));
2385 2385
2386 return (res); 2386 return (res);
2387 } 2387 }
2388 2388
2389 2389
2390 /* factor: 2390 /* factor:
2391 * 2391 *
2392 * factor: ('+'|'-'|'~') factor | power 2392 * factor: ('+'|'-'|'~') factor | power
2393 */ 2393 */
2394 static int 2394 static int
2395 validate_factor(node *tree) 2395 validate_factor(node *tree)
2396 { 2396 {
2397 int nch = NCH(tree); 2397 int nch = NCH(tree);
2398 int res = (validate_ntype(tree, factor) 2398 int res = (validate_ntype(tree, factor)
2399 && (((nch == 2) 2399 && (((nch == 2)
2400 && ((TYPE(CHILD(tree, 0)) == PLUS) 2400 && ((TYPE(CHILD(tree, 0)) == PYTOK_PLUS)
2401 || (TYPE(CHILD(tree, 0)) == MINUS) 2401 || (TYPE(CHILD(tree, 0)) == PYTOK_MINUS)
2402 || (TYPE(CHILD(tree, 0)) == TILDE)) 2402 || (TYPE(CHILD(tree, 0)) == PYTOK_TILDE))
2403 && validate_factor(CHILD(tree, 1))) 2403 && validate_factor(CHILD(tree, 1)))
2404 || ((nch == 1) 2404 || ((nch == 1)
2405 && validate_power(CHILD(tree, 0))))); 2405 && validate_power(CHILD(tree, 0)))));
2406 return (res); 2406 return (res);
2407 } 2407 }
2408 2408
2409 2409
2410 /* power: 2410 /* power:
2411 * 2411 *
2412 * power: atom trailer* ('**' factor)* 2412 * power: atom trailer* ('**' factor)*
(...skipping 25 matching lines...) Expand all
2438 validate_atom(node *tree) 2438 validate_atom(node *tree)
2439 { 2439 {
2440 int pos; 2440 int pos;
2441 int nch = NCH(tree); 2441 int nch = NCH(tree);
2442 int res = validate_ntype(tree, atom); 2442 int res = validate_ntype(tree, atom);
2443 2443
2444 if (res && nch < 1) 2444 if (res && nch < 1)
2445 res = validate_numnodes(tree, nch+1, "atom"); 2445 res = validate_numnodes(tree, nch+1, "atom");
2446 if (res) { 2446 if (res) {
2447 switch (TYPE(CHILD(tree, 0))) { 2447 switch (TYPE(CHILD(tree, 0))) {
2448 case LPAR: 2448 case PYTOK_LPAR:
2449 res = ((nch <= 3) 2449 res = ((nch <= 3)
2450 && (validate_rparen(CHILD(tree, nch - 1)))); 2450 && (validate_rparen(CHILD(tree, nch - 1))));
2451 2451
2452 if (res && (nch == 3)) { 2452 if (res && (nch == 3)) {
2453 if (TYPE(CHILD(tree, 1))==yield_expr) 2453 if (TYPE(CHILD(tree, 1))==yield_expr)
2454 res = validate_yield_expr(CHILD(tree, 1)); 2454 res = validate_yield_expr(CHILD(tree, 1));
2455 else 2455 else
2456 res = validate_testlist_comp(CHILD(tree, 1)); 2456 res = validate_testlist_comp(CHILD(tree, 1));
2457 } 2457 }
2458 break; 2458 break;
2459 case LSQB: 2459 case PYTOK_LSQB:
2460 if (nch == 2) 2460 if (nch == 2)
2461 res = validate_ntype(CHILD(tree, 1), RSQB); 2461 res = validate_ntype(CHILD(tree, 1), PYTOK_RSQB);
2462 else if (nch == 3) 2462 else if (nch == 3)
2463 res = (validate_testlist_comp(CHILD(tree, 1)) 2463 res = (validate_testlist_comp(CHILD(tree, 1))
2464 && validate_ntype(CHILD(tree, 2), RSQB)); 2464 && validate_ntype(CHILD(tree, 2), PYTOK_RSQB));
2465 else { 2465 else {
2466 res = 0; 2466 res = 0;
2467 err_string("illegal list display atom"); 2467 err_string("illegal list display atom");
2468 } 2468 }
2469 break; 2469 break;
2470 case LBRACE: 2470 case PYTOK_LBRACE:
2471 res = ((nch <= 3) 2471 res = ((nch <= 3)
2472 && validate_ntype(CHILD(tree, nch - 1), RBRACE)); 2472 && validate_ntype(CHILD(tree, nch - 1), PYTOK_RBRACE));
2473 2473
2474 if (res && (nch == 3)) 2474 if (res && (nch == 3))
2475 res = validate_dictorsetmaker(CHILD(tree, 1)); 2475 res = validate_dictorsetmaker(CHILD(tree, 1));
2476 break; 2476 break;
2477 case NAME: 2477 case PYTOK_NAME:
2478 case NUMBER: 2478 case PYTOK_NUMBER:
2479 case ELLIPSIS: 2479 case PYTOK_ELLIPSIS:
2480 res = (nch == 1); 2480 res = (nch == 1);
2481 break; 2481 break;
2482 case STRING: 2482 case PYTOK_STRING:
2483 for (pos = 1; res && (pos < nch); ++pos) 2483 for (pos = 1; res && (pos < nch); ++pos)
2484 res = validate_ntype(CHILD(tree, pos), STRING); 2484 res = validate_ntype(CHILD(tree, pos), PYTOK_STRING);
2485 break; 2485 break;
2486 default: 2486 default:
2487 res = 0; 2487 res = 0;
2488 break; 2488 break;
2489 } 2489 }
2490 } 2490 }
2491 return (res); 2491 return (res);
2492 } 2492 }
2493 2493
2494 2494
(...skipping 29 matching lines...) Expand all
2524 ok = validate_comma(CHILD(tree, i)); 2524 ok = validate_comma(CHILD(tree, i));
2525 else if (i != nch) { 2525 else if (i != nch) {
2526 ok = 0; 2526 ok = 0;
2527 err_string("illegal trailing nodes for testlist_comp"); 2527 err_string("illegal trailing nodes for testlist_comp");
2528 } 2528 }
2529 } 2529 }
2530 return ok; 2530 return ok;
2531 } 2531 }
2532 2532
2533 /* decorator: 2533 /* decorator:
2534 * '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 2534 * '@' dotted_name [ '(' [arglist] ')' ] PYTOK_NEWLINE
2535 */ 2535 */
2536 static int 2536 static int
2537 validate_decorator(node *tree) 2537 validate_decorator(node *tree)
2538 { 2538 {
2539 int ok; 2539 int ok;
2540 int nch = NCH(tree); 2540 int nch = NCH(tree);
2541 ok = (validate_ntype(tree, decorator) && 2541 ok = (validate_ntype(tree, decorator) &&
2542 (nch == 3 || nch == 5 || nch == 6) && 2542 (nch == 3 || nch == 5 || nch == 6) &&
2543 validate_at(CHILD(tree, 0)) && 2543 validate_at(CHILD(tree, 0)) &&
2544 validate_dotted_name(CHILD(tree, 1)) && 2544 validate_dotted_name(CHILD(tree, 1)) &&
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
2599 int ok = (validate_ntype(tree, with_stmt) 2599 int ok = (validate_ntype(tree, with_stmt)
2600 && (nch % 2 == 0) 2600 && (nch % 2 == 0)
2601 && validate_name(CHILD(tree, 0), "with") 2601 && validate_name(CHILD(tree, 0), "with")
2602 && validate_colon(RCHILD(tree, -2)) 2602 && validate_colon(RCHILD(tree, -2))
2603 && validate_suite(RCHILD(tree, -1))); 2603 && validate_suite(RCHILD(tree, -1)));
2604 for (i = 1; ok && i < nch - 2; i += 2) 2604 for (i = 1; ok && i < nch - 2; i += 2)
2605 ok = validate_with_item(CHILD(tree, i)); 2605 ok = validate_with_item(CHILD(tree, i));
2606 return ok; 2606 return ok;
2607 } 2607 }
2608 2608
2609 /* funcdef: 'def' NAME parameters ['->' test] ':' suite */ 2609 /* funcdef: 'def' PYTOK_NAME parameters ['->' test] ':' suite */
2610 2610
2611 static int 2611 static int
2612 validate_funcdef(node *tree) 2612 validate_funcdef(node *tree)
2613 { 2613 {
2614 int nch = NCH(tree); 2614 int nch = NCH(tree);
2615 int res = validate_ntype(tree, funcdef); 2615 int res = validate_ntype(tree, funcdef);
2616 if (res) { 2616 if (res) {
2617 if (nch == 5) { 2617 if (nch == 5) {
2618 res = (validate_name(CHILD(tree, 0), "def") 2618 res = (validate_name(CHILD(tree, 0), "def")
2619 && validate_ntype(CHILD(tree, 1), NAME) 2619 && validate_ntype(CHILD(tree, 1), PYTOK_NAME)
2620 && validate_parameters(CHILD(tree, 2)) 2620 && validate_parameters(CHILD(tree, 2))
2621 && validate_colon(CHILD(tree, 3)) 2621 && validate_colon(CHILD(tree, 3))
2622 && validate_suite(CHILD(tree, 4))); 2622 && validate_suite(CHILD(tree, 4)));
2623 } 2623 }
2624 else if (nch == 7) { 2624 else if (nch == 7) {
2625 res = (validate_name(CHILD(tree, 0), "def") 2625 res = (validate_name(CHILD(tree, 0), "def")
2626 && validate_ntype(CHILD(tree, 1), NAME) 2626 && validate_ntype(CHILD(tree, 1), PYTOK_NAME)
2627 && validate_parameters(CHILD(tree, 2)) 2627 && validate_parameters(CHILD(tree, 2))
2628 && validate_rarrow(CHILD(tree, 3)) 2628 && validate_rarrow(CHILD(tree, 3))
2629 && validate_test(CHILD(tree, 4)) 2629 && validate_test(CHILD(tree, 4))
2630 && validate_colon(CHILD(tree, 5)) 2630 && validate_colon(CHILD(tree, 5))
2631 && validate_suite(CHILD(tree, 6))); 2631 && validate_suite(CHILD(tree, 6)));
2632 } 2632 }
2633 else { 2633 else {
2634 res = 0; 2634 res = 0;
2635 err_string("illegal number of children for funcdef"); 2635 err_string("illegal number of children for funcdef");
2636 } 2636 }
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after
2738 int sym = TYPE(CHILD(tree, i)); 2738 int sym = TYPE(CHILD(tree, i));
2739 2739
2740 if (sym == argument) { 2740 if (sym == argument) {
2741 ok = validate_argument(CHILD(tree, i)); 2741 ok = validate_argument(CHILD(tree, i));
2742 if (ok && i+1 != nch) { 2742 if (ok && i+1 != nch) {
2743 err_string("illegal arglist specification" 2743 err_string("illegal arglist specification"
2744 " (extra stuff on end)"); 2744 " (extra stuff on end)");
2745 ok = 0; 2745 ok = 0;
2746 } 2746 }
2747 } 2747 }
2748 else if (sym == STAR) { 2748 else if (sym == PYTOK_STAR) {
2749 ok = validate_star(CHILD(tree, i)); 2749 ok = validate_star(CHILD(tree, i));
2750 if (ok && (nch-i == 2)) 2750 if (ok && (nch-i == 2))
2751 ok = validate_test(CHILD(tree, i+1)); 2751 ok = validate_test(CHILD(tree, i+1));
2752 else if (ok && (nch-i == 5)) 2752 else if (ok && (nch-i == 5))
2753 ok = (validate_test(CHILD(tree, i+1)) 2753 ok = (validate_test(CHILD(tree, i+1))
2754 && validate_comma(CHILD(tree, i+2)) 2754 && validate_comma(CHILD(tree, i+2))
2755 && validate_doublestar(CHILD(tree, i+3)) 2755 && validate_doublestar(CHILD(tree, i+3))
2756 && validate_test(CHILD(tree, i+4))); 2756 && validate_test(CHILD(tree, i+4)));
2757 else { 2757 else {
2758 err_string("illegal use of '*' in arglist"); 2758 err_string("illegal use of '*' in arglist");
2759 ok = 0; 2759 ok = 0;
2760 } 2760 }
2761 } 2761 }
2762 else if (sym == DOUBLESTAR) { 2762 else if (sym == PYTOK_DOUBLESTAR) {
2763 if (nch-i == 2) 2763 if (nch-i == 2)
2764 ok = (validate_doublestar(CHILD(tree, i)) 2764 ok = (validate_doublestar(CHILD(tree, i))
2765 && validate_test(CHILD(tree, i+1))); 2765 && validate_test(CHILD(tree, i+1)));
2766 else { 2766 else {
2767 err_string("illegal use of '**' in arglist"); 2767 err_string("illegal use of '**' in arglist");
2768 ok = 0; 2768 ok = 0;
2769 } 2769 }
2770 } 2770 }
2771 else { 2771 else {
2772 err_string("illegal arglist specification"); 2772 err_string("illegal arglist specification");
(...skipping 23 matching lines...) Expand all
2796 res = (validate_equal(CHILD(tree, 1)) 2796 res = (validate_equal(CHILD(tree, 1))
2797 && validate_test(CHILD(tree, 2))); 2797 && validate_test(CHILD(tree, 2)));
2798 2798
2799 return (res); 2799 return (res);
2800 } 2800 }
2801 2801
2802 2802
2803 2803
2804 /* trailer: 2804 /* trailer:
2805 * 2805 *
2806 * '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 2806 * '(' [arglist] ')' | '[' subscriptlist ']' | '.' PYTOK_NAME
2807 */ 2807 */
2808 static int 2808 static int
2809 validate_trailer(node *tree) 2809 validate_trailer(node *tree)
2810 { 2810 {
2811 int nch = NCH(tree); 2811 int nch = NCH(tree);
2812 int res = validate_ntype(tree, trailer) && ((nch == 2) || (nch == 3)); 2812 int res = validate_ntype(tree, trailer) && ((nch == 2) || (nch == 3));
2813 2813
2814 if (res) { 2814 if (res) {
2815 switch (TYPE(CHILD(tree, 0))) { 2815 switch (TYPE(CHILD(tree, 0))) {
2816 case LPAR: 2816 case PYTOK_LPAR:
2817 res = validate_rparen(CHILD(tree, nch - 1)); 2817 res = validate_rparen(CHILD(tree, nch - 1));
2818 if (res && (nch == 3)) 2818 if (res && (nch == 3))
2819 res = validate_arglist(CHILD(tree, 1)); 2819 res = validate_arglist(CHILD(tree, 1));
2820 break; 2820 break;
2821 case LSQB: 2821 case PYTOK_LSQB:
2822 res = (validate_numnodes(tree, 3, "trailer") 2822 res = (validate_numnodes(tree, 3, "trailer")
2823 && validate_subscriptlist(CHILD(tree, 1)) 2823 && validate_subscriptlist(CHILD(tree, 1))
2824 && validate_ntype(CHILD(tree, 2), RSQB)); 2824 && validate_ntype(CHILD(tree, 2), PYTOK_RSQB));
2825 break; 2825 break;
2826 case DOT: 2826 case PYTOK_DOT:
2827 res = (validate_numnodes(tree, 2, "trailer") 2827 res = (validate_numnodes(tree, 2, "trailer")
2828 && validate_ntype(CHILD(tree, 1), NAME)); 2828 && validate_ntype(CHILD(tree, 1), PYTOK_NAME));
2829 break; 2829 break;
2830 default: 2830 default:
2831 res = 0; 2831 res = 0;
2832 break; 2832 break;
2833 } 2833 }
2834 } 2834 }
2835 else { 2835 else {
2836 (void) validate_numnodes(tree, 2, "trailer"); 2836 (void) validate_numnodes(tree, 2, "trailer");
2837 } 2837 }
2838 return (res); 2838 return (res);
(...skipping 21 matching lines...) Expand all
2860 { 2860 {
2861 int offset = 0; 2861 int offset = 0;
2862 int nch = NCH(tree); 2862 int nch = NCH(tree);
2863 int res = validate_ntype(tree, subscript) && (nch >= 1) && (nch <= 4); 2863 int res = validate_ntype(tree, subscript) && (nch >= 1) && (nch <= 4);
2864 2864
2865 if (!res) { 2865 if (!res) {
2866 if (!PyErr_Occurred()) 2866 if (!PyErr_Occurred())
2867 err_string("invalid number of arguments for subscript node"); 2867 err_string("invalid number of arguments for subscript node");
2868 return (0); 2868 return (0);
2869 } 2869 }
2870 if (TYPE(CHILD(tree, 0)) == DOT) 2870 if (TYPE(CHILD(tree, 0)) == PYTOK_DOT)
2871 /* take care of ('.' '.' '.') possibility */ 2871 /* take care of ('.' '.' '.') possibility */
2872 return (validate_numnodes(tree, 3, "subscript") 2872 return (validate_numnodes(tree, 3, "subscript")
2873 && validate_dot(CHILD(tree, 0)) 2873 && validate_dot(CHILD(tree, 0))
2874 && validate_dot(CHILD(tree, 1)) 2874 && validate_dot(CHILD(tree, 1))
2875 && validate_dot(CHILD(tree, 2))); 2875 && validate_dot(CHILD(tree, 2)));
2876 if (nch == 1) { 2876 if (nch == 1) {
2877 if (TYPE(CHILD(tree, 0)) == test) 2877 if (TYPE(CHILD(tree, 0)) == test)
2878 res = validate_test(CHILD(tree, 0)); 2878 res = validate_test(CHILD(tree, 0));
2879 else 2879 else
2880 res = validate_colon(CHILD(tree, 0)); 2880 res = validate_colon(CHILD(tree, 0));
2881 return (res); 2881 return (res);
2882 } 2882 }
2883 /* Must be [test] ':' [test] [sliceop], 2883 /* Must be [test] ':' [test] [sliceop],
2884 * but at least one of the optional components will 2884 * but at least one of the optional components will
2885 * be present, but we don't know which yet. 2885 * be present, but we don't know which yet.
2886 */ 2886 */
2887 if ((TYPE(CHILD(tree, 0)) != COLON) || (nch == 4)) { 2887 if ((TYPE(CHILD(tree, 0)) != PYTOK_COLON) || (nch == 4)) {
2888 res = validate_test(CHILD(tree, 0)); 2888 res = validate_test(CHILD(tree, 0));
2889 offset = 1; 2889 offset = 1;
2890 } 2890 }
2891 if (res) 2891 if (res)
2892 res = validate_colon(CHILD(tree, offset)); 2892 res = validate_colon(CHILD(tree, offset));
2893 if (res) { 2893 if (res) {
2894 int rem = nch - ++offset; 2894 int rem = nch - ++offset;
2895 if (rem) { 2895 if (rem) {
2896 if (TYPE(CHILD(tree, offset)) == test) { 2896 if (TYPE(CHILD(tree, offset)) == test) {
2897 res = validate_test(CHILD(tree, offset)); 2897 res = validate_test(CHILD(tree, offset));
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
2967 2967
2968 if (nch - i < 1) { 2968 if (nch - i < 1) {
2969 (void) validate_numnodes(tree, 1, "dictorsetmaker"); 2969 (void) validate_numnodes(tree, 1, "dictorsetmaker");
2970 return 0; 2970 return 0;
2971 } 2971 }
2972 2972
2973 res = validate_test(CHILD(tree, i++)); 2973 res = validate_test(CHILD(tree, i++));
2974 if (!res) 2974 if (!res)
2975 return 0; 2975 return 0;
2976 2976
2977 if (nch - i >= 2 && TYPE(CHILD(tree, i)) == COLON) { 2977 if (nch - i >= 2 && TYPE(CHILD(tree, i)) == PYTOK_COLON) {
2978 /* Dictionary display or dictionary comprehension. */ 2978 /* Dictionary display or dictionary comprehension. */
2979 res = (validate_colon(CHILD(tree, i++)) 2979 res = (validate_colon(CHILD(tree, i++))
2980 && validate_test(CHILD(tree, i++))); 2980 && validate_test(CHILD(tree, i++)));
2981 if (!res) 2981 if (!res)
2982 return 0; 2982 return 0;
2983 2983
2984 if (nch - i >= 1 && TYPE(CHILD(tree, i)) == comp_for) { 2984 if (nch - i >= 1 && TYPE(CHILD(tree, i)) == comp_for) {
2985 /* Dictionary comprehension. */ 2985 /* Dictionary comprehension. */
2986 res = validate_comp_for(CHILD(tree, i++)); 2986 res = validate_comp_for(CHILD(tree, i++));
2987 if (!res) 2987 if (!res)
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
3038 3038
3039 3039
3040 static int 3040 static int
3041 validate_eval_input(node *tree) 3041 validate_eval_input(node *tree)
3042 { 3042 {
3043 int pos; 3043 int pos;
3044 int nch = NCH(tree); 3044 int nch = NCH(tree);
3045 int res = (validate_ntype(tree, eval_input) 3045 int res = (validate_ntype(tree, eval_input)
3046 && (nch >= 2) 3046 && (nch >= 2)
3047 && validate_testlist(CHILD(tree, 0)) 3047 && validate_testlist(CHILD(tree, 0))
3048 && validate_ntype(CHILD(tree, nch - 1), ENDMARKER)); 3048 && validate_ntype(CHILD(tree, nch - 1), PYTOK_ENDMARKER));
3049 3049
3050 for (pos = 1; res && (pos < (nch - 1)); ++pos) 3050 for (pos = 1; res && (pos < (nch - 1)); ++pos)
3051 res = validate_ntype(CHILD(tree, pos), NEWLINE); 3051 res = validate_ntype(CHILD(tree, pos), PYTOK_NEWLINE);
3052 3052
3053 return (res); 3053 return (res);
3054 } 3054 }
3055 3055
3056 3056
3057 static int 3057 static int
3058 validate_node(node *tree) 3058 validate_node(node *tree)
3059 { 3059 {
3060 int nch = 0; /* num. children on current node */ 3060 int nch = 0; /* num. children on current node */
3061 int res = 1; /* result value */ 3061 int res = 1; /* result value */
(...skipping 189 matching lines...) Expand 10 before | Expand all | Expand 10 after
3251 int res = validate_eval_input(tree); 3251 int res = validate_eval_input(tree);
3252 3252
3253 if (!res && !PyErr_Occurred()) 3253 if (!res && !PyErr_Occurred())
3254 err_string("could not validate expression tuple"); 3254 err_string("could not validate expression tuple");
3255 3255
3256 return (res); 3256 return (res);
3257 } 3257 }
3258 3258
3259 3259
3260 /* file_input: 3260 /* file_input:
3261 * (NEWLINE | stmt)* ENDMARKER 3261 * (PYTOK_NEWLINE | stmt)* PYTOK_ENDMARKER
3262 */ 3262 */
3263 static int 3263 static int
3264 validate_file_input(node *tree) 3264 validate_file_input(node *tree)
3265 { 3265 {
3266 int j; 3266 int j;
3267 int nch = NCH(tree) - 1; 3267 int nch = NCH(tree) - 1;
3268 int res = ((nch >= 0) 3268 int res = ((nch >= 0)
3269 && validate_ntype(CHILD(tree, nch), ENDMARKER)); 3269 && validate_ntype(CHILD(tree, nch), PYTOK_ENDMARKER));
3270 3270
3271 for (j = 0; res && (j < nch); ++j) { 3271 for (j = 0; res && (j < nch); ++j) {
3272 if (TYPE(CHILD(tree, j)) == stmt) 3272 if (TYPE(CHILD(tree, j)) == stmt)
3273 res = validate_stmt(CHILD(tree, j)); 3273 res = validate_stmt(CHILD(tree, j));
3274 else 3274 else
3275 res = validate_newline(CHILD(tree, j)); 3275 res = validate_newline(CHILD(tree, j));
3276 } 3276 }
3277 /* This stays in to prevent any internal failures from getting to the 3277 /* This stays in to prevent any internal failures from getting to the
3278 * user. Hopefully, this won't be needed. If a user reports getting 3278 * user. Hopefully, this won't be needed. If a user reports getting
3279 * this, we have some debugging to do. 3279 * this, we have some debugging to do.
(...skipping 160 matching lines...) Expand 10 before | Expand all | Expand 10 after
3440 pickle_constructor, NULL); 3440 pickle_constructor, NULL);
3441 Py_XDECREF(res); 3441 Py_XDECREF(res);
3442 } 3442 }
3443 Py_XDECREF(func); 3443 Py_XDECREF(func);
3444 Py_XDECREF(pickle_constructor); 3444 Py_XDECREF(pickle_constructor);
3445 Py_XDECREF(pickler); 3445 Py_XDECREF(pickler);
3446 Py_DECREF(copyreg); 3446 Py_DECREF(copyreg);
3447 } 3447 }
3448 return module; 3448 return module;
3449 } 3449 }
OLDNEW
« no previous file with comments | « Makefile.pre.in ('k') | Parser/acceler.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+