Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(4)

Unified Diff: Parser/parser.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « Parser/listnode.c ('k') | Parser/parsetok.c » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -139,12 +139,12 @@ classify(parser_state *ps, int type, con
grammar *g = ps->p_grammar;
int n = g->g_ll.ll_nlabels;
- if (type == NAME) {
+ if (type == PYTOK_NAME) {
const char *s = str;
label *l = g->g_ll.ll_label;
int i;
for (i = n; i > 0; i--, l++) {
- if (l->lb_type != NAME || l->lb_str == NULL ||
+ if (l->lb_type != PYTOK_NAME || l->lb_str == NULL ||
l->lb_str[0] != s[0] ||
strcmp(l->lb_str, s) != 0)
continue;
@@ -202,14 +202,14 @@ future_hack(parser_state *ps)
return;
ch = CHILD(n, 3);
/* ch can be a star, a parenthesis or import_as_names */
- if (TYPE(ch) == STAR)
+ if (TYPE(ch) == PYTOK_STAR)
return;
- if (TYPE(ch) == LPAR)
+ if (TYPE(ch) == PYTOK_LPAR)
ch = CHILD(n, 4);
for (i = 0; i < NCH(ch); i += 2) {
cch = CHILD(ch, i);
- if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) {
+ if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == PYTOK_NAME) {
char *str_ch = STR(CHILD(cch, 0));
if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) {
ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
@@ -253,7 +253,7 @@ PyParser_AddToken(parser_state *ps, int
if (x != -1) {
if (x & (1<<7)) {
/* Push non-terminal */
- int nt = (x >> 8) + NT_OFFSET;
+ int nt = (x >> 8) + PYTOK_NT_OFFSET;
int arrow = x & ((1<<7)-1);
dfa *d1 = PyGrammar_FindDFA(
ps->p_grammar, nt);
@@ -350,7 +350,7 @@ dumptree(grammar *g, node *n)
l.lb_type = TYPE(n);
l.lb_str = STR(n);
printf("%s", PyGrammar_LabelRepr(&l));
- if (ISNONTERMINAL(TYPE(n))) {
+ if (PYTOK_ISNONTERMINAL(TYPE(n))) {
printf("(");
for (i = 0; i < NCH(n); i++) {
if (i > 0)
@@ -369,13 +369,13 @@ showtree(grammar *g, node *n)
if (n == NULL)
return;
- if (ISNONTERMINAL(TYPE(n))) {
+ if (PYTOK_ISNONTERMINAL(TYPE(n))) {
for (i = 0; i < NCH(n); i++)
showtree(g, CHILD(n, i));
}
- else if (ISTERMINAL(TYPE(n))) {
+ else if (PYTOK_ISTERMINAL(TYPE(n))) {
printf("%s", _PyParser_TokenNames[TYPE(n)]);
- if (TYPE(n) == NUMBER || TYPE(n) == NAME)
+ if (TYPE(n) == PYTOK_NUMBER || TYPE(n) == PYTOK_NAME)
printf("(%s)", STR(n));
printf(" ");
}
@@ -431,18 +431,18 @@ symbol.
As an example, consider this grammar:
-expr: term (OP term)*
+expr: term (PYTOK_OP term)*
term: CONSTANT | '(' expr ')'
The DFA corresponding to the rule for expr is:
-------->.---term-->.------->
- ^ |
- | |
- \----OP----/
+------->.---------term-->.------->
+ ^ |
+ | |
+ \----PYTOK_OP----/
The parse tree generated for the input a+b is:
-(expr: (term: (NAME: a)), (OP: +), (term: (NAME: b)))
+(expr: (term: (PYTOK_NAME: a)), (PYTOK_OP: +), (term: (PYTOK_NAME: b)))
*/
« no previous file with comments | « Parser/listnode.c ('k') | Parser/parsetok.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+