Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(4)

Unified Diff: Parser/pgen.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « Parser/parsetok.c ('k') | Parser/tokenizer.c » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
--- a/Parser/pgen.c
+++ b/Parser/pgen.c
@@ -79,7 +79,7 @@ static nfa *
newnfa(char *name)
{
nfa *nf;
- static int type = NT_OFFSET; /* All types will be disjunct */
+ static int type = PYTOK_NT_OFFSET; /* All types will be disjunct */
nf = (nfa *)PyObject_MALLOC(sizeof(nfa));
if (nf == NULL)
@@ -113,7 +113,7 @@ newnfagrammar(void)
gr->gr_nfa = NULL;
gr->gr_ll.ll_nlabels = 0;
gr->gr_ll.ll_label = NULL;
- addlabel(&gr->gr_ll, ENDMARKER, "EMPTY");
+ addlabel(&gr->gr_ll, PYTOK_ENDMARKER, "EMPTY");
return gr;
}
@@ -128,7 +128,7 @@ addnfa(nfagrammar *gr, char *name)
if (gr->gr_nfa == NULL)
Py_FatalError("out of mem");
gr->gr_nfa[gr->gr_nnfas++] = nf;
- addlabel(&gr->gr_ll, NAME, nf->nf_name);
+ addlabel(&gr->gr_ll, PYTOK_NAME, nf->nf_name);
return nf;
}
@@ -157,10 +157,10 @@ metacompile(node *n)
printf("Compiling (meta-) parse tree into NFA grammar\n");
gr = newnfagrammar();
REQ(n, MSTART);
- i = n->n_nchildren - 1; /* Last child is ENDMARKER */
+ i = n->n_nchildren - 1; /* Last child is PYTOK_ENDMARKER */
n = n->n_child;
for (; --i >= 0; n++) {
- if (n->n_type != NEWLINE)
+ if (n->n_type != PYTOK_NEWLINE)
compile_rule(gr, n);
}
return gr;
@@ -174,15 +174,15 @@ compile_rule(nfagrammar *gr, node *n)
REQ(n, RULE);
REQN(n->n_nchildren, 4);
n = n->n_child;
- REQ(n, NAME);
+ REQ(n, PYTOK_NAME);
nf = addnfa(gr, n->n_str);
n++;
- REQ(n, COLON);
+ REQ(n, PYTOK_COLON);
n++;
REQ(n, RHS);
compile_rhs(&gr->gr_ll, nf, n, &nf->nf_start, &nf->nf_finish);
n++;
- REQ(n, NEWLINE);
+ REQ(n, PYTOK_NEWLINE);
}
static void
@@ -207,7 +207,7 @@ compile_rhs(labellist *ll, nfa *nf, node
addnfaarc(nf, *pa, a, EMPTY);
addnfaarc(nf, b, *pb, EMPTY);
for (; --i >= 0; n++) {
- REQ(n, VBAR);
+ REQ(n, PYTOK_VBAR);
REQN(i, 1);
--i;
n++;
@@ -250,7 +250,7 @@ compile_item(labellist *ll, nfa *nf, nod
i = n->n_nchildren;
REQN(i, 1);
n = n->n_child;
- if (n->n_type == LSQB) {
+ if (n->n_type == PYTOK_LSQB) {
REQN(i, 3);
n++;
REQ(n, RHS);
@@ -262,7 +262,7 @@ compile_item(labellist *ll, nfa *nf, nod
addnfaarc(nf, b, *pb, EMPTY);
REQN(i, 1);
n++;
- REQ(n, RSQB);
+ REQ(n, PYTOK_RSQB);
}
else {
compile_atom(ll, nf, n, pa, pb);
@@ -270,10 +270,10 @@ compile_item(labellist *ll, nfa *nf, nod
return;
n++;
addnfaarc(nf, *pb, *pa, EMPTY);
- if (n->n_type == STAR)
+ if (n->n_type == PYTOK_STAR)
*pb = *pa;
else
- REQ(n, PLUS);
+ REQ(n, PYTOK_PLUS);
}
}
@@ -287,21 +287,21 @@ compile_atom(labellist *ll, nfa *nf, nod
(void)i; /* Don't warn about set but unused */
REQN(i, 1);
n = n->n_child;
- if (n->n_type == LPAR) {
+ if (n->n_type == PYTOK_LPAR) {
REQN(i, 3);
n++;
REQ(n, RHS);
compile_rhs(ll, nf, n, pa, pb);
n++;
- REQ(n, RPAR);
+ REQ(n, PYTOK_RPAR);
}
- else if (n->n_type == NAME || n->n_type == STRING) {
+ else if (n->n_type == PYTOK_NAME || n->n_type == PYTOK_STRING) {
*pa = addnfastate(nf);
*pb = addnfastate(nf);
addnfaarc(nf, *pa, *pb, addlabel(ll, n->n_type, n->n_str));
}
else
- REQ(n, NAME);
+ REQ(n, PYTOK_NAME);
}
static void
« no previous file with comments | « Parser/parsetok.c ('k') | Parser/tokenizer.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+