Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(3)

Delta Between Two Patch Sets: Parser/grammar1.c

Issue 3353: make built-in tokenizer available via Python C API
Left Patch Set: Created 4 years, 10 months ago
Right Patch Set: Created 4 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « Parser/firstsets.c ('k') | Parser/grammar.c » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 1
2 /* Grammar subroutines needed by parser */ 2 /* Grammar subroutines needed by parser */
3 3
4 #include "Python.h" 4 #include "Python.h"
5 #include "pgenheaders.h" 5 #include "pgenheaders.h"
6 #include "grammar.h" 6 #include "grammar.h"
7 #include "token.h" 7 #include "token.h"
8 8
9 /* Return the DFA for the given type */ 9 /* Return the DFA for the given type */
10 10
11 dfa * 11 dfa *
12 PyGrammar_FindDFA(grammar *g, int type) 12 PyGrammar_FindDFA(grammar *g, int type)
13 { 13 {
14 dfa *d; 14 dfa *d;
15 #if 1 15 #if 1
16 /* Massive speed-up */ 16 /* Massive speed-up */
17 d = &g->g_dfa[type - NT_OFFSET]; 17 d = &g->g_dfa[type - PYTOK_NT_OFFSET];
18 assert(d->d_type == type); 18 assert(d->d_type == type);
19 return d; 19 return d;
20 #else 20 #else
21 /* Old, slow version */ 21 /* Old, slow version */
22 int i; 22 int i;
23 23
24 for (i = g->g_ndfas, d = g->g_dfa; --i >= 0; d++) { 24 for (i = g->g_ndfas, d = g->g_dfa; --i >= 0; d++) {
25 if (d->d_type == type) 25 if (d->d_type == type)
26 return d; 26 return d;
27 } 27 }
28 assert(0); 28 assert(0);
29 /* NOTREACHED */ 29 /* NOTREACHED */
30 #endif 30 #endif
31 } 31 }
32 32
33 const char * 33 const char *
34 PyGrammar_LabelRepr(label *lb) 34 PyGrammar_LabelRepr(label *lb)
35 { 35 {
36 static char buf[100]; 36 static char buf[100];
37 37
38 if (lb->lb_type == ENDMARKER) 38 if (lb->lb_type == PYTOK_ENDMARKER)
39 return "EMPTY"; 39 return "EMPTY";
40 else if (ISNONTERMINAL(lb->lb_type)) { 40 else if (PYTOK_ISNONTERMINAL(lb->lb_type)) {
41 if (lb->lb_str == NULL) { 41 if (lb->lb_str == NULL) {
42 PyOS_snprintf(buf, sizeof(buf), "NT%d", lb->lb_type); 42 PyOS_snprintf(buf, sizeof(buf), "NT%d", lb->lb_type);
43 return buf; 43 return buf;
44 } 44 }
45 else 45 else
46 return lb->lb_str; 46 return lb->lb_str;
47 } 47 }
48 else if (lb->lb_type < N_TOKENS) { 48 else if (lb->lb_type < PYTOK_N_TOKENS) {
49 if (lb->lb_str == NULL) 49 if (lb->lb_str == NULL)
50 return _PyParser_TokenNames[lb->lb_type]; 50 return _PyParser_TokenNames[lb->lb_type];
51 else { 51 else {
52 PyOS_snprintf(buf, sizeof(buf), "%.32s(%.32s)", 52 PyOS_snprintf(buf, sizeof(buf), "%.32s(%.32s)",
53 _PyParser_TokenNames[lb->lb_type], lb->lb_str); 53 _PyParser_TokenNames[lb->lb_type], lb->lb_str);
54 return buf; 54 return buf;
55 } 55 }
56 } 56 }
57 else { 57 else {
58 Py_FatalError("invalid label"); 58 Py_FatalError("invalid label");
59 return NULL; 59 return NULL;
60 } 60 }
61 } 61 }
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+