Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1)

Side by Side Diff: Parser/grammar.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 4 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Parser/grammar1.c ('k') | Parser/listnode.c » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 1
2 /* Grammar implementation */ 2 /* Grammar implementation */
3 3
4 #include "Python.h" 4 #include "Python.h"
5 #include "pgenheaders.h" 5 #include "pgenheaders.h"
6 6
7 #include <ctype.h> 7 #include <ctype.h>
8 8
9 #include "token.h" 9 #include "token.h"
10 #include "grammar.h" 10 #include "grammar.h"
(...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after
142 } 142 }
143 143
144 static void 144 static void
145 translabel(grammar *g, label *lb) 145 translabel(grammar *g, label *lb)
146 { 146 {
147 int i; 147 int i;
148 148
149 if (Py_DebugFlag) 149 if (Py_DebugFlag)
150 printf("Translating label %s ...\n", PyGrammar_LabelRepr(lb)); 150 printf("Translating label %s ...\n", PyGrammar_LabelRepr(lb));
151 151
152 if (lb->lb_type == NAME) { 152 if (lb->lb_type == PYTOK_NAME) {
153 for (i = 0; i < g->g_ndfas; i++) { 153 for (i = 0; i < g->g_ndfas; i++) {
154 if (strcmp(lb->lb_str, g->g_dfa[i].d_name) == 0) { 154 if (strcmp(lb->lb_str, g->g_dfa[i].d_name) == 0) {
155 if (Py_DebugFlag) 155 if (Py_DebugFlag)
156 printf( 156 printf(
157 "Label %s is non-terminal %d.\n", 157 "Label %s is non-terminal %d.\n",
158 lb->lb_str, 158 lb->lb_str,
159 g->g_dfa[i].d_type); 159 g->g_dfa[i].d_type);
160 lb->lb_type = g->g_dfa[i].d_type; 160 lb->lb_type = g->g_dfa[i].d_type;
161 free(lb->lb_str); 161 free(lb->lb_str);
162 lb->lb_str = NULL; 162 lb->lb_str = NULL;
163 return; 163 return;
164 } 164 }
165 } 165 }
166 for (i = 0; i < (int)N_TOKENS; i++) { 166 for (i = 0; i < (int)PYTOK_N_TOKENS; i++) {
167 if (strcmp(lb->lb_str, _PyParser_TokenNames[i]) == 0) { 167 if (strcmp(lb->lb_str, _PyParser_TokenNames[i]) == 0) {
168 if (Py_DebugFlag) 168 if (Py_DebugFlag)
169 printf("Label %s is terminal %d.\n", 169 printf("Label %s is terminal %d.\n",
170 lb->lb_str, i); 170 lb->lb_str, i);
171 lb->lb_type = i; 171 lb->lb_type = i;
172 free(lb->lb_str); 172 free(lb->lb_str);
173 lb->lb_str = NULL; 173 lb->lb_str = NULL;
174 return; 174 return;
175 } 175 }
176 } 176 }
177 printf("Can't translate NAME label '%s'\n", lb->lb_str); 177 printf("Can't translate PYTOK_NAME label '%s'\n", lb->lb_str);
178 return; 178 return;
179 } 179 }
180 180
181 if (lb->lb_type == STRING) { 181 if (lb->lb_type == PYTOK_STRING) {
182 if (isalpha(Py_CHARMASK(lb->lb_str[1])) || 182 if (isalpha(Py_CHARMASK(lb->lb_str[1])) ||
183 lb->lb_str[1] == '_') { 183 lb->lb_str[1] == '_') {
184 char *p; 184 char *p;
185 char *src; 185 char *src;
186 char *dest; 186 char *dest;
187 size_t name_len; 187 size_t name_len;
188 if (Py_DebugFlag) 188 if (Py_DebugFlag)
189 printf("Label %s is a keyword\n", lb->lb_str); 189 printf("Label %s is a keyword\n", lb->lb_str);
190 lb->lb_type = NAME; 190 lb->lb_type = PYTOK_NAME;
191 src = lb->lb_str + 1; 191 src = lb->lb_str + 1;
192 p = strchr(src, '\''); 192 p = strchr(src, '\'');
193 if (p) 193 if (p)
194 name_len = p - src; 194 name_len = p - src;
195 else 195 else
196 name_len = strlen(src); 196 name_len = strlen(src);
197 dest = (char *)malloc(name_len + 1); 197 dest = (char *)malloc(name_len + 1);
198 if (!dest) { 198 if (!dest) {
199 printf("Can't alloc dest '%s'\n", src); 199 printf("Can't alloc dest '%s'\n", src);
200 return; 200 return;
201 } 201 }
202 strncpy(dest, src, name_len); 202 strncpy(dest, src, name_len);
203 dest[name_len] = '\0'; 203 dest[name_len] = '\0';
204 free(lb->lb_str); 204 free(lb->lb_str);
205 lb->lb_str = dest; 205 lb->lb_str = dest;
206 } 206 }
207 else if (lb->lb_str[2] == lb->lb_str[0]) { 207 else if (lb->lb_str[2] == lb->lb_str[0]) {
208 int type = (int) PyToken_OneChar(lb->lb_str[1]); 208 int type = (int) PyToken_OneChar(lb->lb_str[1]);
209 if (type != OP) { 209 if (type != PYTOK_OP) {
210 lb->lb_type = type; 210 lb->lb_type = type;
211 free(lb->lb_str); 211 free(lb->lb_str);
212 lb->lb_str = NULL; 212 lb->lb_str = NULL;
213 } 213 }
214 else 214 else
215 printf("Unknown OP label %s\n", 215 printf("Unknown PYTOK_OP label %s\n",
216 lb->lb_str); 216 lb->lb_str);
217 } 217 }
218 else if (lb->lb_str[2] && lb->lb_str[3] == lb->lb_str[0]) { 218 else if (lb->lb_str[2] && lb->lb_str[3] == lb->lb_str[0]) {
219 int type = (int) PyToken_TwoChars(lb->lb_str[1], 219 int type = (int) PyToken_TwoChars(lb->lb_str[1],
220 lb->lb_str[2]); 220 lb->lb_str[2]);
221 if (type != OP) { 221 if (type != PYTOK_OP) {
222 lb->lb_type = type; 222 lb->lb_type = type;
223 free(lb->lb_str); 223 free(lb->lb_str);
224 lb->lb_str = NULL; 224 lb->lb_str = NULL;
225 } 225 }
226 else 226 else
227 printf("Unknown OP label %s\n", 227 printf("Unknown PYTOK_OP label %s\n",
228 lb->lb_str); 228 lb->lb_str);
229 } 229 }
230 else if (lb->lb_str[2] && lb->lb_str[3] && lb->lb_str[4] == lb->lb_str[0 ]) { 230 else if (lb->lb_str[2] && lb->lb_str[3] && lb->lb_str[4] == lb->lb_str[0 ]) {
231 int type = (int) PyToken_ThreeChars(lb->lb_str[1], 231 int type = (int) PyToken_ThreeChars(lb->lb_str[1],
232 lb->lb_str[2], 232 lb->lb_str[2],
233 lb->lb_str[3]); 233 lb->lb_str[3]);
234 if (type != OP) { 234 if (type != PYTOK_OP) {
235 lb->lb_type = type; 235 lb->lb_type = type;
236 free(lb->lb_str); 236 free(lb->lb_str);
237 lb->lb_str = NULL; 237 lb->lb_str = NULL;
238 } 238 }
239 else 239 else
240 printf("Unknown OP label %s\n", 240 printf("Unknown PYTOK_OP label %s\n",
241 lb->lb_str); 241 lb->lb_str);
242 } 242 }
243 else 243 else
244 printf("Can't translate STRING label %s\n", 244 printf("Can't translate PYTOK_STRING label %s\n",
245 lb->lb_str); 245 lb->lb_str);
246 } 246 }
247 else 247 else
248 printf("Can't translate label '%s'\n", 248 printf("Can't translate label '%s'\n",
249 PyGrammar_LabelRepr(lb)); 249 PyGrammar_LabelRepr(lb));
250 } 250 }
OLDNEW
« no previous file with comments | « Parser/grammar1.c ('k') | Parser/listnode.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+