Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1)

Side by Side Diff: Parser/parsetok.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 4 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Parser/parser.c ('k') | Parser/pgen.c » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 1
2 /* Parser-tokenizer link implementation */ 2 /* Parser-tokenizer link implementation */
3 3
4 #include "pgenheaders.h" 4 #include "pgenheaders.h"
5 #include "tokenizer.h" 5 #include "tokenizer.h"
6 #include "node.h" 6 #include "node.h"
7 #include "grammar.h" 7 #include "grammar.h"
8 #include "parser.h" 8 #include "parser.h"
9 #include "parsetok.h" 9 #include "parsetok.h"
10 #include "errcode.h" 10 #include "errcode.h"
11 #include "graminit.h" 11 #include "graminit.h"
12 12
13 13
14 /* Forward */ 14 /* Forward */
15 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); 15 static node *parsetok(PyTokenizer_State *, grammar *, int, perrdetail *, int *);
16 static int initerr(perrdetail *err_ret, PyObject * filename); 16 static int initerr(perrdetail *err_ret, PyObject * filename);
17 17
18 /* Parse input coming from a string. Return error code, print some errors. */ 18 /* Parse input coming from a string. Return error code, print some errors. */
19 node * 19 node *
20 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret) 20 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
21 { 21 {
22 return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0); 22 return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
23 } 23 }
24 24
25 node * 25 node *
(...skipping 12 matching lines...) Expand all
38 int iflags = flags; 38 int iflags = flags;
39 return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start, 39 return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
40 err_ret, &iflags); 40 err_ret, &iflags);
41 } 41 }
42 42
43 node * 43 node *
44 PyParser_ParseStringObject(const char *s, PyObject *filename, 44 PyParser_ParseStringObject(const char *s, PyObject *filename,
45 grammar *g, int start, 45 grammar *g, int start,
46 perrdetail *err_ret, int *flags) 46 perrdetail *err_ret, int *flags)
47 { 47 {
48 struct tok_state *tok; 48 PyTokenizer_State *tok;
49 int exec_input = start == file_input; 49 int exec_input = start == file_input;
50 50
51 if (initerr(err_ret, filename) < 0) 51 if (initerr(err_ret, filename) < 0)
52 return NULL; 52 return NULL;
53 53
54 if (*flags & PyPARSE_IGNORE_COOKIE) 54 if (*flags & PyPARSE_IGNORE_COOKIE)
55 tok = PyTokenizer_FromUTF8(s, exec_input); 55 tok = PyTokenizer_FromUTF8(s, exec_input);
56 else 56 else
57 tok = PyTokenizer_FromString(s, exec_input); 57 tok = PyTokenizer_FromString(s, exec_input);
58 if (tok == NULL) { 58 if (tok == NULL) {
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
111 return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1, 111 return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
112 ps2, err_ret, &iflags); 112 ps2, err_ret, &iflags);
113 } 113 }
114 114
115 node * 115 node *
116 PyParser_ParseFileObject(FILE *fp, PyObject *filename, 116 PyParser_ParseFileObject(FILE *fp, PyObject *filename,
117 const char *enc, grammar *g, int start, 117 const char *enc, grammar *g, int start,
118 const char *ps1, const char *ps2, 118 const char *ps1, const char *ps2,
119 perrdetail *err_ret, int *flags) 119 perrdetail *err_ret, int *flags)
120 { 120 {
121 struct tok_state *tok; 121 PyTokenizer_State *tok;
122 122
123 if (initerr(err_ret, filename) < 0) 123 if (initerr(err_ret, filename) < 0)
124 return NULL; 124 return NULL;
125 125
126 if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) { 126 if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
127 err_ret->error = E_NOMEM; 127 err_ret->error = E_NOMEM;
128 return NULL; 128 return NULL;
129 } 129 }
130 #ifndef PGEN 130 #ifndef PGEN
131 Py_INCREF(err_ret->filename); 131 Py_INCREF(err_ret->filename);
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
174 filename = "<string>"; 174 filename = "<string>";
175 PySys_WriteStderr(msg, filename, lineno); 175 PySys_WriteStderr(msg, filename, lineno);
176 } 176 }
177 #endif 177 #endif
178 #endif 178 #endif
179 179
180 /* Parse input coming from the given tokenizer structure. 180 /* Parse input coming from the given tokenizer structure.
181 Return error code. */ 181 Return error code. */
182 182
183 static node * 183 static node *
184 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, 184 parsetok(PyTokenizer_State *tok, grammar *g, int start, perrdetail *err_ret,
185 int *flags) 185 int *flags)
186 { 186 {
187 parser_state *ps; 187 parser_state *ps;
188 node *n; 188 node *n;
189 int started = 0; 189 int started = 0;
190 190
191 if ((ps = PyParser_New(g, start)) == NULL) { 191 if ((ps = PyParser_New(g, start)) == NULL) {
192 err_ret->error = E_NOMEM; 192 err_ret->error = E_NOMEM;
193 PyTokenizer_Free(tok); 193 PyTokenizer_Free(tok);
194 return NULL; 194 return NULL;
195 } 195 }
196 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 196 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
197 if (*flags & PyPARSE_BARRY_AS_BDFL) 197 if (*flags & PyPARSE_BARRY_AS_BDFL)
198 ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL; 198 ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL;
199 #endif 199 #endif
200 200
201 for (;;) { 201 for (;;) {
202 char *a, *b; 202 char *a, *b;
203 int type; 203 int type;
204 size_t len; 204 size_t len;
205 char *str; 205 char *str;
206 int col_offset; 206 int col_offset;
207 207
208 type = PyTokenizer_Get(tok, &a, &b); 208 type = PyTokenizer_Get(tok, &a, &b);
209 if (type == ERRORTOKEN) { 209 if (type == PYTOK_ERRORTOKEN) {
210 err_ret->error = tok->done; 210 err_ret->error = tok->done;
211 break; 211 break;
212 } 212 }
213 if (type == ENDMARKER && started) { 213 if (type == PYTOK_ENDMARKER && started) {
214 type = NEWLINE; /* Add an extra newline */ 214 type = PYTOK_NEWLINE; /* Add an extra newline */
215 started = 0; 215 started = 0;
216 /* Add the right number of dedent tokens, 216 /* Add the right number of dedent tokens,
217 except if a certain flag is given -- 217 except if a certain flag is given --
218 codeop.py uses this. */ 218 codeop.py uses this. */
219 if (tok->indent && 219 if (tok->indent &&
220 !(*flags & PyPARSE_DONT_IMPLY_DEDENT)) 220 !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
221 { 221 {
222 tok->pendin = -tok->indent; 222 tok->pendin = -tok->indent;
223 tok->indent = 0; 223 tok->indent = 0;
224 } 224 }
225 } 225 }
226 else 226 else
227 started = 1; 227 started = 1;
228 len = b - a; /* XXX this may compute NULL - NULL */ 228 len = b - a; /* XXX this may compute NULL - NULL */
229 str = (char *) PyObject_MALLOC(len + 1); 229 str = (char *) PyObject_MALLOC(len + 1);
230 if (str == NULL) { 230 if (str == NULL) {
231 err_ret->error = E_NOMEM; 231 err_ret->error = E_NOMEM;
232 break; 232 break;
233 } 233 }
234 if (len > 0) 234 if (len > 0)
235 strncpy(str, a, len); 235 strncpy(str, a, len);
236 str[len] = '\0'; 236 str[len] = '\0';
237 237
238 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 238 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
239 if (type == NOTEQUAL) { 239 if (type == PYTOK_NOTEQUAL) {
240 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 240 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
241 strcmp(str, "!=")) { 241 strcmp(str, "!=")) {
242 PyObject_FREE(str); 242 PyObject_FREE(str);
243 err_ret->error = E_SYNTAX; 243 err_ret->error = E_SYNTAX;
244 break; 244 break;
245 } 245 }
246 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 246 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
247 strcmp(str, "<>")) { 247 strcmp(str, "<>")) {
248 PyObject_FREE(str); 248 PyObject_FREE(str);
249 err_ret->text = "with Barry as BDFL, use '<>' " 249 err_ret->text = "with Barry as BDFL, use '<>' "
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after
374 else { 374 else {
375 err_ret->filename = PyUnicode_FromString("<string>"); 375 err_ret->filename = PyUnicode_FromString("<string>");
376 if (err_ret->filename == NULL) { 376 if (err_ret->filename == NULL) {
377 err_ret->error = E_ERROR; 377 err_ret->error = E_ERROR;
378 return -1; 378 return -1;
379 } 379 }
380 } 380 }
381 #endif 381 #endif
382 return 0; 382 return 0;
383 } 383 }
OLDNEW
« no previous file with comments | « Parser/parser.c ('k') | Parser/pgen.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+