Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(264929)

Delta Between Two Patch Sets: Parser/parsetok.c

Issue 11619: On Windows, don't encode filenames in the import machinery
Left Patch Set: Created 8 years, 4 months ago
Right Patch Set: Created 5 years, 11 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « Modules/symtablemodule.c ('k') | Python/ast.c » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 1
2 /* Parser-tokenizer link implementation */ 2 /* Parser-tokenizer link implementation */
3 3
4 #include "pgenheaders.h" 4 #include "pgenheaders.h"
5 #include "tokenizer.h" 5 #include "tokenizer.h"
6 #include "node.h" 6 #include "node.h"
7 #include "grammar.h" 7 #include "grammar.h"
8 #include "parser.h" 8 #include "parser.h"
9 #include "parsetok.h" 9 #include "parsetok.h"
10 #include "errcode.h" 10 #include "errcode.h"
11 #include "graminit.h" 11 #include "graminit.h"
12 12
13 13
14 /* Forward */ 14 /* Forward */
15 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); 15 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
16 static int initerr(perrdetail *err_ret, const char* filename); 16 static int initerr(perrdetail *err_ret, PyObject * filename);
17 17
18 /* Parse input coming from a string. Return error code, print some errors. */ 18 /* Parse input coming from a string. Return error code, print some errors. */
19 node * 19 node *
20 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret) 20 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
21 { 21 {
22 return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0); 22 return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
23 } 23 }
24 24
25 node * 25 node *
26 PyParser_ParseStringFlags(const char *s, grammar *g, int start, 26 PyParser_ParseStringFlags(const char *s, grammar *g, int start,
27 perrdetail *err_ret, int flags) 27 perrdetail *err_ret, int flags)
28 { 28 {
29 return PyParser_ParseStringFlagsFilename(s, NULL, 29 return PyParser_ParseStringFlagsFilename(s, NULL,
30 g, start, err_ret, flags); 30 g, start, err_ret, flags);
31 } 31 }
32 32
33 node * 33 node *
34 PyParser_ParseStringFlagsFilename(const char *s, const char *filename, 34 PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
35 grammar *g, int start, 35 grammar *g, int start,
36 perrdetail *err_ret, int flags) 36 perrdetail *err_ret, int flags)
37 { 37 {
38 int iflags = flags; 38 int iflags = flags;
39 return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start, 39 return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
40 err_ret, &iflags); 40 err_ret, &iflags);
41 } 41 }
42 42
43 node * 43 node *
44 PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename, 44 PyParser_ParseStringObject(const char *s, PyObject *filename,
45 grammar *g, int start, 45 grammar *g, int start,
46 perrdetail *err_ret, int *flags) 46 perrdetail *err_ret, int *flags)
47 { 47 {
48 struct tok_state *tok; 48 struct tok_state *tok;
49 int exec_input = start == file_input; 49 int exec_input = start == file_input;
50 50
51 if (initerr(err_ret, filename) < 0) 51 if (initerr(err_ret, filename) < 0)
52 return NULL; 52 return NULL;
53 53
54 if (*flags & PyPARSE_IGNORE_COOKIE) 54 if (*flags & PyPARSE_IGNORE_COOKIE)
55 tok = PyTokenizer_FromUTF8(s, exec_input); 55 tok = PyTokenizer_FromUTF8(s, exec_input);
56 else 56 else
57 tok = PyTokenizer_FromString(s, exec_input); 57 tok = PyTokenizer_FromString(s, exec_input);
58 if (tok == NULL) { 58 if (tok == NULL) {
59 err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM; 59 err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
60 return NULL; 60 return NULL;
61 } 61 }
62 62
63 #ifndef PGEN 63 #ifndef PGEN
64 Py_INCREF(err_ret->filename); 64 Py_INCREF(err_ret->filename);
65 tok->filename = err_ret->filename; 65 tok->filename = err_ret->filename;
66 #endif 66 #endif
67 return parsetok(tok, g, start, err_ret, flags); 67 return parsetok(tok, g, start, err_ret, flags);
68 } 68 }
69 69
70 node *
71 PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename_str,
72 grammar *g, int start,
73 perrdetail *err_ret, int *flags)
74 {
75 PyObject *filename;
76 node *n;
77 filename = NULL;
78 #ifndef PGEN
79 if (filename_str != NULL) {
80 filename = PyUnicode_DecodeFSDefault(filename_str);
81 if (filename == NULL) {
82 err_ret->error = E_ERROR;
83 return NULL;
84 }
85 }
86 #endif
87 n = PyParser_ParseStringObject(s, filename, g, start, err_ret, flags);
88 #ifndef PGEN
89 Py_XDECREF(filename);
90 #endif
91 return n;
92 }
93
70 /* Parse input coming from a file. Return error code, print some errors. */ 94 /* Parse input coming from a file. Return error code, print some errors. */
71 95
72 node * 96 node *
73 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start, 97 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
74 char *ps1, char *ps2, perrdetail *err_ret) 98 char *ps1, char *ps2, perrdetail *err_ret)
75 { 99 {
76 return PyParser_ParseFileFlags(fp, filename, NULL, 100 return PyParser_ParseFileFlags(fp, filename, NULL,
77 g, start, ps1, ps2, err_ret, 0); 101 g, start, ps1, ps2, err_ret, 0);
78 } 102 }
79 103
80 node * 104 node *
81 PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc, 105 PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
82 grammar *g, int start, 106 grammar *g, int start,
83 char *ps1, char *ps2, perrdetail *err_ret, int flags) 107 char *ps1, char *ps2, perrdetail *err_ret, int flags)
84 { 108 {
85 int iflags = flags; 109 int iflags = flags;
86 return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1, 110 return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
87 ps2, err_ret, &iflags); 111 ps2, err_ret, &iflags);
88 } 112 }
89 113
90 node * 114 node *
115 PyParser_ParseFileObject(FILE *fp, PyObject *filename,
116 const char *enc, grammar *g, int start,
117 char *ps1, char *ps2, perrdetail *err_ret,
118 int *flags)
119 {
120 struct tok_state *tok;
121
122 if (initerr(err_ret, filename) < 0)
123 return NULL;
124
125 if ((tok = PyTokenizer_FromFile(fp, (char *)enc, ps1, ps2)) == NULL) {
126 err_ret->error = E_NOMEM;
127 return NULL;
128 }
129 #ifndef PGEN
130 Py_INCREF(err_ret->filename);
131 tok->filename = err_ret->filename;
132 #endif
133 return parsetok(tok, g, start, err_ret, flags);
134 }
135
136 node *
91 PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, 137 PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
92 const char *enc, grammar *g, int start, 138 const char *enc, grammar *g, int start,
93 char *ps1, char *ps2, perrdetail *err_ret, int *flags) 139 char *ps1, char *ps2, perrdetail *err_ret, int *flags)
94 { 140 {
95 struct tok_state *tok; 141 node *n;
96 142 PyObject *fileobj;
97 if (initerr(err_ret, filename) < 0) 143 #ifndef PGEN
98 return NULL; 144 if (filename != NULL) {
99 145 fileobj = PyUnicode_DecodeFSDefault(filename);
100 if ((tok = PyTokenizer_FromFile(fp, (char *)enc, ps1, ps2)) == NULL) { 146 if (fileobj == NULL) {
101 err_ret->error = E_NOMEM; 147 err_ret->error = E_ERROR;
102 return NULL; 148 return NULL;
103 } 149 }
104 #ifndef PGEN 150 }
105 Py_INCREF(err_ret->filename); 151 else
106 tok->filename = err_ret->filename; 152 #endif
107 #endif 153 fileobj = NULL;
108 return parsetok(tok, g, start, err_ret, flags); 154 n = PyParser_ParseFileObject(fp, fileobj, enc, g,
155 start, ps1, ps2, err_ret, flags);
156 #ifndef PGEN
157 Py_XDECREF(fileobj);
158 #endif
159 return n;
109 } 160 }
110 161
111 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 162 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
112 #if 0 163 #if 0
113 static char with_msg[] = 164 static char with_msg[] =
114 "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n"; 165 "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
115 166
116 static char as_msg[] = 167 static char as_msg[] =
117 "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n"; 168 "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
118 169
(...skipping 180 matching lines...) Expand 10 before | Expand all | Expand 10 after
299 n = r; 350 n = r;
300 } 351 }
301 352
302 done: 353 done:
303 PyTokenizer_Free(tok); 354 PyTokenizer_Free(tok);
304 355
305 return n; 356 return n;
306 } 357 }
307 358
308 static int 359 static int
309 initerr(perrdetail *err_ret, const char *filename) 360 initerr(perrdetail *err_ret, PyObject *filename)
310 { 361 {
311 err_ret->error = E_OK; 362 err_ret->error = E_OK;
312 err_ret->lineno = 0; 363 err_ret->lineno = 0;
313 err_ret->offset = 0; 364 err_ret->offset = 0;
314 err_ret->text = NULL; 365 err_ret->text = NULL;
315 err_ret->token = -1; 366 err_ret->token = -1;
316 err_ret->expected = -1; 367 err_ret->expected = -1;
317 #ifndef PGEN 368 #ifndef PGEN
318 if (filename) 369 if (filename) {
319 err_ret->filename = PyUnicode_DecodeFSDefault(filename); 370 Py_INCREF(filename);
320 else 371 err_ret->filename = filename;
372 }
373 else {
321 err_ret->filename = PyUnicode_FromString("<string>"); 374 err_ret->filename = PyUnicode_FromString("<string>");
322 if (err_ret->filename == NULL) { 375 if (err_ret->filename == NULL) {
323 err_ret->error = E_ERROR; 376 err_ret->error = E_ERROR;
324 return -1; 377 return -1;
378 }
325 } 379 }
326 #endif 380 #endif
327 return 0; 381 return 0;
328 } 382 }
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+