Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(2)

Unified Diff: Parser/parsetok.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 5 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « Parser/parser.c ('k') | Parser/pgen.c » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -12,7 +12,7 @@
/* Forward */
-static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
+static node *parsetok(PyTokenizer_State *, grammar *, int, perrdetail *, int *);
static int initerr(perrdetail *err_ret, PyObject * filename);
/* Parse input coming from a string. Return error code, print some errors. */
@@ -45,7 +45,7 @@ PyParser_ParseStringObject(const char *s
grammar *g, int start,
perrdetail *err_ret, int *flags)
{
- struct tok_state *tok;
+ PyTokenizer_State *tok;
int exec_input = start == file_input;
if (initerr(err_ret, filename) < 0)
@@ -118,7 +118,7 @@ PyParser_ParseFileObject(FILE *fp, PyObj
const char *ps1, const char *ps2,
perrdetail *err_ret, int *flags)
{
- struct tok_state *tok;
+ PyTokenizer_State *tok;
if (initerr(err_ret, filename) < 0)
return NULL;
@@ -181,7 +181,7 @@ warn(const char *msg, const char *filena
Return error code. */
static node *
-parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
+parsetok(PyTokenizer_State *tok, grammar *g, int start, perrdetail *err_ret,
int *flags)
{
parser_state *ps;
@@ -206,12 +206,12 @@ parsetok(struct tok_state *tok, grammar
int col_offset;
type = PyTokenizer_Get(tok, &a, &b);
- if (type == ERRORTOKEN) {
+ if (type == PYTOK_ERRORTOKEN) {
err_ret->error = tok->done;
break;
}
- if (type == ENDMARKER && started) {
- type = NEWLINE; /* Add an extra newline */
+ if (type == PYTOK_ENDMARKER && started) {
+ type = PYTOK_NEWLINE; /* Add an extra newline */
started = 0;
/* Add the right number of dedent tokens,
except if a certain flag is given --
@@ -236,7 +236,7 @@ parsetok(struct tok_state *tok, grammar
str[len] = '\0';
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
- if (type == NOTEQUAL) {
+ if (type == PYTOK_NOTEQUAL) {
if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
strcmp(str, "!=")) {
PyObject_FREE(str);
« no previous file with comments | « Parser/parser.c ('k') | Parser/pgen.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+