Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(21)

Unified Diff: Parser/tokenizer.c

Issue 3353: make built-in tokenizer available via Python C API
Patch Set: Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « Parser/pgen.c ('k') | Parser/tokenizer.h » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -40,9 +40,9 @@ extern char *PyOS_Readline(FILE *, FILE
#define TABSIZE 8
/* Forward */
-static struct tok_state *tok_new(void);
-static int tok_nextc(struct tok_state *tok);
-static void tok_backup(struct tok_state *tok, int c);
+static PyTokenizer_State *tok_new(void);
+static int tok_nextc(PyTokenizer_State *tok);
+static void tok_backup(PyTokenizer_State *tok, int c);
/* Token names */
@@ -110,11 +110,11 @@ const char *_PyParser_TokenNames[] = {
/* Create and initialize a new tok_state structure */
-static struct tok_state *
+static PyTokenizer_State *
tok_new(void)
{
- struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
- sizeof(struct tok_state));
+ PyTokenizer_State *tok = (PyTokenizer_State *)PyMem_MALLOC(
+ sizeof(PyTokenizer_State));
if (tok == NULL)
return NULL;
tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
@@ -133,7 +133,7 @@ tok_new(void)
tok->alterror = 1;
tok->alttabsize = 1;
tok->altindstack[0] = 0;
- tok->decoding_state = STATE_INIT;
+ tok->decoding_state = PYTOKENIZER_STATE_INIT;
tok->decoding_erred = 0;
tok->read_coding_spec = 0;
tok->enc = NULL;
@@ -148,7 +148,7 @@ tok_new(void)
}
static char *
-new_string(const char *s, Py_ssize_t len, struct tok_state *tok)
+new_string(const char *s, Py_ssize_t len, PyTokenizer_State *tok)
{
char* result = (char *)PyMem_MALLOC(len + 1);
if (!result) {
@@ -163,19 +163,19 @@ new_string(const char *s, Py_ssize_t len
#ifdef PGEN
static char *
-decoding_fgets(char *s, int size, struct tok_state *tok)
+decoding_fgets(char *s, int size, PyTokenizer_State *tok)
{
return fgets(s, size, tok->fp);
}
static int
-decoding_feof(struct tok_state *tok)
+decoding_feof(PyTokenizer_State *tok)
{
return feof(tok->fp);
}
static char *
-decode_str(const char *str, int exec_input, struct tok_state *tok)
+decode_str(const char *str, int exec_input, PyTokenizer_State *tok)
{
return new_string(str, strlen(str), tok);
}
@@ -183,7 +183,7 @@ decode_str(const char *str, int exec_inp
#else /* PGEN */
static char *
-error_ret(struct tok_state *tok) /* XXX */
+error_ret(PyTokenizer_State *tok) /* XXX */
{
tok->decoding_erred = 1;
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
@@ -225,7 +225,7 @@ get_normal_name(char *s) /* for u
/* Return the coding spec in S, or NULL if none is found. */
static int
-get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok)
+get_coding_spec(const char *s, char **spec, Py_ssize_t size, PyTokenizer_State *tok)
{
Py_ssize_t i;
*spec = NULL;
@@ -278,8 +278,8 @@ get_coding_spec(const char *s, char **sp
Return 1 on success, 0 on failure. */
static int
-check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
- int set_readline(struct tok_state *, const char *))
+check_coding_spec(const char* line, Py_ssize_t size, PyTokenizer_State *tok,
+ int set_readline(PyTokenizer_State *, const char *))
{
char *cs;
int r = 1;
@@ -307,14 +307,14 @@ check_coding_spec(const char* line, Py_s
}
tok->read_coding_spec = 1;
if (tok->encoding == NULL) {
- assert(tok->decoding_state == STATE_RAW);
+ assert(tok->decoding_state == PYTOKENIZER_STATE_RAW);
if (strcmp(cs, "utf-8") == 0) {
tok->encoding = cs;
} else {
r = set_readline(tok, cs);
if (r) {
tok->encoding = cs;
- tok->decoding_state = STATE_NORMAL;
+ tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
}
else {
PyErr_Format(PyExc_SyntaxError,
@@ -337,14 +337,14 @@ check_coding_spec(const char* line, Py_s
Return 1 on success, 0 on failure. */
static int
-check_bom(int get_char(struct tok_state *),
- void unget_char(int, struct tok_state *),
- int set_readline(struct tok_state *, const char *),
- struct tok_state *tok)
+check_bom(int get_char(PyTokenizer_State *),
+ void unget_char(int, PyTokenizer_State *),
+ int set_readline(PyTokenizer_State *, const char *),
+ PyTokenizer_State *tok)
{
int ch1, ch2, ch3;
ch1 = get_char(tok);
- tok->decoding_state = STATE_RAW;
+ tok->decoding_state = PYTOKENIZER_STATE_RAW;
if (ch1 == EOF) {
return 1;
} else if (ch1 == 0xEF) {
@@ -373,7 +373,7 @@ check_bom(int get_char(struct tok_state
}
if (!set_readline(tok, "utf-16-be"))
return 0;
- tok->decoding_state = STATE_NORMAL;
+ tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
} else if (ch1 == 0xFF) {
ch2 = get_char(tok);
if (ch2 != 0xFE) {
@@ -383,7 +383,7 @@ check_bom(int get_char(struct tok_state
}
if (!set_readline(tok, "utf-16-le"))
return 0;
- tok->decoding_state = STATE_NORMAL;
+ tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
#endif
} else {
unget_char(ch1, tok);
@@ -414,7 +414,7 @@ check_bom(int get_char(struct tok_state
*/
static char *
-fp_readl(char *s, int size, struct tok_state *tok)
+fp_readl(char *s, int size, PyTokenizer_State *tok)
{
PyObject* bufobj;
const char *buf;
@@ -485,7 +485,7 @@ error:
Return 1 on success, 0 on failure. */
static int
-fp_setreadl(struct tok_state *tok, const char* enc)
+fp_setreadl(PyTokenizer_State *tok, const char* enc)
{
PyObject *readline = NULL, *stream = NULL, *io = NULL;
_Py_IDENTIFIER(open);
@@ -533,13 +533,13 @@ fp_setreadl(struct tok_state *tok, const
/* Fetch the next byte from TOK. */
-static int fp_getc(struct tok_state *tok) {
+static int fp_getc(PyTokenizer_State *tok) {
return getc(tok->fp);
}
/* Unfetch the last byte back into TOK. */
-static void fp_ungetc(int c, struct tok_state *tok) {
+static void fp_ungetc(int c, PyTokenizer_State *tok) {
ungetc(c, tok->fp);
}
@@ -575,17 +575,17 @@ static int valid_utf8(const unsigned cha
if necessary. */
static char *
-decoding_fgets(char *s, int size, struct tok_state *tok)
+decoding_fgets(char *s, int size, PyTokenizer_State *tok)
{
char *line = NULL;
int badchar = 0;
for (;;) {
- if (tok->decoding_state == STATE_NORMAL) {
+ if (tok->decoding_state == PYTOKENIZER_STATE_NORMAL) {
/* We already have a codec associated with
this input. */
line = fp_readl(s, size, tok);
break;
- } else if (tok->decoding_state == STATE_RAW) {
+ } else if (tok->decoding_state == PYTOKENIZER_STATE_RAW) {
/* We want a 'raw' read. */
line = Py_UniversalNewlineFgets(s, size,
tok->fp, NULL);
@@ -596,7 +596,7 @@ decoding_fgets(char *s, int size, struct
reader functions from now on. */
if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
return error_ret(tok);
- assert(tok->decoding_state != STATE_INIT);
+ assert(tok->decoding_state != PYTOKENIZER_STATE_INIT);
}
}
if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
@@ -632,9 +632,9 @@ decoding_fgets(char *s, int size, struct
}
static int
-decoding_feof(struct tok_state *tok)
+decoding_feof(PyTokenizer_State *tok)
{
- if (tok->decoding_state != STATE_NORMAL) {
+ if (tok->decoding_state != PYTOKENIZER_STATE_NORMAL) {
return feof(tok->fp);
} else {
PyObject* buf = tok->decoding_buffer;
@@ -654,14 +654,14 @@ decoding_feof(struct tok_state *tok)
/* Fetch a byte from TOK, using the string buffer. */
static int
-buf_getc(struct tok_state *tok) {
+buf_getc(PyTokenizer_State *tok) {
return Py_CHARMASK(*tok->str++);
}
/* Unfetch a byte from TOK, using the string buffer. */
static void
-buf_ungetc(int c, struct tok_state *tok) {
+buf_ungetc(int c, PyTokenizer_State *tok) {
tok->str--;
assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
}
@@ -670,7 +670,7 @@ buf_ungetc(int c, struct tok_state *tok)
tokenizer, this means to just record the encoding. */
static int
-buf_setreadl(struct tok_state *tok, const char* enc) {
+buf_setreadl(PyTokenizer_State *tok, const char* enc) {
tok->enc = enc;
return 1;
}
@@ -691,7 +691,7 @@ translate_into_utf8(const char* str, con
static char *
-translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
+translate_newlines(const char *s, int exec_input, PyTokenizer_State *tok) {
int skip_next_lf = 0;
size_t needed_length = strlen(s) + 2, final_length;
char *buf, *current;
@@ -736,7 +736,7 @@ translate_newlines(const char *s, int ex
inside TOK. */
static const char *
-decode_str(const char *input, int single, struct tok_state *tok)
+decode_str(const char *input, int single, PyTokenizer_State *tok)
{
PyObject* utf8 = NULL;
const char *str;
@@ -795,10 +795,10 @@ decode_str(const char *input, int single
/* Set up tokenizer for string */
-struct tok_state *
+PyTokenizer_State *
PyTokenizer_FromString(const char *str, int exec_input)
{
- struct tok_state *tok = tok_new();
+ PyTokenizer_State *tok = tok_new();
if (tok == NULL)
return NULL;
str = decode_str(str, exec_input, tok);
@@ -812,10 +812,10 @@ PyTokenizer_FromString(const char *str,
return tok;
}
-struct tok_state *
+PyTokenizer_State *
PyTokenizer_FromUTF8(const char *str, int exec_input)
{
- struct tok_state *tok = tok_new();
+ PyTokenizer_State *tok = tok_new();
if (tok == NULL)
return NULL;
#ifndef PGEN
@@ -825,7 +825,7 @@ PyTokenizer_FromUTF8(const char *str, in
PyTokenizer_Free(tok);
return NULL;
}
- tok->decoding_state = STATE_RAW;
+ tok->decoding_state = PYTOKENIZER_STATE_RAW;
tok->read_coding_spec = 1;
tok->enc = NULL;
tok->str = str;
@@ -843,11 +843,11 @@ PyTokenizer_FromUTF8(const char *str, in
/* Set up tokenizer for file */
-struct tok_state *
+PyTokenizer_State *
PyTokenizer_FromFile(FILE *fp, const char* enc,
const char *ps1, const char *ps2)
{
- struct tok_state *tok = tok_new();
+ PyTokenizer_State *tok = tok_new();
if (tok == NULL)
return NULL;
if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
@@ -868,7 +868,7 @@ PyTokenizer_FromFile(FILE *fp, const cha
return NULL;
}
strcpy(tok->encoding, enc);
- tok->decoding_state = STATE_NORMAL;
+ tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
}
return tok;
}
@@ -877,7 +877,7 @@ PyTokenizer_FromFile(FILE *fp, const cha
/* Free a tok_state structure */
void
-PyTokenizer_Free(struct tok_state *tok)
+PyTokenizer_Free(PyTokenizer_State *tok)
{
if (tok->encoding != NULL)
PyMem_FREE(tok->encoding);
@@ -896,7 +896,7 @@ PyTokenizer_Free(struct tok_state *tok)
/* Get next char, updating state; error code goes into tok->done */
static int
-tok_nextc(struct tok_state *tok)
+tok_nextc(PyTokenizer_State *tok)
{
for (;;) {
if (tok->cur != tok->inp) {
@@ -1093,7 +1093,7 @@ tok_nextc(struct tok_state *tok)
/* Back-up one character */
static void
-tok_backup(struct tok_state *tok, int c)
+tok_backup(PyTokenizer_State *tok, int c)
{
if (c != EOF) {
if (--tok->cur < tok->buf)
@@ -1110,30 +1110,30 @@ int
PyToken_OneChar(int c)
{
switch (c) {
- case '(': return LPAR;
- case ')': return RPAR;
- case '[': return LSQB;
- case ']': return RSQB;
- case ':': return COLON;
- case ',': return COMMA;
- case ';': return SEMI;
- case '+': return PLUS;
- case '-': return MINUS;
- case '*': return STAR;
- case '/': return SLASH;
- case '|': return VBAR;
- case '&': return AMPER;
- case '<': return LESS;
- case '>': return GREATER;
- case '=': return EQUAL;
- case '.': return DOT;
- case '%': return PERCENT;
- case '{': return LBRACE;
- case '}': return RBRACE;
- case '^': return CIRCUMFLEX;
- case '~': return TILDE;
- case '@': return AT;
- default: return OP;
+ case '(': return PYTOK_LPAR;
+ case ')': return PYTOK_RPAR;
+ case '[': return PYTOK_LSQB;
+ case ']': return PYTOK_RSQB;
+ case ':': return PYTOK_COLON;
+ case ',': return PYTOK_COMMA;
+ case ';': return PYTOK_SEMI;
+ case '+': return PYTOK_PLUS;
+ case '-': return PYTOK_MINUS;
+ case '*': return PYTOK_STAR;
+ case '/': return PYTOK_SLASH;
+ case '|': return PYTOK_VBAR;
+ case '&': return PYTOK_AMPER;
+ case '<': return PYTOK_LESS;
+ case '>': return PYTOK_GREATER;
+ case '=': return PYTOK_EQUAL;
+ case '.': return PYTOK_DOT;
+ case '%': return PYTOK_PERCENT;
+ case '{': return PYTOK_LBRACE;
+ case '}': return PYTOK_RBRACE;
+ case '^': return PYTOK_CIRCUMFLEX;
+ case '~': return PYTOK_TILDE;
+ case '@': return PYTOK_AT;
+ default: return PYTOK_OP;
}
}
@@ -1144,77 +1144,77 @@ PyToken_TwoChars(int c1, int c2)
switch (c1) {
case '=':
switch (c2) {
- case '=': return EQEQUAL;
+ case '=': return PYTOK_EQEQUAL;
}
break;
case '!':
switch (c2) {
- case '=': return NOTEQUAL;
+ case '=': return PYTOK_NOTEQUAL;
}
break;
case '<':
switch (c2) {
- case '>': return NOTEQUAL;
- case '=': return LESSEQUAL;
- case '<': return LEFTSHIFT;
+ case '>': return PYTOK_NOTEQUAL;
+ case '=': return PYTOK_LESSEQUAL;
+ case '<': return PYTOK_LEFTSHIFT;
}
break;
case '>':
switch (c2) {
- case '=': return GREATEREQUAL;
- case '>': return RIGHTSHIFT;
+ case '=': return PYTOK_GREATEREQUAL;
+ case '>': return PYTOK_RIGHTSHIFT;
}
break;
case '+':
switch (c2) {
- case '=': return PLUSEQUAL;
+ case '=': return PYTOK_PLUSEQUAL;
}
break;
case '-':
switch (c2) {
- case '=': return MINEQUAL;
- case '>': return RARROW;
+ case '=': return PYTOK_MINEQUAL;
+ case '>': return PYTOK_RARROW;
}
break;
case '*':
switch (c2) {
- case '*': return DOUBLESTAR;
- case '=': return STAREQUAL;
+ case '*': return PYTOK_DOUBLESTAR;
+ case '=': return PYTOK_STAREQUAL;
}
break;
case '/':
switch (c2) {
- case '/': return DOUBLESLASH;
- case '=': return SLASHEQUAL;
+ case '/': return PYTOK_DOUBLESLASH;
+ case '=': return PYTOK_SLASHEQUAL;
}
break;
case '|':
switch (c2) {
- case '=': return VBAREQUAL;
+ case '=': return PYTOK_VBAREQUAL;
}
break;
case '%':
switch (c2) {
- case '=': return PERCENTEQUAL;
+ case '=': return PYTOK_PERCENTEQUAL;
}
break;
case '&':
switch (c2) {
- case '=': return AMPEREQUAL;
+ case '=': return PYTOK_AMPEREQUAL;
}
break;
case '^':
switch (c2) {
- case '=': return CIRCUMFLEXEQUAL;
+ case '=': return PYTOK_CIRCUMFLEXEQUAL;
}
break;
case '@':
switch (c2) {
- case '=': return ATEQUAL;
+ case '=': return PYTOK_ATEQUAL;
}
break;
}
- return OP;
+ return PYTOK_OP;
}
int
@@ -1226,7 +1226,7 @@ PyToken_ThreeChars(int c1, int c2, int c
case '<':
switch (c3) {
case '=':
- return LEFTSHIFTEQUAL;
+ return PYTOK_LEFTSHIFTEQUAL;
}
break;
}
@@ -1236,7 +1236,7 @@ PyToken_ThreeChars(int c1, int c2, int c
case '>':
switch (c3) {
case '=':
- return RIGHTSHIFTEQUAL;
+ return PYTOK_RIGHTSHIFTEQUAL;
}
break;
}
@@ -1246,7 +1246,7 @@ PyToken_ThreeChars(int c1, int c2, int c
case '*':
switch (c3) {
case '=':
- return DOUBLESTAREQUAL;
+ return PYTOK_DOUBLESTAREQUAL;
}
break;
}
@@ -1256,7 +1256,7 @@ PyToken_ThreeChars(int c1, int c2, int c
case '/':
switch (c3) {
case '=':
- return DOUBLESLASHEQUAL;
+ return PYTOK_DOUBLESLASHEQUAL;
}
break;
}
@@ -1266,17 +1266,17 @@ PyToken_ThreeChars(int c1, int c2, int c
case '.':
switch (c3) {
case '.':
- return ELLIPSIS;
+ return PYTOK_ELLIPSIS;
}
break;
}
break;
}
- return OP;
+ return PYTOK_OP;
}
static int
-indenterror(struct tok_state *tok)
+indenterror(PyTokenizer_State *tok)
{
if (tok->alterror) {
tok->done = E_TABSPACE;
@@ -1303,7 +1303,7 @@ indenterror(struct tok_state *tok)
All identifier strings are guaranteed to be "ready" unicode objects.
*/
static int
-verify_identifier(struct tok_state *tok)
+verify_identifier(PyTokenizer_State *tok)
{
PyObject *s;
int result;
@@ -1328,7 +1328,7 @@ verify_identifier(struct tok_state *tok)
/* Get next token, after space stripping etc. */
static int
-tok_get(struct tok_state *tok, char **p_start, char **p_end)
+tok_get(PyTokenizer_State *tok, char **p_start, char **p_end)
{
int c;
int blankline, nonascii;
@@ -1361,7 +1361,7 @@ tok_get(struct tok_state *tok, char **p_
if (c == '#' || c == '\n') {
/* Lines with only whitespace and/or comments
shouldn't affect the indentation and are
- not passed to the parser as NEWLINE tokens,
+ not passed to the parser as PYTOK_NEWLINE tokens,
except *totally* empty lines in interactive
mode, which signal the end of a command group. */
if (col == 0 && c == '\n' && tok->prompt != NULL)
@@ -1376,19 +1376,19 @@ tok_get(struct tok_state *tok, char **p_
/* No change */
if (altcol != tok->altindstack[tok->indent]) {
if (indenterror(tok))
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
}
else if (col > tok->indstack[tok->indent]) {
/* Indent -- always one */
- if (tok->indent+1 >= MAXINDENT) {
+ if (tok->indent+1 >= PYTOKENIZER_MAXINDENT) {
tok->done = E_TOODEEP;
tok->cur = tok->inp;
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
if (altcol <= tok->altindstack[tok->indent]) {
if (indenterror(tok))
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
tok->pendin++;
tok->indstack[++tok->indent] = col;
@@ -1404,11 +1404,11 @@ tok_get(struct tok_state *tok, char **p_
if (col != tok->indstack[tok->indent]) {
tok->done = E_DEDENT;
tok->cur = tok->inp;
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
if (altcol != tok->altindstack[tok->indent]) {
if (indenterror(tok))
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
}
}
@@ -1420,11 +1420,11 @@ tok_get(struct tok_state *tok, char **p_
if (tok->pendin != 0) {
if (tok->pendin < 0) {
tok->pendin++;
- return DEDENT;
+ return PYTOK_DEDENT;
}
else {
tok->pendin--;
- return INDENT;
+ return PYTOK_INDENT;
}
}
@@ -1445,7 +1445,7 @@ tok_get(struct tok_state *tok, char **p_
/* Check for EOF and errors now */
if (c == EOF) {
- return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
+ return tok->done == E_EOF ? PYTOK_ENDMARKER : PYTOK_ERRORTOKEN;
}
/* Identifier (most frequent token!) */
@@ -1478,11 +1478,11 @@ tok_get(struct tok_state *tok, char **p_
if (nonascii &&
!verify_identifier(tok)) {
tok->done = E_IDENTIFIER;
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
*p_start = tok->start;
*p_end = tok->cur;
- return NAME;
+ return PYTOK_NAME;
}
/* Newline */
@@ -1493,7 +1493,7 @@ tok_get(struct tok_state *tok, char **p_
*p_start = tok->start;
*p_end = tok->cur - 1; /* Leave '\n' out of the string */
tok->cont_line = 0;
- return NEWLINE;
+ return PYTOK_NEWLINE;
}
/* Period or number starting with period? */
@@ -1506,7 +1506,7 @@ tok_get(struct tok_state *tok, char **p_
if (c == '.') {
*p_start = tok->start;
*p_end = tok->cur;
- return ELLIPSIS;
+ return PYTOK_ELLIPSIS;
} else {
tok_backup(tok, c);
}
@@ -1516,7 +1516,7 @@ tok_get(struct tok_state *tok, char **p_
}
*p_start = tok->start;
*p_end = tok->cur;
- return DOT;
+ return PYTOK_DOT;
}
/* Number */
@@ -1535,7 +1535,7 @@ tok_get(struct tok_state *tok, char **p_
if (!isxdigit(c)) {
tok->done = E_TOKEN;
tok_backup(tok, c);
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
do {
c = tok_nextc(tok);
@@ -1547,7 +1547,7 @@ tok_get(struct tok_state *tok, char **p_
if (c < '0' || c >= '8') {
tok->done = E_TOKEN;
tok_backup(tok, c);
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
do {
c = tok_nextc(tok);
@@ -1559,7 +1559,7 @@ tok_get(struct tok_state *tok, char **p_
if (c != '0' && c != '1') {
tok->done = E_TOKEN;
tok_backup(tok, c);
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
do {
c = tok_nextc(tok);
@@ -1584,7 +1584,7 @@ tok_get(struct tok_state *tok, char **p_
else if (nonzero) {
tok->done = E_TOKEN;
tok_backup(tok, c);
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
}
}
@@ -1613,14 +1613,14 @@ tok_get(struct tok_state *tok, char **p_
if (!isdigit(c)) {
tok->done = E_TOKEN;
tok_backup(tok, c);
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
} else if (!isdigit(c)) {
tok_backup(tok, c);
tok_backup(tok, e);
*p_start = tok->start;
*p_end = tok->cur;
- return NUMBER;
+ return PYTOK_NUMBER;
}
do {
c = tok_nextc(tok);
@@ -1635,7 +1635,7 @@ tok_get(struct tok_state *tok, char **p_
tok_backup(tok, c);
*p_start = tok->start;
*p_end = tok->cur;
- return NUMBER;
+ return PYTOK_NUMBER;
}
letter_quote:
@@ -1666,12 +1666,12 @@ tok_get(struct tok_state *tok, char **p_
else
tok->done = E_EOLS;
tok->cur = tok->inp;
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
if (quote_size == 1 && c == '\n') {
tok->done = E_EOLS;
tok->cur = tok->inp;
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
if (c == quote)
end_quote_size += 1;
@@ -1684,7 +1684,7 @@ tok_get(struct tok_state *tok, char **p_
*p_start = tok->start;
*p_end = tok->cur;
- return STRING;
+ return PYTOK_STRING;
}
/* Line continuation */
@@ -1693,7 +1693,7 @@ tok_get(struct tok_state *tok, char **p_
if (c != '\n') {
tok->done = E_LINECONT;
tok->cur = tok->inp;
- return ERRORTOKEN;
+ return PYTOK_ERRORTOKEN;
}
tok->cont_line = 1;
goto again; /* Read next line */
@@ -1703,10 +1703,10 @@ tok_get(struct tok_state *tok, char **p_
{
int c2 = tok_nextc(tok);
int token = PyToken_TwoChars(c, c2);
- if (token != OP) {
+ if (token != PYTOK_OP) {
int c3 = tok_nextc(tok);
int token3 = PyToken_ThreeChars(c, c2, c3);
- if (token3 != OP) {
+ if (token3 != PYTOK_OP) {
token = token3;
} else {
tok_backup(tok, c3);
@@ -1739,11 +1739,11 @@ tok_get(struct tok_state *tok, char **p_
}
int
-PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
+PyTokenizer_Get(PyTokenizer_State *tok, char **p_start, char **p_end)
{
int result = tok_get(tok, p_start, p_end);
if (tok->decoding_erred) {
- result = ERRORTOKEN;
+ result = PYTOK_ERRORTOKEN;
tok->done = E_DECODE;
}
return result;
@@ -1762,7 +1762,7 @@ PyTokenizer_Get(struct tok_state *tok, c
char *
PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
{
- struct tok_state *tok;
+ PyTokenizer_State *tok;
FILE *fp;
char *p_start =NULL , *p_end =NULL , *encoding = NULL;
@@ -1823,7 +1823,7 @@ void
tok_dump(int type, char *start, char *end)
{
printf("%s", _PyParser_TokenNames[type]);
- if (type == NAME || type == NUMBER || type == STRING || type == OP)
+ if (type == PYTOK_NAME || type == PYTOK_NUMBER || type == PYTOK_STRING || type == PYTOK_OP)
printf("(%.*s)", (int)(end - start), start);
}
« no previous file with comments | « Parser/pgen.c ('k') | Parser/tokenizer.h » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+