Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(6)

Delta Between Two Patch Sets: Include/token.h

Issue 3353: make built-in tokenizer available via Python C API
Left Patch Set: Created 4 years, 10 months ago
Right Patch Set: Created 4 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « Include/Python.h ('k') | Include/tokenizer.h » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 1
2 /* Token types */ 2 /* Token types */
3 #ifndef Py_LIMITED_API 3 #ifndef Py_LIMITED_API
4 #ifndef Py_TOKEN_H 4 #ifndef Py_TOKEN_H
5 #define Py_TOKEN_H 5 #define Py_TOKEN_H
6 #ifdef __cplusplus 6 #ifdef __cplusplus
7 extern "C" { 7 extern "C" {
8 #endif 8 #endif
9 9
10 #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioc tl.h */ 10 #undef PYTOK_TILDE /* Prevent clash of our definition with system macro. Ex AI X, ioctl.h */
11 11
12 #define ENDMARKER» 0 12 #define PYTOK_ENDMARKER»0
13 #define NAME» » 1 13 #define PYTOK_NAME» » 1
14 #define NUMBER» » 2 14 #define PYTOK_NUMBER» » 2
15 #define STRING» » 3 15 #define PYTOK_STRING» » 3
16 #define NEWLINE»» 4 16 #define PYTOK_NEWLINE» » 4
17 #define INDENT» » 5 17 #define PYTOK_INDENT» » 5
18 #define DEDENT» » 6 18 #define PYTOK_DEDENT» » 6
19 #define LPAR» » 7 19 #define PYTOK_LPAR» » 7
20 #define RPAR» » 8 20 #define PYTOK_RPAR» » 8
21 #define LSQB» » 9 21 #define PYTOK_LSQB» » 9
22 #define RSQB» » 10 22 #define PYTOK_RSQB» » 10
23 #define COLON» » 11 23 #define PYTOK_COLON» » 11
24 #define COMMA» » 12 24 #define PYTOK_COMMA» » 12
25 #define SEMI» » 13 25 #define PYTOK_SEMI» » 13
26 #define PLUS» » 14 26 #define PYTOK_PLUS» » 14
27 #define MINUS» » 15 27 #define PYTOK_MINUS» » 15
28 #define STAR» » 16 28 #define PYTOK_STAR» » 16
29 #define SLASH» » 17 29 #define PYTOK_SLASH» » 17
30 #define VBAR» » 18 30 #define PYTOK_VBAR» » 18
31 #define AMPER» » 19 31 #define PYTOK_AMPER» » 19
32 #define LESS» » 20 32 #define PYTOK_LESS» » 20
33 #define GREATER»» 21 33 #define PYTOK_GREATER» » 21
34 #define EQUAL» » 22 34 #define PYTOK_EQUAL» » 22
35 #define DOT» » 23 35 #define PYTOK_DOT» » 23
36 #define PERCENT»» 24 36 #define PYTOK_PERCENT» » 24
37 #define LBRACE» » 25 37 #define PYTOK_LBRACE» » 25
38 #define RBRACE» » 26 38 #define PYTOK_RBRACE» » 26
39 #define EQEQUAL»» 27 39 #define PYTOK_EQEQUAL» » 27
40 #define NOTEQUAL» 28 40 #define PYTOK_NOTEQUAL» 28
41 #define LESSEQUAL» 29 41 #define PYTOK_LESSEQUAL»29
42 #define GREATEREQUAL» 30 42 #define PYTOK_GREATEREQUAL» 30
43 #define TILDE» » 31 43 #define PYTOK_TILDE» » 31
44 #define CIRCUMFLEX» 32 44 #define PYTOK_CIRCUMFLEX» 32
45 #define LEFTSHIFT» 33 45 #define PYTOK_LEFTSHIFT»33
46 #define RIGHTSHIFT» 34 46 #define PYTOK_RIGHTSHIFT» 34
47 #define DOUBLESTAR» 35 47 #define PYTOK_DOUBLESTAR» 35
48 #define PLUSEQUAL» 36 48 #define PYTOK_PLUSEQUAL»36
49 #define MINEQUAL» 37 49 #define PYTOK_MINEQUAL» 37
50 #define STAREQUAL» 38 50 #define PYTOK_STAREQUAL»38
51 #define SLASHEQUAL» 39 51 #define PYTOK_SLASHEQUAL» 39
52 #define PERCENTEQUAL» 40 52 #define PYTOK_PERCENTEQUAL» 40
53 #define AMPEREQUAL» 41 53 #define PYTOK_AMPEREQUAL» 41
54 #define VBAREQUAL» 42 54 #define PYTOK_VBAREQUAL»42
55 #define CIRCUMFLEXEQUAL»43 55 #define PYTOK_CIRCUMFLEXEQUAL» 43
56 #define LEFTSHIFTEQUAL» 44 56 #define PYTOK_LEFTSHIFTEQUAL» 44
57 #define RIGHTSHIFTEQUAL»45 57 #define PYTOK_RIGHTSHIFTEQUAL» 45
58 #define DOUBLESTAREQUAL»46 58 #define PYTOK_DOUBLESTAREQUAL» 46
59 #define DOUBLESLASH» 47 59 #define PYTOK_DOUBLESLASH» 47
60 #define DOUBLESLASHEQUAL 48 60 #define PYTOK_DOUBLESLASHEQUAL 48
61 #define AT 49 61 #define PYTOK_AT 49
62 #define ATEQUAL»» 50 62 #define PYTOK_ATEQUAL» » 50
63 #define RARROW 51 63 #define PYTOK_RARROW 51
64 #define ELLIPSIS 52 64 #define PYTOK_ELLIPSIS 52
65 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */ 65 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
66 #define OP» » 53 66 #define PYTOK_OP» » 53
67 #define ERRORTOKEN» 54 67 #define PYTOK_ERRORTOKEN» 54
68 #define N_TOKENS» 55 68 #define PYTOK_N_TOKENS» 55
69 69
70 /* Special definitions for cooperation with parser */ 70 /* Special definitions for cooperation with parser */
71 71
72 #define NT_OFFSET» » 256 72 #define PYTOK_NT_OFFSET»» 256
73 73
74 #define ISTERMINAL(x)» » ((x) < NT_OFFSET) 74 #define PYTOK_ISTERMINAL(x)» » ((x) < PYTOK_NT_OFFSET)
75 #define ISNONTERMINAL(x)» ((x) >= NT_OFFSET) 75 #define PYTOK_ISNONTERMINAL(x)» ((x) >= PYTOK_NT_OFFSET)
76 #define ISEOF(x)» » ((x) == ENDMARKER) 76 #define PYTOK_ISEOF(x)» » ((x) == PYTOK_ENDMARKER)
77 77
78 78
79 PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */ 79 PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */
80 PyAPI_FUNC(int) PyToken_OneChar(int); 80 PyAPI_FUNC(int) PyToken_OneChar(int);
81 PyAPI_FUNC(int) PyToken_TwoChars(int, int); 81 PyAPI_FUNC(int) PyToken_TwoChars(int, int);
82 PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int); 82 PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
83 83
84 #ifdef __cplusplus 84 #ifdef __cplusplus
85 } 85 }
86 #endif 86 #endif
87 #endif /* !Py_TOKEN_H */ 87 #endif /* !Py_TOKEN_H */
88 #endif /* Py_LIMITED_API */ 88 #endif /* Py_LIMITED_API */
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+