| OLD | NEW |
| 1 #ifndef Py_TOKENIZER_H | 1 #ifndef Py_TOKENIZER_H |
| 2 #define Py_TOKENIZER_H | 2 #define Py_TOKENIZER_H |
| 3 #ifdef __cplusplus | 3 #ifdef __cplusplus |
| 4 extern "C" { | 4 extern "C" { |
| 5 #endif | 5 #endif |
| 6 | 6 |
| 7 #include "object.h" | 7 #include "object.h" |
| 8 | 8 |
| 9 /* Tokenizer interface */ | 9 /* Tokenizer interface */ |
| 10 | 10 |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 62 }; | 62 }; |
| 63 | 63 |
| 64 extern struct tok_state *PyTokenizer_FromString(const char *, int); | 64 extern struct tok_state *PyTokenizer_FromString(const char *, int); |
| 65 extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); | 65 extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); |
| 66 extern struct tok_state *PyTokenizer_FromFile(FILE *, char*, | 66 extern struct tok_state *PyTokenizer_FromFile(FILE *, char*, |
| 67 char *, char *); | 67 char *, char *); |
| 68 extern void PyTokenizer_Free(struct tok_state *); | 68 extern void PyTokenizer_Free(struct tok_state *); |
| 69 extern int PyTokenizer_Get(struct tok_state *, char **, char **); | 69 extern int PyTokenizer_Get(struct tok_state *, char **, char **); |
| 70 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, | 70 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, |
| 71 int len, int *offset); | 71 int len, int *offset); |
| 72 extern char * PyTokenizer_FindEncoding(int); | 72 |
| 73 /* Get the encoding of a Python file. Check for the coding cookie and check if |
| 74 the file starts with a BOM. |
| 75 |
| 76 PyTokenizer_FindEncodingFilename() returns NULL when it can't find the |
| 77 encoding in the first or second line of the file (in which case the encoding |
| 78 should be assumed to be UTF-8). |
| 79 |
| 80 The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed |
| 81 by the caller. |
| 82 */ |
| 83 extern char* PyTokenizer_FindEncodingFilename(int fd, const char* filename); |
| 84 extern char* PyTokenizer_FindEncoding(int fd); |
| 73 | 85 |
| 74 #ifdef __cplusplus | 86 #ifdef __cplusplus |
| 75 } | 87 } |
| 76 #endif | 88 #endif |
| 77 #endif /* !Py_TOKENIZER_H */ | 89 #endif /* !Py_TOKENIZER_H */ |
| OLD | NEW |