imp.find_module('test/badsyntax_pep3120') causes segfault

Parser/tokenizer.c

 
 /* Tokenizer implementation */
 
 #include "Python.h"
 #include "pgenheaders.h"
 
 #include <ctype.h>
 #include <assert.h>
 
 #include "tokenizer.h"
@@ skipping 1659 matching lines @@
 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
 {
     int result = tok_get(tok, p_start, p_end);
     if (tok->decoding_erred) {
         result = ERRORTOKEN;
         tok->done = E_DECODE;
     }
     return result;
 }
 
-/* Get -*- encoding -*- from a Python file.
-
-   PyTokenizer_FindEncoding returns NULL when it can't find the encoding in
-   the first or second line of the file (in which case the encoding
-   should be assumed to be PyUnicode_GetDefaultEncoding()).
-
-   The char * returned is malloc'ed via PyMem_MALLOC() and thus must be freed
-   by the caller.
-*/
 char *
-PyTokenizer_FindEncoding(int fd)
+PyTokenizer_FindEncodingFilename(int fd, const char *filename)
 {
     struct tok_state *tok;
     FILE *fp;
     char *p_start =NULL , *p_end =NULL , *encoding = NULL;
 
     fd = dup(fd);
     if (fd < 0) {
         return NULL;
     }
     fp = fdopen(fd, "r");
     if (fp == NULL) {
         return NULL;
     }
     tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
     if (tok == NULL) {
         fclose(fp);
         return NULL;
     }
+    if (filename)
+        tok->filename = filename;
+    else
+        tok->filename = "<string>";
     while (tok->lineno < 2 && tok->done == E_OK) {
         PyTokenizer_Get(tok, &p_start, &p_end);
     }
     fclose(fp);
     if (tok->encoding) {
         encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
         if (encoding)
-        strcpy(encoding, tok->encoding);
+            strcpy(encoding, tok->encoding);
     }
     PyTokenizer_Free(tok);
     return encoding;
 }
 
+char *
+PyTokenizer_FindEncoding(int fd)
+{
+    return PyTokenizer_FindEncodingFilename(fd, NULL);
+}
+
 #ifdef Py_DEBUG
 
 void
 tok_dump(int type, char *start, char *end)
 {
     printf("%s", _PyParser_TokenNames[type]);
     if (type == NAME || type == NUMBER || type == STRING || type == OP)
         printf("(%.*s)", (int)(end - start), start);
 }
 
 #endif