diff -r 99ef4501205b Lib/test/test_pep263.py --- a/Lib/test/test_pep263.py Thu May 10 16:11:12 2012 +0100 +++ b/Lib/test/test_pep263.py Wed May 16 08:49:48 2012 +0200 @@ -2,6 +2,7 @@ import unittest from test import support +from test.script_helper import assert_python_failure class PEP263Test(unittest.TestCase): @@ -55,6 +56,23 @@ class PEP263Test(unittest.TestCase): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') + def test_too_long_line(self): + # BUFSIZ at the best the maximum buffer size of any platform + # (e.g. Linux uses 8192) + BUFSIZ = 8192 + code = '#' + repr('\u00A1' * BUFSIZ) + '\n' + filename = 'test.py' + self.addCleanup(support.unlink, filename) + with open(filename, 'w') as f: + f.write(code) + returncode, stdout, stderr = assert_python_failure(filename) + self.assertTrue( + stderr.startswith( + b'File "test.py", line 1\n' + b'SyntaxError: Line 1 of file test.py is longer than the internal buffer'), + "stderr=%a" % stderr) + + def test_main(): support.run_unittest(PEP263Test) diff -r 99ef4501205b Parser/tokenizer.c --- a/Parser/tokenizer.c Thu May 10 16:11:12 2012 +0100 +++ b/Parser/tokenizer.c Wed May 16 08:49:48 2012 +0200 @@ -546,6 +546,7 @@ static char * decoding_fgets(char *s, int size, struct tok_state *tok) { char *line = NULL; + size_t len; int badchar = 0; for (;;) { if (tok->decoding_state == STATE_NORMAL) { @@ -557,6 +558,15 @@ decoding_fgets(char *s, int size, struct /* We want a 'raw' read. */ line = Py_UniversalNewlineFgets(s, size, tok->fp, NULL); + if (line != NULL) { + len = strlen(line); + if (1 < len && line[len-1] != '\n') { + PyErr_Format(PyExc_SyntaxError, + "Line %i of file %U is longer than the internal buffer (%i)", + tok->lineno + 1, tok->filename, size); + return error_ret(tok); + } + } break; } else { /* We have not yet determined the encoding.