diff -r c41c68a18bb6 Lib/idlelib/IOBinding.py --- a/Lib/idlelib/IOBinding.py Thu Sep 05 18:02:31 2013 +0300 +++ b/Lib/idlelib/IOBinding.py Sat Sep 07 17:49:25 2013 +0300 @@ -63,7 +63,7 @@ encoding = locale_encoding ### KBK 07Sep07 This is used all over IDLE, check! ### 'encoding' is used below in encode(), check! -coding_re = re.compile("coding[:=]\s*([-\w_.]+)") +coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) def coding_spec(data): """Return the encoding declaration according to PEP 263. @@ -84,14 +84,16 @@ lines = data # consider only the first two lines if '\n' in lines: - lst = lines.split('\n')[:2] + lst = lines.split('\n', 2)[:2] elif '\r' in lines: - lst = lines.split('\r')[:2] + lst = lines.split('\r', 2)[:2] else: - lst = list(lines) - str = '\n'.join(lst) - match = coding_re.search(str) - if not match: + lst = [lines] + for line in lst: + match = coding_re.match(line) + if match is not None: + break + else: return None name = match.group(1) try: diff -r c41c68a18bb6 Lib/lib2to3/pgen2/tokenize.py --- a/Lib/lib2to3/pgen2/tokenize.py Thu Sep 05 18:02:31 2013 +0300 +++ b/Lib/lib2to3/pgen2/tokenize.py Sat Sep 07 17:49:25 2013 +0300 @@ -236,7 +236,7 @@ startline = False toks_append(tokval) -cookie_re = re.compile("coding[:=]\s*([-\w.]+)") +cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" @@ -281,11 +281,10 @@ line_string = line.decode('ascii') except UnicodeDecodeError: return None - - matches = cookie_re.findall(line_string) - if not matches: + match = cookie_re.match(line_string) + if not match: return None - encoding = _get_normal_name(matches[0]) + encoding = _get_normal_name(match.group(1)) try: codec = lookup(encoding) except LookupError: diff -r c41c68a18bb6 Lib/lib2to3/tests/data/false_encoding.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/lib2to3/tests/data/false_encoding.py Sat Sep 07 17:49:25 2013 +0300 @@ -0,0 +1,2 @@ +#!/usr/bin/env python +print '#coding=0' diff -r c41c68a18bb6 Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py Thu Sep 05 18:02:31 2013 +0300 +++ b/Lib/lib2to3/tests/test_refactor.py Sat Sep 07 17:49:25 2013 +0300 @@ -271,6 +271,10 @@ fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") self.check_file_refactoring(fn) + def test_false_file_encoding(self): + fn = os.path.join(TEST_DATA_DIR, "false_encoding.py") + data = self.check_file_refactoring(fn) + def test_bom(self): fn = os.path.join(TEST_DATA_DIR, "bom.py") data = self.check_file_refactoring(fn) diff -r c41c68a18bb6 Lib/test/test_importlib/source/test_source_encoding.py --- a/Lib/test/test_importlib/source/test_source_encoding.py Thu Sep 05 18:02:31 2013 +0300 +++ b/Lib/test/test_importlib/source/test_source_encoding.py Sat Sep 07 17:49:25 2013 +0300 @@ -10,7 +10,7 @@ import unittest -CODING_RE = re.compile(r'coding[:=]\s*([-\w.]+)') +CODING_RE = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) class EncodingTest(unittest.TestCase): @@ -41,7 +41,7 @@ def create_source(self, encoding): encoding_line = "# coding={0}".format(encoding) - assert CODING_RE.search(encoding_line) + assert CODING_RE.match(encoding_line) source_lines = [encoding_line.encode('utf-8')] source_lines.append(self.source_line.encode(encoding)) return b'\n'.join(source_lines) @@ -50,7 +50,7 @@ # Make sure that an encoding that has never been a standard one for # Python works. encoding_line = "# coding=koi8-r" - assert CODING_RE.search(encoding_line) + assert CODING_RE.match(encoding_line) source = "{0}\na=42\n".format(encoding_line).encode("koi8-r") self.run_test(source) diff -r c41c68a18bb6 Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py Thu Sep 05 18:02:31 2013 +0300 +++ b/Lib/test/test_tokenize.py Sat Sep 07 17:49:25 2013 +0300 @@ -946,6 +946,13 @@ readline = self.get_readline((b'# coding: bad\n',)) self.assertRaises(SyntaxError, detect_encoding, readline) + def test_false_encoding(self): + # Issue 18873: "Encoding" detected in non-comment lines + readline = self.get_readline((b'print("#coding=fake")',)) + encoding, consumed_lines = detect_encoding(readline) + self.assertEqual(encoding, 'utf-8') + self.assertEqual(consumed_lines, [b'print("#coding=fake")']) + def test_open(self): filename = support.TESTFN + '.py' self.addCleanup(support.unlink, filename) diff -r c41c68a18bb6 Lib/tokenize.py --- a/Lib/tokenize.py Thu Sep 05 18:02:31 2013 +0300 +++ b/Lib/tokenize.py Sat Sep 07 17:49:25 2013 +0300 @@ -31,7 +31,7 @@ from codecs import lookup, BOM_UTF8 import collections from io import TextIOWrapper -cookie_re = re.compile("coding[:=]\s*([-\w.]+)") +cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) import token __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding", @@ -372,10 +372,10 @@ msg = '{} for {!r}'.format(msg, filename) raise SyntaxError(msg) - matches = cookie_re.findall(line_string) - if not matches: + match = cookie_re.match(line_string) + if not match: return None - encoding = _get_normal_name(matches[0]) + encoding = _get_normal_name(match.group(1)) try: codec = lookup(encoding) except LookupError: diff -r c41c68a18bb6 Tools/scripts/findnocoding.py --- a/Tools/scripts/findnocoding.py Thu Sep 05 18:02:31 2013 +0300 +++ b/Tools/scripts/findnocoding.py Sat Sep 07 17:49:25 2013 +0300 @@ -32,13 +32,13 @@ "no sophisticated Python source file search will be done.", file=sys.stderr) -decl_re = re.compile(rb"coding[=:]\s*([-\w.]+)") +decl_re = re.compile(rb'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') def get_declaration(line): - match = decl_re.search(line) + match = decl_re.match(line) if match: return match.group(1) - return '' + return b'' def has_correct_encoding(text, codec): try: