Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(16165)

Side by Side Diff: Lib/test/test_tokenize.py

Issue 18873: "Encoding" detected in non-comment lines
Patch Set: Created 6 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Lib/test/test_importlib/source/test_source_encoding.py ('k') | Lib/tokenize.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 doctests = """ 1 doctests = """
2 Tests for the tokenize module. 2 Tests for the tokenize module.
3 3
4 The tests can be really simple. Given a small fragment of source 4 The tests can be really simple. Given a small fragment of source
5 code, print out a table with tokens. The ENDMARK is omitted for 5 code, print out a table with tokens. The ENDMARK is omitted for
6 brevity. 6 brevity.
7 7
8 >>> dump_tokens("1 + 1") 8 >>> dump_tokens("1 + 1")
9 ENCODING 'utf-8' (0, 0) (0, 0) 9 ENCODING 'utf-8' (0, 0) (0, 0)
10 NUMBER '1' (1, 0) (1, 1) 10 NUMBER '1' (1, 0) (1, 1)
(...skipping 927 matching lines...) Expand 10 before | Expand all | Expand 10 after
938 self.assertEqual(encoding, 'utf-8-sig') 938 self.assertEqual(encoding, 'utf-8-sig')
939 self.assertEqual(consumed_lines, [b'print(something)\n']) 939 self.assertEqual(consumed_lines, [b'print(something)\n'])
940 940
941 readline = self.get_readline((b'\xef\xbb\xbf',)) 941 readline = self.get_readline((b'\xef\xbb\xbf',))
942 encoding, consumed_lines = detect_encoding(readline) 942 encoding, consumed_lines = detect_encoding(readline)
943 self.assertEqual(encoding, 'utf-8-sig') 943 self.assertEqual(encoding, 'utf-8-sig')
944 self.assertEqual(consumed_lines, []) 944 self.assertEqual(consumed_lines, [])
945 945
946 readline = self.get_readline((b'# coding: bad\n',)) 946 readline = self.get_readline((b'# coding: bad\n',))
947 self.assertRaises(SyntaxError, detect_encoding, readline) 947 self.assertRaises(SyntaxError, detect_encoding, readline)
948
949 def test_false_encoding(self):
950 # Issue 18873: "Encoding" detected in non-comment lines
951 readline = self.get_readline((b'print("#coding=fake")',))
952 encoding, consumed_lines = detect_encoding(readline)
953 self.assertEqual(encoding, 'utf-8')
954 self.assertEqual(consumed_lines, [b'print("#coding=fake")'])
948 955
949 def test_open(self): 956 def test_open(self):
950 filename = support.TESTFN + '.py' 957 filename = support.TESTFN + '.py'
951 self.addCleanup(support.unlink, filename) 958 self.addCleanup(support.unlink, filename)
952 959
953 # test coding cookie 960 # test coding cookie
954 for encoding in ('iso-8859-15', 'utf-8'): 961 for encoding in ('iso-8859-15', 'utf-8'):
955 with open(filename, 'w', encoding=encoding) as fp: 962 with open(filename, 'w', encoding=encoding) as fp:
956 print("# coding: %s" % encoding, file=fp) 963 print("# coding: %s" % encoding, file=fp)
957 print("print('euro:\u20ac')", file=fp) 964 print("print('euro:\u20ac')", file=fp)
(...skipping 160 matching lines...) Expand 10 before | Expand all | Expand 10 after
1118 def test_main(): 1125 def test_main():
1119 from test import test_tokenize 1126 from test import test_tokenize
1120 support.run_doctest(test_tokenize, True) 1127 support.run_doctest(test_tokenize, True)
1121 support.run_unittest(TestTokenizerAdheresToPep0263) 1128 support.run_unittest(TestTokenizerAdheresToPep0263)
1122 support.run_unittest(Test_Tokenize) 1129 support.run_unittest(Test_Tokenize)
1123 support.run_unittest(TestDetectEncoding) 1130 support.run_unittest(TestDetectEncoding)
1124 support.run_unittest(TestTokenize) 1131 support.run_unittest(TestTokenize)
1125 1132
1126 if __name__ == "__main__": 1133 if __name__ == "__main__":
1127 test_main() 1134 test_main()
OLDNEW
« no previous file with comments | « Lib/test/test_importlib/source/test_source_encoding.py ('k') | Lib/tokenize.py » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+