Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1)

Side by Side Diff: Lib/test/test_tokenize.py

Issue 23840: tokenize.open() leaks an open binary file on TextIOWrapper error
Patch Set: Created 4 years, 9 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | Lib/tokenize.py » ('j') | Lib/tokenize.py » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 doctests = """ 1 doctests = """
2 Tests for the tokenize module. 2 Tests for the tokenize module.
3 3
4 The tests can be really simple. Given a small fragment of source 4 The tests can be really simple. Given a small fragment of source
5 code, print out a table with tokens. The ENDMARKER is omitted for 5 code, print out a table with tokens. The ENDMARKER is omitted for
6 brevity. 6 brevity.
7 7
8 >>> dump_tokens("1 + 1") 8 >>> dump_tokens("1 + 1")
9 ENCODING 'utf-8' (0, 0) (0, 0) 9 ENCODING 'utf-8' (0, 0) (0, 0)
10 NUMBER '1' (1, 0) (1, 1) 10 NUMBER '1' (1, 0) (1, 1)
(...skipping 630 matching lines...) Expand 10 before | Expand all | Expand 10 after
641 NAME 'grün' (2, 0) (2, 4) 641 NAME 'grün' (2, 0) (2, 4)
642 OP '=' (2, 5) (2, 6) 642 OP '=' (2, 5) (2, 6)
643 STRING "U'green'" (2, 7) (2, 15) 643 STRING "U'green'" (2, 7) (2, 15)
644 """ 644 """
645 645
646 from test import support 646 from test import support
647 from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, 647 from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
648 STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, 648 STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
649 open as tokenize_open, Untokenizer) 649 open as tokenize_open, Untokenizer)
650 from io import BytesIO 650 from io import BytesIO
651 from unittest import TestCase 651 from unittest import TestCase, mock
652 import os, sys, glob 652 import os, sys, glob
653 import token 653 import token
654 654
655 def dump_tokens(s): 655 def dump_tokens(s):
656 """Print out the tokens in s in a table format. 656 """Print out the tokens in s in a table format.
657 657
658 The ENDMARKER is omitted. 658 The ENDMARKER is omitted.
659 """ 659 """
660 f = BytesIO(s.encode('utf-8')) 660 f = BytesIO(s.encode('utf-8'))
661 for type, token, start, end, line in tokenize(f.readline): 661 for type, token, start, end, line in tokenize(f.readline):
(...skipping 362 matching lines...) Expand 10 before | Expand all | Expand 10 after
1024 with tokenize_open(filename) as fp: 1024 with tokenize_open(filename) as fp:
1025 self.assertEqual(fp.encoding, encoding) 1025 self.assertEqual(fp.encoding, encoding)
1026 self.assertEqual(fp.mode, 'r') 1026 self.assertEqual(fp.mode, 'r')
1027 1027
1028 # test BOM (no coding cookie) 1028 # test BOM (no coding cookie)
1029 with open(filename, 'w', encoding='utf-8-sig') as fp: 1029 with open(filename, 'w', encoding='utf-8-sig') as fp:
1030 print("print('euro:\u20ac')", file=fp) 1030 print("print('euro:\u20ac')", file=fp)
1031 with tokenize_open(filename) as fp: 1031 with tokenize_open(filename) as fp:
1032 self.assertEqual(fp.encoding, 'utf-8-sig') 1032 self.assertEqual(fp.encoding, 'utf-8-sig')
1033 self.assertEqual(fp.mode, 'r') 1033 self.assertEqual(fp.mode, 'r')
1034
1035 # test close file of unrecognized encoding
haypo 2015/04/02 00:08:21 You should put the new test in a new function.
1036 m = mock.mock_open(read_data=b'#coding:xxx')
1037 with mock.patch('tokenize._builtin_open', m):
1038 try:
1039 tokenize_open(filename)
1040 except:
1041 pass
1042 self.assertTrue(m().close.called)
1034 1043
1035 def test_filename_in_exception(self): 1044 def test_filename_in_exception(self):
1036 # When possible, include the file name in the exception. 1045 # When possible, include the file name in the exception.
1037 path = 'some_file_path' 1046 path = 'some_file_path'
1038 lines = ( 1047 lines = (
1039 b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S 1048 b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S
1040 ) 1049 )
1041 class Bunk: 1050 class Bunk:
1042 def __init__(self, lines, path): 1051 def __init__(self, lines, path):
1043 self.name = path 1052 self.name = path
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after
1228 from test import test_tokenize 1237 from test import test_tokenize
1229 support.run_doctest(test_tokenize, True) 1238 support.run_doctest(test_tokenize, True)
1230 support.run_unittest(TestTokenizerAdheresToPep0263) 1239 support.run_unittest(TestTokenizerAdheresToPep0263)
1231 support.run_unittest(Test_Tokenize) 1240 support.run_unittest(Test_Tokenize)
1232 support.run_unittest(TestDetectEncoding) 1241 support.run_unittest(TestDetectEncoding)
1233 support.run_unittest(TestTokenize) 1242 support.run_unittest(TestTokenize)
1234 support.run_unittest(UntokenizeTest) 1243 support.run_unittest(UntokenizeTest)
1235 1244
1236 if __name__ == "__main__": 1245 if __name__ == "__main__":
1237 test_main() 1246 test_main()
OLDNEW
« no previous file with comments | « no previous file | Lib/tokenize.py » ('j') | Lib/tokenize.py » ('J')

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+