Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(7)

Delta Between Two Patch Sets: Lib/token.py

Issue 3353: make built-in tokenizer available via Python C API
Left Patch Set: Created 5 years, 9 months ago
Right Patch Set: Created 4 years, 11 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Right: Side by side diff | Download
« no previous file with change/comment | « Include/tokenizer.h ('k') | Makefile.pre.in » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
(no file at all)
1 """Token constants (from "token.h").""" 1 """Token constants (from "token.h")."""
2 2
3 __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] 3 __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
4 4
5 # This file is automatically generated; please don't muck it up! 5 # This file is automatically generated; please don't muck it up!
6 # 6 #
7 # To update the symbols in this file, 'cd' to the top directory of 7 # To update the symbols in this file, 'cd' to the top directory of
8 # the python source tree after building the interpreter and run: 8 # the python source tree after building the interpreter and run:
9 # 9 #
10 # ./python Lib/token.py 10 # ./python Lib/token.py
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after
93 if len(args) > 1: 93 if len(args) > 1:
94 outFileName = args[1] 94 outFileName = args[1]
95 try: 95 try:
96 fp = open(inFileName) 96 fp = open(inFileName)
97 except OSError as err: 97 except OSError as err:
98 sys.stdout.write("I/O error: %s\n" % str(err)) 98 sys.stdout.write("I/O error: %s\n" % str(err))
99 sys.exit(1) 99 sys.exit(1)
100 with fp: 100 with fp:
101 lines = fp.read().split("\n") 101 lines = fp.read().split("\n")
102 prog = re.compile( 102 prog = re.compile(
103 "#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)", 103 "#define[ \t][ \t]*PYTOK_([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
104 re.IGNORECASE) 104 re.IGNORECASE)
105 tokens = {} 105 tokens = {}
106 for line in lines: 106 for line in lines:
107 match = prog.match(line) 107 match = prog.match(line)
108 if match: 108 if match:
109 name, val = match.group(1, 2) 109 name, val = match.group(1, 2)
110 val = int(val) 110 val = int(val)
111 tokens[val] = name # reverse so we can sort them... 111 tokens[val] = name # reverse so we can sort them...
112 keys = sorted(tokens.keys()) 112 keys = sorted(tokens.keys())
113 # load the output skeleton from the target: 113 # load the output skeleton from the target:
(...skipping 18 matching lines...) Expand all
132 fp = open(outFileName, 'w') 132 fp = open(outFileName, 'w')
133 except OSError as err: 133 except OSError as err:
134 sys.stderr.write("I/O error: %s\n" % str(err)) 134 sys.stderr.write("I/O error: %s\n" % str(err))
135 sys.exit(4) 135 sys.exit(4)
136 with fp: 136 with fp:
137 fp.write("\n".join(format)) 137 fp.write("\n".join(format))
138 138
139 139
140 if __name__ == "__main__": 140 if __name__ == "__main__":
141 _main() 141 _main()
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+