import ast import tokenize import io source = '"""\n"""\n' # bug appears also with indentation, eg. 'if True:\n """\n """\n' root = ast.parse(source) # some ast nodes have -1 in col_offset field print("\nNODES:") for node in ast.walk(root): if "lineno" in node._attributes: print(node, node.lineno, node.col_offset) # no token has -1 in column field print("\nTOKENS:") for tok in tokenize.tokenize(io.BytesIO(source.encode('utf-8')).readline): print(tok)