diff -r faae99459b43 Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py	Sat Oct 06 18:05:14 2012 +0100
+++ b/Lib/test/test_tokenize.py	Sat Oct 06 20:46:13 2012 -0400
@@ -1109,6 +1109,10 @@
                                   token.NAME, token.AMPER, token.NUMBER,
                                   token.RPAR)
 
+    def test_pathological_trailing_whitespace(self):
+        # See http://bugs.python.org/issue16152
+        self.assertExactTypeEqual('@'+' '*10, token.AT)
+
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}
 
 def test_main():
diff -r faae99459b43 Lib/tokenize.py
--- a/Lib/tokenize.py	Sat Oct 06 18:05:14 2012 +0100
+++ b/Lib/tokenize.py	Sat Oct 06 20:46:13 2012 -0400
@@ -162,7 +162,7 @@
                 group("'", r'\\\r?\n'),
                 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
 def _compile(expr):
@@ -555,6 +555,8 @@
             if pseudomatch:                                # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
+                if start == end:
+                    continue
                 token, initial = line[start:end], line[start]
 
                 if (initial in numchars or                  # ordinary number