diff -r 63c3a3f28b42 Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py	Sat Oct 06 18:26:40 2012 -0400
+++ b/Lib/test/test_tokenize.py	Sat Oct 06 20:46:23 2012 -0400
@@ -550,6 +550,10 @@
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+
+Pathological whitespace (http://bugs.python.org/issue16152)
+    >>> dump_tokens("@          ")
+    OP         '@'           (1, 0) (1, 1)
 """
 
 
diff -r 63c3a3f28b42 Lib/tokenize.py
--- a/Lib/tokenize.py	Sat Oct 06 18:26:40 2012 -0400
+++ b/Lib/tokenize.py	Sat Oct 06 20:46:23 2012 -0400
@@ -95,7 +95,7 @@
                 group("'", r'\\\r?\n'),
                 r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
 tokenprog, pseudoprog, single3prog, double3prog = map(
@@ -362,6 +362,8 @@
             if pseudomatch:                                # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
+                if start == end:
+                    continue
                 token, initial = line[start:end], line[start]
 
                 if initial in numchars or \