diff -r 829117ae2e55 Parser/tokenizer.c
--- a/Parser/tokenizer.c	Tue Jul 19 16:46:09 2016 -0500
+++ b/Parser/tokenizer.c	Thu Jul 21 23:44:19 2016 +1000
@@ -1319,6 +1319,7 @@
     if (s == NULL || PyUnicode_READY(s) == -1) {
         if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
             PyErr_Clear();
+            tok->inp = tok->cur = tok->start + 1;
             tok->done = E_IDENTIFIER;
         } else {
             tok->done = E_ERROR;
@@ -1327,8 +1328,10 @@
     }
     result = PyUnicode_IsIdentifier(s);
     Py_DECREF(s);
-    if (result == 0)
+    if (result == 0) {
+        tok->inp = tok->cur = tok->start + 1;
         tok->done = E_IDENTIFIER;
+    }
     return result;
 }
 #endif