diff -r e390e60fcb20 Lib/base64.py --- a/Lib/base64.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/base64.py Sun Jun 29 22:35:22 2008 +0200 @@ -39,7 +39,7 @@ def _translate(s, altchars): return s.translate(translation) - + # Base64 encoding/decoding uses binascii def b64encode(s, altchars=None): @@ -126,7 +126,7 @@ def urlsafe_b64decode(s): return b64decode(s, b'-_') - + # Base32 encoding/decoding must be done in Python _b32alphabet = { 0: b'A', 9: b'J', 18: b'S', 27: b'3', @@ -225,7 +225,7 @@ def b32decode(s, casefold=False, map01=N # characters because this will tell us how many null bytes to remove from # the end of the decoded string. padchars = 0 - mo = re.search('(?P[=]*)$', s) + mo = re.search(b'(?P[=]*)$', s) if mo: padchars = len(mo.group('pad')) if padchars > 0: @@ -262,7 +262,7 @@ def b32decode(s, casefold=False, map01=N return b''.join(parts) - + # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. @@ -291,12 +291,12 @@ def b16decode(s, casefold=False): raise TypeError("expected bytes, not %s" % s.__class__.__name__) if casefold: s = s.upper() - if re.search('[^0-9A-F]', s): + if re.search(b'[^0-9A-F]', s): raise binascii.Error('Non-base16 digit found') return binascii.unhexlify(s) - + # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. The files should be opened in binary mode. @@ -353,7 +353,7 @@ def decodestring(s): return binascii.a2b_base64(s) - + # Usable as a script... def main(): """Small main program""" diff -r e390e60fcb20 Lib/encodings/idna.py --- a/Lib/encodings/idna.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/encodings/idna.py Sun Jun 29 22:35:22 2008 +0200 @@ -176,12 +176,10 @@ class Codec(codecs.Codec): return "", 0 # IDNA allows decoding to operate on Unicode strings, too. - if isinstance(input, bytes): - labels = dots.split(input) - else: - # Force to bytes + if not isinstance(input, bytes): + # XXX obviously wrong, see #3232 input = bytes(input) - labels = input.split(b".") + labels = input.split(b".") if labels and len(labels[-1]) == 0: trailing_dot = '.' diff -r e390e60fcb20 Lib/py_compile.py --- a/Lib/py_compile.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/py_compile.py Sun Jun 29 22:35:22 2008 +0200 @@ -86,7 +86,7 @@ def read_encoding(file, default): line = f.readline() if not line: break - m = re.match(r".*\bcoding:\s*(\S+)\b", line) + m = re.match(br".*\bcoding:\s*(\S+)\b", line) if m: return m.group(1).decode("ascii") return default diff -r e390e60fcb20 Lib/re.py --- a/Lib/re.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/re.py Sun Jun 29 22:35:22 2008 +0200 @@ -64,11 +64,18 @@ resulting RE will match the second chara \Z Matches only at the end of the string. \b Matches the empty string, but only at the start or end of a word. \B Matches the empty string, but not at the start or end of a word. - \d Matches any decimal digit; equivalent to the set [0-9]. - \D Matches any non-digit character; equivalent to the set [^0-9]. + \d Matches any decimal digit; equivalent to the set [0-9] in + bytes patterns or string patterns with the ASCII flag. + In string patterns without the ASCII flag, it will match the whole + range of Unicode digits. + \D Matches any non-digit character; equivalent to [^\d]. \s Matches any whitespace character; equivalent to [ \t\n\r\f\v]. \S Matches any non-whitespace character; equiv. to [^ \t\n\r\f\v]. - \w Matches any alphanumeric character; equivalent to [a-zA-Z0-9_]. + \w Matches any alphanumeric character; equivalent to [a-zA-Z0-9_] + in bytes patterns or string patterns with the ASCII flag. + In string patterns without the ASCII flag, it will match the + range of Unicode alphanumeric characters (letters plus digits + plus underscore). With LOCALE, it will match the set [0-9_] plus characters defined as letters for the current locale. \W Matches the complement of \w. @@ -87,6 +94,12 @@ This module exports the following functi escape Backslash all non-alphanumerics in a string. Some of the functions in this module takes flags as optional parameters: + A ASCII For string patterns, make \w, \W, \b, \B, \d, \D + match the corresponding ASCII character categories + (rather than the whole Unicode categories, which is the + default). + For bytes patterns, this flag is the only available + behaviour and needn't be specified. I IGNORECASE Perform case-insensitive matching. L LOCALE Make \w, \W, \b, \B, dependent on the current locale. M MULTILINE "^" matches the beginning of lines (after a newline) @@ -95,7 +108,8 @@ Some of the functions in this module tak as the end of the string. S DOTALL "." matches any character at all, including the newline. X VERBOSE Ignore whitespace and comments for nicer looking RE's. - U UNICODE Make \w, \W, \b, \B, dependent on the Unicode locale. + U UNICODE For compatibility only. Ignored for string patterns (it + is the default), and forbidden for bytes patterns. This module also defines an exception 'error'. @@ -107,16 +121,17 @@ import sre_parse # public symbols __all__ = [ "match", "search", "sub", "subn", "split", "findall", - "compile", "purge", "template", "escape", "I", "L", "M", "S", "X", - "U", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", + "compile", "purge", "template", "escape", "A", "I", "L", "M", "S", "X", + "U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", "UNICODE", "error" ] __version__ = "2.2.1" # flags +A = ASCII = sre_compile.SRE_FLAG_ASCII # assume ascii "locale" I = IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE # ignore case L = LOCALE = sre_compile.SRE_FLAG_LOCALE # assume current 8-bit locale -U = UNICODE = sre_compile.SRE_FLAG_UNICODE # assume unicode locale +U = UNICODE = sre_compile.SRE_FLAG_UNICODE # assume unicode "locale" M = MULTILINE = sre_compile.SRE_FLAG_MULTILINE # make anchors look for newline S = DOTALL = sre_compile.SRE_FLAG_DOTALL # make dot match newline X = VERBOSE = sre_compile.SRE_FLAG_VERBOSE # ignore whitespace and comments diff -r e390e60fcb20 Lib/sre_constants.py --- a/Lib/sre_constants.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/sre_constants.py Sun Jun 29 22:35:22 2008 +0200 @@ -207,9 +207,10 @@ SRE_FLAG_LOCALE = 4 # honour system loca SRE_FLAG_LOCALE = 4 # honour system locale SRE_FLAG_MULTILINE = 8 # treat target as multiline string SRE_FLAG_DOTALL = 16 # treat target as a single string -SRE_FLAG_UNICODE = 32 # use unicode locale +SRE_FLAG_UNICODE = 32 # use unicode "locale" SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments SRE_FLAG_DEBUG = 128 # debugging +SRE_FLAG_ASCII = 256 # use ascii "locale" # flags for INFO primitive SRE_INFO_PREFIX = 1 # has prefix diff -r e390e60fcb20 Lib/sre_parse.py --- a/Lib/sre_parse.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/sre_parse.py Sun Jun 29 22:35:22 2008 +0200 @@ -200,7 +200,7 @@ class Tokenizer: except IndexError: raise error("bogus escape (end of line)") if isinstance(self.string, bytes): - char = chr(c) + c = chr(c) char = char + c self.index = self.index + len(char) self.next = char @@ -672,6 +672,18 @@ def _parse(source, state): return subpattern +def fix_flags(src, flags): + # Check and fix flags according to the type of pattern (str or bytes) + if isinstance(src, str): + if not flags & SRE_FLAG_ASCII: + flags |= SRE_FLAG_UNICODE + elif flags & SRE_FLAG_UNICODE: + raise ValueError("ASCII and UNICODE flags are incompatible") + else: + if flags & SRE_FLAG_UNICODE: + raise ValueError("can't use UNICODE flag with a bytes pattern") + return flags + def parse(str, flags=0, pattern=None): # parse 're' pattern into list of (opcode, argument) tuples @@ -683,6 +695,7 @@ def parse(str, flags=0, pattern=None): pattern.str = str p = _parse_sub(source, pattern, 0) + p.pattern.flags = fix_flags(str, p.pattern.flags) tail = source.get() if tail == ")": diff -r e390e60fcb20 Lib/tarfile.py --- a/Lib/tarfile.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/tarfile.py Sun Jun 29 22:35:22 2008 +0200 @@ -1368,7 +1368,7 @@ class TarInfo(object): # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(r"(\d+) ([^=]+)=", re.U) + regex = re.compile(br"(\d+) ([^=]+)=") pos = 0 while True: match = regex.match(buf, pos) diff -r e390e60fcb20 Lib/test/re_tests.py --- a/Lib/test/re_tests.py Sat Jun 28 20:23:49 2008 +0200 +++ b/Lib/test/re_tests.py Sun Jun 29 22:35:22 2008 +0200 @@ -661,14 +661,10 @@ 123""", SUCCEED, 'found', 'abc'), ('^([ab]*?)(?tp_as_buffer; - if (!buffer || !buffer->bf_getbuffer || + if (!buffer || !buffer->bf_getbuffer || (*buffer->bf_getbuffer)(string, &view, PyBUF_SIMPLE) < 0) { PyErr_SetString(PyExc_TypeError, "expected string or buffer"); return NULL; @@ -1717,7 +1717,7 @@ getstring(PyObject* string, Py_ssize_t* if (PyBytes_Check(string) || bytes == size) charsize = 1; #if defined(HAVE_UNICODE) - else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) + else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) charsize = sizeof(Py_UNICODE); #endif else { @@ -1729,7 +1729,7 @@ getstring(PyObject* string, Py_ssize_t* *p_charsize = charsize; if (ptr == NULL) { - PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "Buffer is NULL"); } return ptr; @@ -1753,6 +1753,17 @@ state_init(SRE_STATE* state, PatternObje ptr = getstring(string, &length, &charsize); if (!ptr) return NULL; + + if (charsize == 1 && pattern->charsize > 1) { + PyErr_SetString(PyExc_TypeError, + "can't use a string pattern on a bytes-like object"); + return NULL; + } + if (charsize > 1 && pattern->charsize == 1) { + PyErr_SetString(PyExc_TypeError, + "can't use a bytes pattern on a string-like object"); + return NULL; + } /* adjust boundaries */ if (start < 0) @@ -2700,6 +2711,16 @@ _compile(PyObject* self_, PyObject* args return NULL; } + if (pattern == Py_None) + self->charsize = -1; + else { + Py_ssize_t p_length; + if (!getstring(pattern, &p_length, &self->charsize)) { + PyObject_DEL(self); + return NULL; + } + } + Py_INCREF(pattern); self->pattern = pattern; diff -r e390e60fcb20 Modules/sre.h --- a/Modules/sre.h Sat Jun 28 20:23:49 2008 +0200 +++ b/Modules/sre.h Sun Jun 29 22:35:22 2008 +0200 @@ -30,6 +30,7 @@ typedef struct { PyObject* pattern; /* pattern source (or None) */ int flags; /* flags used when compiling pattern source */ PyObject *weakreflist; /* List of weak references */ + int charsize; /* pattern charsize (or -1) */ /* pattern code */ Py_ssize_t codesize; SRE_CODE code[1];