diff -r 15f46b850257 Doc/library/re.rst --- a/Doc/library/re.rst Sun Jan 18 17:40:17 2015 +0100 +++ b/Doc/library/re.rst Sun Jan 18 18:51:50 2015 +0200 @@ -626,17 +626,34 @@ form. That way, separator components are always found at the same relative indices within the result list. - Note that *split* will never split a string on an empty pattern match. - For example: + .. note:: - >>> re.split('x*', 'foo') - ['foo'] - >>> re.split("(?m)^$", "foo\n\nbar\n") - ['foo\n\nbar\n'] + *split* will never split a string on an empty pattern match. + For example: + + >>> re.split('x*', 'foo') + ['foo'] + + This behavior will changed in future. For unambiguity use patterns + which never match an empty string (e.g. ``'x+'`` instead of ``'x*'``). + + Patterns which can match only empty strings are rejected. + + >>> re.split("(?m)^$", "foo\n\nbar\n") + Traceback (most recent call last): + File "", line 1, in + ... + ValueError: split() won't split a string on empty pattern match. .. versionchanged:: 3.1 Added the optional flags argument. + .. versionchanged:: 3.5 + Rejected patterns which can match only empty strings. + + .. deprecated-removed:: 3.5 3.7 + Splitting on a pattern which could match an empty string now raises + a deprecation warning. .. function:: findall(pattern, string, flags=0) diff -r 15f46b850257 Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst Sun Jan 18 17:40:17 2015 +0100 +++ b/Doc/whatsnew/3.5.rst Sun Jan 18 18:51:50 2015 +0200 @@ -470,6 +470,13 @@ Changes in the Python API simply define :meth:`~importlib.machinery.Loader.create_module` to return ``None`` (:issue:`23014`). +* :func:`re.split` now raises a deprecation warning if the pattern could match + an empty string. For now re.split() will never split a string on an empty + pattern match, but this will changed in future. For compatibility use + patterns which couldn't match an empty string (e.g. ``'x+'`` instead of + ``'x*'``). Patterns which could match only an empty string (e.g. ``\b``) + never worked as expected and now are rejected at all. + Changes in the C API -------------------- diff -r 15f46b850257 Lib/sre_compile.py --- a/Lib/sre_compile.py Sun Jan 18 17:40:17 2015 +0100 +++ b/Lib/sre_compile.py Sun Jan 18 18:51:50 2015 +0200 @@ -414,8 +414,11 @@ def _compile_info(code, pattern, flags): # this contains min/max pattern width, and an optional literal # prefix or a character map lo, hi = pattern.getwidth() + if hi > MAXCODE: + hi = MAXCODE if lo == 0: - return # not worth it + code.extend([INFO, 4, 0, lo, hi]) + return # look for a literal prefix prefix = [] prefixappend = prefix.append @@ -495,10 +498,7 @@ def _compile_info(code, pattern, flags): else: emit(MAXCODE) prefix = prefix[:MAXCODE] - if hi < MAXCODE: - emit(hi) - else: - emit(0) + emit(min(hi, MAXCODE)) # add literal prefix if prefix: emit(len(prefix)) # length diff -r 15f46b850257 Lib/test/test_re.py --- a/Lib/test/test_re.py Sun Jan 18 17:40:17 2015 +0100 +++ b/Lib/test/test_re.py Sun Jan 18 18:51:50 2015 +0200 @@ -251,28 +251,28 @@ class ReTests(unittest.TestCase): for string in ":a:b::c", S(":a:b::c"): self.assertTypedEqual(re.split(":", string), ['', 'a', 'b', '', 'c']) - self.assertTypedEqual(re.split(":*", string), + self.assertTypedEqual(re.split(":+", string), ['', 'a', 'b', 'c']) - self.assertTypedEqual(re.split("(:*)", string), + self.assertTypedEqual(re.split("(:+)", string), ['', ':', 'a', ':', 'b', '::', 'c']) for string in (b":a:b::c", B(b":a:b::c"), bytearray(b":a:b::c"), memoryview(b":a:b::c")): self.assertTypedEqual(re.split(b":", string), [b'', b'a', b'b', b'', b'c']) - self.assertTypedEqual(re.split(b":*", string), + self.assertTypedEqual(re.split(b":+", string), [b'', b'a', b'b', b'c']) - self.assertTypedEqual(re.split(b"(:*)", string), + self.assertTypedEqual(re.split(b"(:+)", string), [b'', b':', b'a', b':', b'b', b'::', b'c']) for a, b, c in ("\xe0\xdf\xe7", "\u0430\u0431\u0432", "\U0001d49c\U0001d49e\U0001d4b5"): string = ":%s:%s::%s" % (a, b, c) self.assertEqual(re.split(":", string), ['', a, b, '', c]) - self.assertEqual(re.split(":*", string), ['', a, b, c]) - self.assertEqual(re.split("(:*)", string), + self.assertEqual(re.split(":+", string), ['', a, b, c]) + self.assertEqual(re.split("(:+)", string), ['', ':', a, ':', b, '::', c]) - self.assertEqual(re.split("(?::*)", ":a:b::c"), ['', 'a', 'b', 'c']) - self.assertEqual(re.split("(:)*", ":a:b::c"), + self.assertEqual(re.split("(?::+)", ":a:b::c"), ['', 'a', 'b', 'c']) + self.assertEqual(re.split("(:)+", ":a:b::c"), ['', ':', 'a', ':', 'b', ':', 'c']) self.assertEqual(re.split("([b:]+)", ":a:b::c"), ['', ':', 'a', ':b::', 'c']) @@ -282,13 +282,34 @@ class ReTests(unittest.TestCase): self.assertEqual(re.split("(?:b)|(?::+)", ":a:b::c"), ['', 'a', '', '', 'c']) + for sep, expected in [ + (':*', ['', 'a', 'b', 'c']), + ('(?::*)', ['', 'a', 'b', 'c']), + ('(:*)', ['', ':', 'a', ':', 'b', '::', 'c']), + ('(:)*', ['', ':', 'a', ':', 'b', ':', 'c']), + ]: + with self.subTest(sep=sep), self.assertWarns(DeprecationWarning): + self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) + + for sep, expected in [ + ('', [':a:b::c']), + (r'\b', [':a:b::c']), + (r'(?=:)', [':a:b::c']), + (r'(?<=:)', [':a:b::c']), + ]: + with self.subTest(sep=sep), self.assertRaises(ValueError): + self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) + def test_qualified_re_split(self): self.assertEqual(re.split(":", ":a:b::c", maxsplit=2), ['', 'a', 'b::c']) self.assertEqual(re.split(':', 'a:b:c:d', maxsplit=2), ['a', 'b', 'c:d']) self.assertEqual(re.split("(:)", ":a:b::c", maxsplit=2), ['', ':', 'a', ':', 'b::c']) - self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), + self.assertEqual(re.split("(:+)", ":a:b::c", maxsplit=2), ['', ':', 'a', ':', 'b::c']) + with self.assertWarns(DeprecationWarning): + self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), + ['', ':', 'a', ':', 'b::c']) def test_re_findall(self): self.assertEqual(re.findall(":+", "abc"), []) diff -r 15f46b850257 Modules/_sre.c --- a/Modules/_sre.c Sun Jan 18 17:40:17 2015 +0100 +++ b/Modules/_sre.c Sun Jan 18 18:51:50 2015 +0200 @@ -863,6 +863,19 @@ pattern_split(PatternObject* self, PyObj if (!string) return NULL; + assert(self->codesize != 0); + if (self->code[0] != SRE_OP_INFO || self->code[3] == 0) { + if (self->code[0] == SRE_OP_INFO && self->code[4] == 0) { + PyErr_SetString(PyExc_ValueError, + "split() won't split a string on empty pattern match."); + return NULL; + } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "split() won't split a string on empty pattern match.", + 1) < 0) + return NULL; + } + string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX); if (!string) return NULL;