diff -r c0b0dda16009 Lib/sre_compile.py --- a/Lib/sre_compile.py Fri Sep 19 15:23:30 2014 +0800 +++ b/Lib/sre_compile.py Sun Sep 21 19:17:53 2014 +0300 @@ -30,6 +30,44 @@ _SUCCESS_CODES = set([SUCCESS, FAILURE]) _ASSERT_CODES = set([ASSERT, ASSERT_NOT]) +_equivalences = ( + # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I + #(0x69, 0x131), # iı + # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S + (0x73, 0x17f), # sſ + # MICRO SIGN, GREEK SMALL LETTER MU + (0xb5, 0x3bc), # µμ + # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI + (0x345, 0x3b9, 0x1fbe), # \u0345ιι + # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + (0x390, 0x1fd3), # ΐΐ + # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + (0x3b0, 0x1fe3), # ΰΰ + # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL + (0x3b2, 0x3d0), # βϐ + # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL + (0x3b5, 0x3f5), # εϵ + # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL + (0x3b8, 0x3d1), # θϑ + # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL + (0x3ba, 0x3f0), # κϰ + # GREEK SMALL LETTER PI, GREEK PI SYMBOL + (0x3c0, 0x3d6), # πϖ + # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL + (0x3c1, 0x3f1), # ρϱ + # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA + (0x3c2, 0x3c3), # ςσ + # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL + (0x3c6, 0x3d5), # φϕ + # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE + (0x1e61, 0x1e9b), # ṡẛ + # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST + (0xfb05, 0xfb06), # ſtst +) + +_ignorecase_fixes = {i: tuple(j for j in t if i != j) + for t in _equivalences for i in t} + def _compile(code, pattern, flags): # internal: compile a (sub)pattern emit = code.append @@ -41,21 +79,38 @@ for op, av in pattern: if op in LITERAL_CODES: if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - emit(_sre.getlower(av, flags)) + lo = _sre.getlower(av, flags) + if (lo in _ignorecase_fixes and + not (flags & SRE_FLAG_LOCALE) and + flags & SRE_FLAG_UNICODE): + emit(OPCODES[IN_IGNORE]) + skip = _len(code); emit(0) + if op is NOT_LITERAL: + emit(OPCODES[NEGATE]) + for k in (lo,) + _ignorecase_fixes[lo]: + emit(OPCODES[LITERAL]) + emit(k) + emit(OPCODES[FAILURE]) + code[skip] = _len(code) - skip + else: + emit(OPCODES[OP_IGNORE[op]]) + emit(lo) else: emit(OPCODES[op]) emit(av) elif op is IN: + fixes = None if flags & SRE_FLAG_IGNORECASE: emit(OPCODES[OP_IGNORE[op]]) def fixup(literal, flags=flags): return _sre.getlower(literal, flags) + if not (flags & SRE_FLAG_LOCALE) and flags & SRE_FLAG_UNICODE: + fixes = _ignorecase_fixes else: emit(OPCODES[op]) fixup = _identityfunction skip = _len(code); emit(0) - _compile_charset(av, flags, code, fixup) + _compile_charset(av, flags, code, fixup, fixes) code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: @@ -169,12 +224,12 @@ else: raise ValueError("unsupported operand type", op) -def _compile_charset(charset, flags, code, fixup=None): +def _compile_charset(charset, flags, code, fixup=None, fixes=None): # compile charset subprogram emit = code.append if fixup is None: fixup = _identityfunction - for op, av in _optimize_charset(charset, fixup): + for op, av in _optimize_charset(charset, fixup, fixes): emit(OPCODES[op]) if op is NEGATE: pass @@ -198,7 +253,7 @@ raise error("internal: unsupported set operator") emit(OPCODES[FAILURE]) -def _optimize_charset(charset, fixup): +def _optimize_charset(charset, fixup, fixes): # internal: optimize character set out = [] tail = [] @@ -207,10 +262,24 @@ while True: try: if op is LITERAL: - charmap[fixup(av)] = 1 + if fixup is not _identityfunction: + lo = fixup(av) + charmap[lo] = 1 + if fixes and lo in fixes: + for k in fixes[lo]: + charmap[k] = 1 + else: + charmap[av] = 1 elif op is RANGE: - for i in range(fixup(av[0]), fixup(av[1])+1): - charmap[i] = 1 + if fixup is not _identityfunction: + for i in map(fixup, range(av[0], av[1]+1)): + charmap[i] = 1 + if fixes and i in fixes: + for k in fixes[i]: + charmap[k] = 1 + else: + for i in range(av[0], av[1]+1): + charmap[i] = 1 elif op is NEGATE: out.append((op, av)) else: diff -r c0b0dda16009 Lib/test/test_re.py --- a/Lib/test/test_re.py Fri Sep 19 15:23:30 2014 +0800 +++ b/Lib/test/test_re.py Sun Sep 21 19:17:53 2014 +0300 @@ -582,6 +582,29 @@ self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") + self.assertTrue(re.match(r"ΜΜΜ ΒΒΒ", "Μµμ Ββϐ", re.I)) + self.assertTrue(re.match(r"µμ βϐ", "ΜΜ ΒΒ", re.I)) + self.assertTrue(re.match(r"µμ βϐ", "μµ ϐβ", re.I)) + self.assertTrue(re.match(r"SSS ṠṠṠ", "Ssſ Ṡṡẛ", re.I)) + self.assertTrue(re.match(r"sſ ṡẛ", "SS ṠṠ", re.I)) + self.assertTrue(re.match(r"sſ ṡẛ ſtst", "ſs ẛṡ stſt", re.I)) + self.assertTrue(re.match(r"Ι{3}", "\u0345ιι", re.I)) + self.assertTrue(re.match(r"\u0345ιι", "ΙΙΙ", re.I)) + self.assertTrue(re.match(r"(\u0345ιι){2}", "ι\u0345ιιι\u0345", re.I)) + + def test_ignore_case_set(self): + self.assertTrue(re.match(r"[19Μ]{3} [19Β]{3}", "Μµμ Ββϐ", re.I)) + self.assertTrue(re.match(r"[19µ][19μ] [19β][19ϐ]", "ΜΜ ΒΒ", re.I)) + self.assertTrue(re.match(r"[19µ][19μ] [19β][19ϐ]", "μµ ϐβ", re.I)) + self.assertTrue(re.match(r"[19S]{3} [19Ṡ]{3}", "Ssſ Ṡṡẛ", re.I)) + self.assertTrue(re.match(r"[19s][19ſ] [19ṡ][19ẛ]", "SS ṠṠ", re.I)) + self.assertTrue(re.match(r"[19s][19ſ] [19ṡ][19ẛ] [19st][19st]", + "ſs ẛṡ stſt", re.I)) + self.assertTrue(re.match(r"[19Ι]{3}", "\u0345ιι", re.I)) + self.assertTrue(re.match(r"[19\u0345][19ι][19ι]", "ΙΙΙ", re.I)) + self.assertTrue(re.match(r"([19\u0345][19ι][19ι]){2}", + "ι\u0345ιιι\u0345", re.I)) + def test_category(self): self.assertEqual(re.match(r"(\s)", " ").group(1), " ")