Index: Lib/re.py =================================================================== --- re.py Tue Feb 05 18:31:37 2008 +++ re.py Mon Sep 08 16:28:07 2008 @@ -36,8 +36,10 @@ "+" Matches 1 or more (greedy) repetitions of the preceding RE. "?" Matches 0 or 1 (greedy) of the preceding RE. *?,+?,?? Non-greedy versions of the previous three special characters. + *+,++,?+ Possessive versions of the previous three special characters. {m,n} Matches from m to n repetitions of the preceding RE. {m,n}? Non-greedy version of the above. + {m,n}+ Possessive version of the above. "\\" Either escapes special characters or signals a special sequence. [] Indicates a set of characters. A "^" as the first character indicates a complementing set. @@ -45,10 +47,12 @@ (...) Matches the RE inside the parentheses. The contents can be retrieved or matched later in the string. (?iLmsux) Set the I, L, M, S, U, or X flag for the RE (see below). - (?:...) Non-grouping version of regular parentheses. + (?:...) Non-capturing version of regular parentheses. (?P...) The substring matched by the group is accessible by name. (?P=name) Matches the text matched earlier by the group named name. (?#...) A comment; ignored. + (?>...) Atomic group. Like (?:...) but won't retry the RE within the + parentheses. (?=...) Matches if ... matches next, but doesn't consume the string. (?!...) Matches if ... doesn't match next. (?<=...) Matches if preceded by ... (must be fixed length). @@ -111,7 +115,7 @@ "U", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", "UNICODE", "error" ] -__version__ = "2.2.1" +__version__ = "2.2.2" # flags I = IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE # ignore case Index: Lib/sre_compile.py =================================================================== --- sre_compile.py Thu Jan 03 20:08:15 2008 +++ sre_compile.py Tue Sep 09 20:38:04 2008 @@ -17,266 +17,414 @@ assert _sre.MAGIC == MAGIC, "SRE module mismatch" if _sre.CODESIZE == 2: - MAXCODE = 65535 + MAXCODE = 0xFFFF + _subchunk_size = 16 else: MAXCODE = 0xFFFFFFFFL + _subchunk_size = 32 +_subchunk_mask = (1 << _subchunk_size) - 1 -def _identityfunction(x): - return x +ASSERT_CODES = { + ASSERT: END_ASSERT, + ASSERT_NOT: END_ASSERT_NOT, +} + +CATEGORY_CODES = [ + (ANY, ANY, ANY), + (BOUNDARY, LOC_BOUNDARY, UNI_BOUNDARY), + (DIGIT, DIGIT, UNI_DIGIT), + (NOT_BOUNDARY, LOC_NOT_BOUNDARY, UNI_NOT_BOUNDARY), + (NOT_DIGIT, NOT_DIGIT, UNI_NOT_DIGIT), + (NOT_WHITESPACE, NOT_WHITESPACE, UNI_NOT_WHITESPACE), + (NOT_WORD, LOC_NOT_WORD, UNI_NOT_WORD), + (WHITESPACE, WHITESPACE, UNI_WHITESPACE), + (WORD, LOC_WORD, UNI_WORD), +] + +IGNORE_CODES = [ + (BIGCHARSET, BIGCHARSET_IGNORE), + (CHARSET, CHARSET_IGNORE), + (GROUPREF, GROUPREF_IGNORE), + (IN, IN_IGNORE), + (LITERAL, LITERAL_IGNORE), + (LITERAL_STRING, LITERAL_STRING_IGNORE), + (NOT_BIGCHARSET, NOT_BIGCHARSET_IGNORE), + (NOT_CHARSET, NOT_CHARSET_IGNORE), + (NOT_IN, NOT_IN_IGNORE), + (NOT_LITERAL, NOT_LITERAL_IGNORE), + (NOT_RANGE, NOT_RANGE_IGNORE), + (RANGE, RANGE_IGNORE), +] + +LINE_CODES = [ + (END_OF_LINE, END_OF_STRING_2), + (END_OF_STRING, END_OF_STRING), + (START_OF_LINE, START_OF_STRING), + (START_OF_STRING, START_OF_STRING), +] + +REPEAT_CODES = { + REPEAT_MAX: END_REPEAT_MAX, + REPEAT_MIN: END_REPEAT_MIN, + REPEAT_POSS: END_REPEAT_POSS, +} + +SINGLE_CHARACTER_CODES = set([ + ANY, + BIGCHARSET, + CHARSET, + DIGIT, + IN, + LITERAL, + NOT_BIGCHARSET, + NOT_CHARSET, + NOT_DIGIT, + NOT_IN, + NOT_LITERAL, + NOT_RANGE, + NOT_WHITESPACE, + NOT_WORD, + RANGE, + WHITESPACE, + WORD, +]) + +REVERSE_CODES = [ + (ANY, ANY_REV), + (ANY_ALL, ANY_ALL_REV), + (ASSERT, ASSERT), + (ASSERT_NOT, ASSERT_NOT), + (ATOMIC, ATOMIC), + (BIGCHARSET, BIGCHARSET_REV), + (BIGCHARSET_IGNORE, BIGCHARSET_IGNORE_REV), + (BOUNDARY, BOUNDARY), + (BRANCH, BRANCH), + (CALL, CALL), + (CHARSET, CHARSET_REV), + (CHARSET_IGNORE, CHARSET_IGNORE_REV), + (DIGIT, DIGIT_REV), + (END_ASSERT, END_ASSERT), + (END_ASSERT_NOT, END_ASSERT_NOT), + (END_ATOMIC, END_ATOMIC), + (END_OF_LINE, END_OF_LINE), + (END_OF_STRING, END_OF_STRING), + (END_OF_STRING_2, END_OF_STRING_2), + (END_REPEAT_MAX, END_REPEAT_MAX_REV), + (END_REPEAT_MIN, END_REPEAT_MIN_REV), + (END_REPEAT_ONE_MAX, END_REPEAT_ONE_MAX_REV), + (END_REPEAT_ONE_MIN, END_REPEAT_ONE_MIN_REV), + (END_REPEAT_ONE_POSS, END_REPEAT_ONE_POSS_REV), + (END_REPEAT_POSS, END_REPEAT_POSS_REV), + (FAILURE, FAILURE), + (GROUPREF, GROUPREF_REV), + (GROUPREF_EXISTS, GROUPREF_EXISTS), + (GROUPREF_IGNORE, GROUPREF_IGNORE_REV), + (IN, IN_REV), + (IN_IGNORE, IN_IGNORE_REV), + (JUMP, JUMP), + (LITERAL, LITERAL_REV), + (LITERAL_IGNORE, LITERAL_IGNORE_REV), + (LITERAL_STRING, LITERAL_STRING_REV), + (LITERAL_STRING_IGNORE, LITERAL_STRING_IGNORE_REV), + (LOC_BOUNDARY, LOC_BOUNDARY), + (LOC_NOT_BOUNDARY, LOC_NOT_BOUNDARY), + (LOC_NOT_WORD, LOC_NOT_WORD_REV), + (LOC_WORD, LOC_WORD_REV), + (MARK, MARK), + (NOT_BIGCHARSET, NOT_BIGCHARSET_REV), + (NOT_BIGCHARSET_IGNORE, NOT_BIGCHARSET_IGNORE_REV), + (NOT_BOUNDARY, NOT_BOUNDARY), + (NOT_CHARSET, NOT_CHARSET_REV), + (NOT_CHARSET_IGNORE, NOT_CHARSET_IGNORE_REV), + (NOT_DIGIT, NOT_DIGIT_REV), + (NOT_IN, NOT_IN_REV), + (NOT_IN_IGNORE, NOT_IN_IGNORE_REV), + (NOT_LITERAL, NOT_LITERAL_REV), + (NOT_LITERAL_IGNORE, NOT_LITERAL_IGNORE_REV), + (NOT_RANGE, NOT_RANGE_REV), + (NOT_RANGE_IGNORE, NOT_RANGE_IGNORE_REV), + (NOT_WHITESPACE, NOT_WHITESPACE_REV), + (NOT_WORD, NOT_WORD_REV), + (RANGE, RANGE_REV), + (RANGE_IGNORE, RANGE_IGNORE_REV), + (REPEAT_MAX, REPEAT_MAX_REV), + (REPEAT_MIN, REPEAT_MIN_REV), + (REPEAT_ONE_MAX, REPEAT_ONE_MAX_REV), + (REPEAT_ONE_MIN, REPEAT_ONE_MIN_REV), + (REPEAT_ONE_POSS, REPEAT_ONE_POSS_REV), + (REPEAT_POSS, REPEAT_POSS_REV), + (START_OF_LINE, START_OF_LINE), + (START_OF_STRING, START_OF_STRING), + (SUBPATTERN, SUBPATTERN), + (SUCCESS, SUCCESS), + (UNI_BOUNDARY, UNI_BOUNDARY), + (UNI_DIGIT, UNI_DIGIT_REV), + (UNI_NOT_BOUNDARY, UNI_NOT_BOUNDARY), + (UNI_NOT_DIGIT, UNI_NOT_DIGIT_REV), + (UNI_NOT_WHITESPACE, UNI_NOT_WHITESPACE_REV), + (UNI_NOT_WORD, UNI_NOT_WORD_REV), + (UNI_WHITESPACE, UNI_WHITESPACE_REV), + (UNI_WORD, UNI_WORD_REV), + (WHITESPACE, WHITESPACE_REV), + (WORD, WORD_REV), +] + +REPEAT_ONE_CODES = { + REPEAT_MAX: REPEAT_ONE_MAX, + REPEAT_MIN: REPEAT_ONE_MIN, + REPEAT_POSS: REPEAT_ONE_POSS, +} + +BIGCHARSET_SET = set([BIGCHARSET, NOT_BIGCHARSET]) +CATEGORY_SET = set([op for op, loc_op, uni_op in CATEGORY_CODES]) +CHARSET_SET = set([CHARSET, NOT_CHARSET]) +IN_SET = set([IN, NOT_IN]) +LINE_SET = set([END_OF_LINE, END_OF_STRING, START_OF_LINE, START_OF_STRING]) +LITERAL_SET = set([LITERAL, NOT_LITERAL]) +RANGE_SET = set([NOT_RANGE, RANGE]) +REPEAT_SET = set([REPEAT_MAX, REPEAT_MIN, REPEAT_POSS]) -def set(seq): - s = {} - for elem in seq: - s[elem] = 1 - return s - -_LITERAL_CODES = set([LITERAL, NOT_LITERAL]) -_REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT]) -_SUCCESS_CODES = set([SUCCESS, FAILURE]) -_ASSERT_CODES = set([ASSERT, ASSERT_NOT]) - -def _compile(code, pattern, flags): +def _compile(code, pattern, flags, info, dir=1): # internal: compile a (sub)pattern emit = code.append - _len = len - LITERAL_CODES = _LITERAL_CODES - REPEATING_CODES = _REPEATING_CODES - SUCCESS_CODES = _SUCCESS_CODES - ASSERT_CODES = _ASSERT_CODES + literal_string = [] + fix_op = {} + if flags & SRE_FLAG_LOCALE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = loc_op + elif flags & SRE_FLAG_UNICODE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = uni_op + else: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = op + if flags & SRE_FLAG_DOTALL: + fix_op[ANY] = ANY_ALL + else: + fix_op[ANY] = ANY + if flags & SRE_FLAG_IGNORECASE: + def fix_case(ch): + return _sre.getlower(ch, flags) + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op_ignore + else: + def fix_case(ch): + return ch + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op + if flags & SRE_FLAG_MULTILINE: + for line_op, op in LINE_CODES: + fix_op[line_op] = line_op + else: + for line_op, op in LINE_CODES: + fix_op[line_op] = op + if dir < 0: + fix_direction = dict((op, op_rev) for op, op_rev in REVERSE_CODES) + else: + fix_direction = dict((op, op) for op, op_rev in REVERSE_CODES) + if dir < 0: + # Within lookbehind, so reverse the order of the matching. + pattern = reversed(pattern) for op, av in pattern: - if op in LITERAL_CODES: - if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - emit(_sre.getlower(av, flags)) - else: + if op in IN_SET: + op, av = _optimize_in(op, av, flags) + if op is LITERAL: + literal_string.append(av) + else: + if literal_string: + if dir < 0: + literal_string = literal_string[::-1] + emit_literal_string(emit, literal_string, fix_case, fix_op, fix_direction) + literal_string = [] + if op in ASSERT_CODES: emit(OPCODES[op]) - emit(av) - elif op is IN: - if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - def fixup(literal, flags=flags): - return _sre.getlower(literal, flags) - else: + skip = len(code); emit(0) + _compile(code, av[1], flags, info, av[0]) + emit(OPCODES[ASSERT_CODES[op]]) + code[skip] = len(code) - skip + elif op in BIGCHARSET_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + _compile_bigcharset(av, flags, code) + elif op is BRANCH: emit(OPCODES[op]) - fixup = _identityfunction - skip = _len(code); emit(0) - _compile_charset(av, flags, code, fixup) - code[skip] = _len(code) - skip - elif op is ANY: - if flags & SRE_FLAG_DOTALL: - emit(OPCODES[ANY_ALL]) - else: - emit(OPCODES[ANY]) - elif op in REPEATING_CODES: - if flags & SRE_FLAG_TEMPLATE: - raise error, "internal: unsupported template operator" - emit(OPCODES[REPEAT]) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) + tail = [] + tailappend = tail.append + for av in av[1]: + skip = len(code); emit(0) + _compile(code, av, flags, info, dir) + emit(OPCODES[JUMP]) + tailappend(len(code)); emit(0) + code[skip] = len(code) - skip + emit(0) # end of branchs + for tail in tail: + code[tail] = len(code) - tail + elif op is CALL: + emit(OPCODES[op]) + skip = len(code); emit(0) + _compile(code, av, flags, info, dir) emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - elif _simple(av) and op is not REPEAT: - if op is MAX_REPEAT: - emit(OPCODES[REPEAT_ONE]) + code[skip] = len(code) - skip + elif op in CHARSET_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + _compile_charset(av, flags, code) + elif op in CATEGORY_SET or op in LINE_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + elif op is GROUPREF: + emit(OPCODES[fix_direction[fix_op[op]]]) + emit(av - 1) + elif op is GROUPREF_EXISTS: + emit(OPCODES[op]) + emit(av[0] - 1) + skipyes = len(code); emit(0) + _compile(code, av[1], flags, info, dir) + if av[2]: + emit(OPCODES[JUMP]) + skipno = len(code); emit(0) + code[skipyes] = len(code) - skipyes + 1 + _compile(code, av[2], flags, info, dir) + code[skipno] = len(code) - skipno else: - emit(OPCODES[MIN_REPEAT_ONE]) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) - emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - else: - emit(OPCODES[REPEAT]) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) - code[skip] = _len(code) - skip - if op is MAX_REPEAT: - emit(OPCODES[MAX_UNTIL]) + code[skipyes] = len(code) - skipyes + 1 + elif op in IN_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + _compile_in(av, flags, code, fix_direction) + elif op in LITERAL_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + emit(fix_case(av)) + elif op in RANGE_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + emit(fix_case(av[0])) + emit(fix_case(av[1])) + elif op in REPEAT_SET: + if flags & SRE_FLAG_TEMPLATE: + raise error, "internal: unsupported template operator" else: - emit(OPCODES[MIN_UNTIL]) - elif op is SUBPATTERN: - if av[0]: - emit(OPCODES[MARK]) - emit((av[0]-1)*2) - # _compile_info(code, av[1], flags) - _compile(code, av[1], flags) - if av[0]: - emit(OPCODES[MARK]) - emit((av[0]-1)*2+1) - elif op in SUCCESS_CODES: - emit(OPCODES[op]) - elif op in ASSERT_CODES: - emit(OPCODES[op]) - skip = _len(code); emit(0) - if av[0] >= 0: - emit(0) # look ahead - else: - lo, hi = av[1].getwidth() - if lo != hi: - raise error, "look-behind requires fixed-width pattern" - emit(lo) # look behind - _compile(code, av[1], flags) - emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - elif op is CALL: - emit(OPCODES[op]) - skip = _len(code); emit(0) - _compile(code, av, flags) - emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - elif op is AT: - emit(OPCODES[op]) - if flags & SRE_FLAG_MULTILINE: - av = AT_MULTILINE.get(av, av) - if flags & SRE_FLAG_LOCALE: - av = AT_LOCALE.get(av, av) - elif flags & SRE_FLAG_UNICODE: - av = AT_UNICODE.get(av, av) - emit(ATCODES[av]) - elif op is BRANCH: - emit(OPCODES[op]) - tail = [] - tailappend = tail.append - for av in av[1]: - skip = _len(code); emit(0) - # _compile_info(code, av, flags) - _compile(code, av, flags) - emit(OPCODES[JUMP]) - tailappend(_len(code)); emit(0) - code[skip] = _len(code) - skip - emit(0) # end of branch - for tail in tail: - code[tail] = _len(code) - tail - elif op is CATEGORY: - emit(OPCODES[op]) - if flags & SRE_FLAG_LOCALE: - av = CH_LOCALE[av] - elif flags & SRE_FLAG_UNICODE: - av = CH_UNICODE[av] - emit(CHCODES[av]) - elif op is GROUPREF: - if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - else: - emit(OPCODES[op]) - emit(av-1) - elif op is GROUPREF_EXISTS: - emit(OPCODES[op]) - emit(av[0]-1) - skipyes = _len(code); emit(0) - _compile(code, av[1], flags) - if av[2]: - emit(OPCODES[JUMP]) - skipno = _len(code); emit(0) - code[skipyes] = _len(code) - skipyes + 1 - _compile(code, av[2], flags) - code[skipno] = _len(code) - skipno + single = get_single_character(av[2]) + if single: + emit(OPCODES[fix_direction[REPEAT_ONE_CODES[op]]]) + skip = len(code); emit(0) + emit(info.repeat_count) + info.repeat_count += 1 + emit(av[0]) + emit(av[1]) + _compile(code, single, flags, info, dir) + code[skip] = len(code) - skip + else: + emit(OPCODES[fix_direction[op]]) + skip = len(code); emit(0) + emit(info.repeat_count) + info.repeat_count += 1 + emit(av[0]) + emit(av[1]) + _compile(code, av[2], flags, info, dir) + emit(OPCODES[fix_direction[REPEAT_CODES[op]]]) + offset = len(code) - skip + code[skip] = offset + emit(offset) + elif op is SUBPATTERN: + if av[0]: + group_index = info.group_count + info.group_count += 1 + emit(OPCODES[MARK]) + emit(group_index * 2) + _compile(code, av[1], flags, info, dir) + if av[0]: + emit(OPCODES[MARK]) + emit(group_index * 2 + 1) + elif op is ATOMIC: + emit(OPCODES[ATOMIC]) + skip = len(code); emit(0) + _compile(code, av[1], flags, info, dir) + emit(OPCODES[END_ATOMIC]) + code[skip] = len(code) - skip else: - code[skipyes] = _len(code) - skipyes + 1 - else: - raise ValueError, ("unsupported operand type", op) + raise ValueError, ("unsupported operand type", op) + if literal_string: + if dir < 0: + literal_string = literal_string[::-1] + emit_literal_string(emit, literal_string, fix_case, fix_op, fix_direction) + +def emit_literal_string(emit, literal_string, fix_case, fix_op, fix_direction): + if len(literal_string) > 1: + emit(OPCODES[fix_direction[fix_op[LITERAL_STRING]]]) + emit(len(literal_string)) + for ch in literal_string: + emit(fix_case(ch)) + else: + emit(OPCODES[fix_direction[fix_op[LITERAL]]]) + emit(fix_case(literal_string[0])) + +def get_single_character(pattern): + if len(pattern) == 1 and pattern[0][0] in SINGLE_CHARACTER_CODES: + return pattern + return None -def _compile_charset(charset, flags, code, fixup=None): +def _compile_in(charset, flags, code, fix_direction): # compile charset subprogram emit = code.append - if fixup is None: - fixup = _identityfunction - for op, av in _optimize_charset(charset, fixup): - emit(OPCODES[op]) - if op is NEGATE: - pass + fix_op = {} + if flags & SRE_FLAG_LOCALE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = loc_op + elif flags & SRE_FLAG_UNICODE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = uni_op + else: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = op + if flags & SRE_FLAG_IGNORECASE: + def fix_case(ch): + return _sre.getlower(ch, flags) + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op_ignore + else: + def fix_case(ch): + return ch + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op + skip = len(code); emit(0) + for op, av in charset: + if op is BIGCHARSET: + emit(OPCODES[fix_direction[op]]) + _compile_bigcharset(av, flags, code) + elif op in CATEGORY_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + elif op is CHARSET: + emit(OPCODES[fix_direction[op]]) + _compile_charset(av, flags, code) elif op is LITERAL: - emit(fixup(av)) + emit(OPCODES[fix_direction[op]]) + emit(fix_case(av)) elif op is RANGE: - emit(fixup(av[0])) - emit(fixup(av[1])) - elif op is CHARSET: - code.extend(av) - elif op is BIGCHARSET: - code.extend(av) - elif op is CATEGORY: - if flags & SRE_FLAG_LOCALE: - emit(CHCODES[CH_LOCALE[av]]) - elif flags & SRE_FLAG_UNICODE: - emit(CHCODES[CH_UNICODE[av]]) - else: - emit(CHCODES[av]) + emit(OPCODES[fix_direction[op]]) + emit(fix_case(av[0])) + emit(fix_case(av[1])) else: - raise error, "internal: unsupported set operator" - emit(OPCODES[FAILURE]) + raise error, "internal: unsupported charset operator" + code[skip] = len(code) - skip -def _optimize_charset(charset, fixup): - # internal: optimize character set - out = [] - outappend = out.append - charmap = [0]*256 - try: - for op, av in charset: - if op is NEGATE: - outappend((op, av)) - elif op is LITERAL: - charmap[fixup(av)] = 1 - elif op is RANGE: - for i in range(fixup(av[0]), fixup(av[1])+1): - charmap[i] = 1 - elif op is CATEGORY: - # XXX: could append to charmap tail - return charset # cannot compress - except IndexError: - # character set contains unicode characters - return _optimize_unicode(charset, fixup) - # compress character map - i = p = n = 0 - runs = [] - runsappend = runs.append - for c in charmap: - if c: - if n == 0: - p = i - n = n + 1 - elif n: - runsappend((p, n)) - n = 0 - i = i + 1 - if n: - runsappend((p, n)) - if len(runs) <= 2: - # use literal/range - for p, n in runs: - if n == 1: - outappend((LITERAL, p)) - else: - outappend((RANGE, (p, p+n-1))) - if len(out) < len(charset): - return out - else: - # use bitmap - data = _mk_bitmap(charmap) - outappend((CHARSET, data)) - return out - return charset - -def _mk_bitmap(bits): - data = [] - dataappend = data.append - if _sre.CODESIZE == 2: - start = (1, 0) - else: - start = (1L, 0L) - m, v = start - for c in bits: - if c: - v = v + m - m = m + m - if m > MAXCODE: - dataappend(v) - m, v = start - return data +def _compile_charset(av, flags, code): + emit = code.append + for i in range(256 / _subchunk_size): + emit(av & _subchunk_mask) + av >>= _subchunk_size + +def _compile_bigcharset(av, flags, code): + emit = code.append + skip = len(code); emit(0) + bits, shift = 0, 0 + for index in av[0]: + bits |= index << shift; shift += 8 + if shift == _subchunk_size: + emit(bits) + bits, shift = 0, 0 + for chunk in av[1]: + for i in range(256 / _subchunk_size): + emit(chunk & _subchunk_mask) + chunk >>= _subchunk_size + code[skip] = len(code) - skip + +# The characters may be mapped to a bitmap. # To represent a big charset, first a bitmap of all characters in the # set is constructed. Then, this bitmap is sliced into chunks of 256 @@ -304,171 +452,94 @@ # in particular, that negated charsets cannot be represented as # bigcharsets. -def _optimize_unicode(charset, fixup): - try: - import array - except ImportError: - return charset - charmap = [0]*65536 - negate = 0 - try: - for op, av in charset: - if op is NEGATE: - negate = 1 - elif op is LITERAL: - charmap[fixup(av)] = 1 - elif op is RANGE: - for i in xrange(fixup(av[0]), fixup(av[1])+1): - charmap[i] = 1 - elif op is CATEGORY: - # XXX: could expand category - return charset # cannot compress - except IndexError: - # non-BMP characters - return charset - if negate: - if sys.maxunicode != 65535: - # XXX: negation does not work with big charsets - return charset - for i in xrange(65536): - charmap[i] = not charmap[i] - comps = {} - mapping = [0]*256 - block = 0 - data = [] - for i in xrange(256): - chunk = tuple(charmap[i*256:(i+1)*256]) - new = comps.setdefault(chunk, block) - mapping[i] = new - if new == block: - block = block + 1 - data = data + _mk_bitmap(chunk) - header = [block] - if _sre.CODESIZE == 2: - code = 'H' - else: - code = 'I' - # Convert block indices to byte array of 256 bytes - mapping = array.array('b', mapping).tostring() - # Convert byte array to word array - mapping = array.array(code, mapping) - assert mapping.itemsize == _sre.CODESIZE - header = header + mapping.tolist() - data[0:0] = header - return [(BIGCHARSET, data)] - -def _simple(av): - # check if av is a "simple" operator - lo, hi = av[2].getwidth() - if lo == 0 and hi == MAXREPEAT: - raise error, "nothing to repeat" - return lo == hi == 1 and av[2][0][0] != SUBPATTERN - -def _compile_info(code, pattern, flags): - # internal: compile an info block. in the current version, - # this contains min/max pattern width, and an optional literal - # prefix or a character map - lo, hi = pattern.getwidth() - if lo == 0: - return # not worth it - # look for a literal prefix - prefix = [] - prefixappend = prefix.append - prefix_skip = 0 - charset = [] # not used - charsetappend = charset.append - if not (flags & SRE_FLAG_IGNORECASE): - # look for literal prefix - for op, av in pattern.data: - if op is LITERAL: - if len(prefix) == prefix_skip: - prefix_skip = prefix_skip + 1 - prefixappend(av) - elif op is SUBPATTERN and len(av[1]) == 1: - op, av = av[1][0] - if op is LITERAL: - prefixappend(av) - else: - break - else: - break - # if no prefix, look for charset prefix - if not prefix and pattern.data: - op, av = pattern.data[0] - if op is SUBPATTERN and av[1]: - op, av = av[1][0] - if op is LITERAL: - charsetappend((op, av)) - elif op is BRANCH: - c = [] - cappend = c.append - for p in av[1]: - if not p: - break - op, av = p[0] - if op is LITERAL: - cappend((op, av)) - else: - break - else: - charset = c - elif op is BRANCH: - c = [] - cappend = c.append - for p in av[1]: - if not p: - break - op, av = p[0] - if op is LITERAL: - cappend((op, av)) - else: - break - else: - charset = c - elif op is IN: - charset = av -## if prefix: -## print "*** PREFIX", prefix, prefix_skip -## if charset: -## print "*** CHARSET", charset - # add an info block - emit = code.append - emit(OPCODES[INFO]) - skip = len(code); emit(0) - # literal flag - mask = 0 - if prefix: - mask = SRE_INFO_PREFIX - if len(prefix) == prefix_skip == len(pattern.data): - mask = mask + SRE_INFO_LITERAL - elif charset: - mask = mask + SRE_INFO_CHARSET - emit(mask) - # pattern length - if lo < MAXCODE: - emit(lo) +def _ones(n): + return (1 << n) - 1 + +NOT_CODES = [ + (BIGCHARSET, NOT_BIGCHARSET), + (CHARSET, NOT_CHARSET), + (DIGIT, NOT_DIGIT), + (LITERAL, NOT_LITERAL), + (RANGE, NOT_RANGE), + (WHITESPACE, NOT_WHITESPACE), + (WORD, NOT_WORD), +] + +def _optimize_in(op, charset, flags): + # internal: optimize character set + if flags & SRE_FLAG_IGNORECASE: + def fix_case(ch): + return _sre.getlower(ch, flags) else: - emit(MAXCODE) - prefix = prefix[:MAXCODE] - if hi < MAXCODE: - emit(hi) + def fix_case(ch): + return ch + fix_op = {} + if op is IN: + for o, not_o in NOT_CODES: + fix_op[o] = o + fix_op[not_o] = not_o else: - emit(0) - # add literal prefix - if prefix: - emit(len(prefix)) # length - emit(prefix_skip) # skip - code.extend(prefix) - # generate overlap table - table = [-1] + ([0]*len(prefix)) - for i in xrange(len(prefix)): - table[i+1] = table[i]+1 - while table[i+1] > 0 and prefix[i] != prefix[table[i+1]-1]: - table[i+1] = table[table[i+1]-1]+1 - code.extend(table[1:]) # don't store first entry - elif charset: - _compile_charset(charset, flags, code) - code[skip] = len(code) - skip + for o, not_o in NOT_CODES: + fix_op[o] = not_o + fix_op[not_o] = o + # consolidate the ranges + ranges = [] + categories = [] + for o, a in charset: + if o is LITERAL: + c = fix_case(a) + ranges.append((c, c)) + elif o is RANGE: + r = iter(xrange(a[0], a[1] + 1)) + start = fix_case(r.next()) + end = start + for c in r: + c = fix_case(c) + if c - end != 1: + ranges.append((start, end)) + start = c + end = c + ranges.append((start, end)) + else: + categories.append((o, a)) + ranges.sort() + for i in range(len(ranges) - 2, -1, -1): + if ranges[i + 1][0] - ranges[i][1] <= 1: + ranges[i : i + 2] = [(ranges[i][0], max(ranges[i][1], ranges[i + 1][1]))] + # try to optimise + if len(ranges) == 1: + for r in ranges: + if r[0] == r[1]: + categories.append((LITERAL, r[0])) + else: + categories.append((RANGE, r)) + elif len(ranges) > 1: + if ranges[-1][1] <= 255: + # small (bytestring) charset + bitmap = 0 + for r in ranges: + bitmap |= _ones(r[1] + 1) ^ _ones(r[0]) + categories.append((CHARSET, bitmap)) + else: + # big (unicode) charset + bitmap = 0 + for r in ranges: + bitmap |= _ones(r[1] + 1) ^ _ones(r[0]) + chunk_mask = _ones(256) + index_table = [] + reverse_index = {} + for high_byte in range(256): + chunk = bitmap & chunk_mask + index = reverse_index.setdefault(chunk, len(reverse_index)) + index_table.append(index) + bitmap >>= 256 + chunks = [(index, chunk) for chunk, index in reverse_index.items()] + chunks.sort() + categories.append((BIGCHARSET, (index_table, [chunk for index, chunk in chunks]))) + if len(categories) == 1: + o, a = categories[0] + return (fix_op[o], a) + return (op, categories) try: unicode @@ -484,17 +555,23 @@ return 0 def _code(p, flags): - flags = p.pattern.flags | flags code = [] # compile info block - _compile_info(code, p, flags) + #_compile_info(code, p, flags) # compile the pattern - _compile(code, p.data, flags) + class Record(object): + pass + info = Record() + info.group_count = 0 + info.repeat_count = 0 + _compile(code, p.data, flags, info) code.append(OPCODES[SUCCESS]) + + code = [info.group_count * 2, info.repeat_count, len(code)] + code return code Index: Lib/sre_constants.py =================================================================== --- sre_constants.py Wed Aug 25 03:22:30 2004 +++ sre_constants.py Mon Sep 08 16:30:22 2008 @@ -17,7 +17,7 @@ # max code word in this release -MAXREPEAT = 65535 +MAX_REPEATS = 65535 # SRE standard exception (access as sre.error) # should this really be here? @@ -25,123 +25,270 @@ class error(Exception): pass -# operators - +# codes FAILURE = "failure" SUCCESS = "success" ANY = "any" ANY_ALL = "any_all" +ANY_ALL_REV = "any_all_rev" +ANY_REV = "any_rev" ASSERT = "assert" ASSERT_NOT = "assert_not" -AT = "at" +ATOMIC = "atomic" BIGCHARSET = "bigcharset" +BIGCHARSET_IGNORE = "bigcharset_ignore" +BIGCHARSET_IGNORE_REV = "bigcharset_ignore_rev" +BIGCHARSET_REV = "bigcharset_rev" +BOUNDARY = "boundary" BRANCH = "branch" CALL = "call" -CATEGORY = "category" CHARSET = "charset" +CHARSET_IGNORE = "charset_ignore" +CHARSET_IGNORE_REV = "charset_ignore_rev" +CHARSET_REV = "charset_rev" +DIGIT = "digit" +DIGIT_REV = "digit_rev" +END_ASSERT = "end_assert" +END_ASSERT_NOT = "end_assert_not" +END_ATOMIC = "end_atomic" +END_OF_LINE = "end_of_line" +END_OF_STRING = "end_of_string" +END_OF_STRING_2 = "end_of_string_2" +END_REPEAT_MAX = "end_repeat_max" +END_REPEAT_MAX_REV = "end_repeat_max_rev" +END_REPEAT_MIN = "end_repeat_min" +END_REPEAT_MIN_REV = "end_repeat_min_rev" +END_REPEAT_ONE_MAX = "end_repeat_one_max" +END_REPEAT_ONE_MAX_REV = "end_repeat_one_max_rev" +END_REPEAT_ONE_MIN = "end_repeat_one_min" +END_REPEAT_ONE_MIN_REV = "end_repeat_one_min_rev" +END_REPEAT_ONE_POSS = "end_repeat_one_poss" +END_REPEAT_ONE_POSS_REV = "end_repeat_one_poss_rev" +END_REPEAT_POSS = "end_repeat_poss" +END_REPEAT_POSS_REV = "end_repeat_poss_rev" GROUPREF = "groupref" -GROUPREF_IGNORE = "groupref_ignore" GROUPREF_EXISTS = "groupref_exists" +GROUPREF_IGNORE = "groupref_ignore" +GROUPREF_IGNORE_REV = "groupref_ignore_rev" +GROUPREF_REV = "groupref_rev" IN = "in" IN_IGNORE = "in_ignore" -INFO = "info" +IN_IGNORE_REV = "in_ignore_rev" +IN_REV = "in_rev" JUMP = "jump" LITERAL = "literal" LITERAL_IGNORE = "literal_ignore" +LITERAL_IGNORE_REV = "literal_ignore_rev" +LITERAL_REV = "literal_rev" +LITERAL_STRING = "literal_string" +LITERAL_STRING_IGNORE = "literal_string_ignore" +LITERAL_STRING_IGNORE_REV = "literal_string_ignore_rev" +LITERAL_STRING_REV = "literal_string_rev" +LOC_BOUNDARY = "loc_boundary" +LOC_NOT_BOUNDARY = "loc_not_boundary" +LOC_NOT_WORD = "loc_not_word" +LOC_NOT_WORD_REV = "loc_not_word_rev" +LOC_WORD = "loc_word" +LOC_WORD_REV = "loc_word_rev" MARK = "mark" -MAX_REPEAT = "max_repeat" -MAX_UNTIL = "max_until" -MIN_REPEAT = "min_repeat" -MIN_UNTIL = "min_until" -NEGATE = "negate" +NOT_BIGCHARSET = "not_bigcharset" +NOT_BIGCHARSET_IGNORE = "not_bigcharset_ignore" +NOT_BIGCHARSET_IGNORE_REV = "not_bigcharset_ignore_rev" +NOT_BIGCHARSET_REV = "not_bigcharset_rev" +NOT_BOUNDARY = "not_boundary" +NOT_CHARSET = "not_charset" +NOT_CHARSET_IGNORE = "not_charset_ignore" +NOT_CHARSET_IGNORE_REV = "not_charset_ignore_rev" +NOT_CHARSET_REV = "not_charset_rev" +NOT_DIGIT = "not_digit" +NOT_DIGIT_REV = "not_digit_rev" +NOT_IN = "not_in" +NOT_IN_IGNORE = "not_in_ignore" +NOT_IN_IGNORE_REV = "not_in_ignore_rev" +NOT_IN_REV = "not_in_rev" NOT_LITERAL = "not_literal" NOT_LITERAL_IGNORE = "not_literal_ignore" +NOT_LITERAL_IGNORE_REV = "not_literal_ignore_rev" +NOT_LITERAL_REV = "not_literal_rev" +NOT_RANGE = "not_range" +NOT_RANGE_IGNORE = "not_range_ignore" +NOT_RANGE_IGNORE_REV = "not_range_ignore_rev" +NOT_RANGE_REV = "not_range_rev" +NOT_WHITESPACE = "not_whitespace" +NOT_WHITESPACE_REV = "not_whitespace_rev" +NOT_WORD = "not_word" +NOT_WORD_REV = "not_word_rev" RANGE = "range" -REPEAT = "repeat" -REPEAT_ONE = "repeat_one" +RANGE_IGNORE = "range_ignore" +RANGE_IGNORE_REV = "range_ignore_rev" +RANGE_REV = "range_rev" +REPEAT_MAX = "repeat_max" +REPEAT_MAX_REV = "repeat_max_rev" +REPEAT_MIN = "repeat_min" +REPEAT_MIN_REV = "repeat_min_rev" +REPEAT_ONE_MAX = "repeat_one_max" +REPEAT_ONE_MAX_REV = "repeat_one_max_rev" +REPEAT_ONE_MIN = "repeat_one_min" +REPEAT_ONE_MIN_REV = "repeat_one_min_rev" +REPEAT_ONE_POSS = "repeat_one_poss" +REPEAT_ONE_POSS_REV = "repeat_one_poss_rev" +REPEAT_POSS = "repeat_poss" +REPEAT_POSS_REV = "repeat_poss_rev" +START_OF_LINE = "start_of_line" +START_OF_STRING = "start_of_string" SUBPATTERN = "subpattern" -MIN_REPEAT_ONE = "min_repeat_one" - -# positions -AT_BEGINNING = "at_beginning" -AT_BEGINNING_LINE = "at_beginning_line" -AT_BEGINNING_STRING = "at_beginning_string" -AT_BOUNDARY = "at_boundary" -AT_NON_BOUNDARY = "at_non_boundary" -AT_END = "at_end" -AT_END_LINE = "at_end_line" -AT_END_STRING = "at_end_string" -AT_LOC_BOUNDARY = "at_loc_boundary" -AT_LOC_NON_BOUNDARY = "at_loc_non_boundary" -AT_UNI_BOUNDARY = "at_uni_boundary" -AT_UNI_NON_BOUNDARY = "at_uni_non_boundary" - -# categories -CATEGORY_DIGIT = "category_digit" -CATEGORY_NOT_DIGIT = "category_not_digit" -CATEGORY_SPACE = "category_space" -CATEGORY_NOT_SPACE = "category_not_space" -CATEGORY_WORD = "category_word" -CATEGORY_NOT_WORD = "category_not_word" -CATEGORY_LINEBREAK = "category_linebreak" -CATEGORY_NOT_LINEBREAK = "category_not_linebreak" -CATEGORY_LOC_WORD = "category_loc_word" -CATEGORY_LOC_NOT_WORD = "category_loc_not_word" -CATEGORY_UNI_DIGIT = "category_uni_digit" -CATEGORY_UNI_NOT_DIGIT = "category_uni_not_digit" -CATEGORY_UNI_SPACE = "category_uni_space" -CATEGORY_UNI_NOT_SPACE = "category_uni_not_space" -CATEGORY_UNI_WORD = "category_uni_word" -CATEGORY_UNI_NOT_WORD = "category_uni_not_word" -CATEGORY_UNI_LINEBREAK = "category_uni_linebreak" -CATEGORY_UNI_NOT_LINEBREAK = "category_uni_not_linebreak" +UNI_BOUNDARY = "uni_boundary" +UNI_DIGIT = "uni_digit" +UNI_DIGIT_REV = "uni_digit_rev" +UNI_NOT_BOUNDARY = "uni_not_boundary" +UNI_NOT_DIGIT = "uni_not_digit" +UNI_NOT_DIGIT_REV = "uni_not_digit_rev" +UNI_NOT_WHITESPACE = "uni_not_whitespace" +UNI_NOT_WHITESPACE_REV = "uni_not_whitespace_rev" +UNI_NOT_WORD = "uni_not_word" +UNI_NOT_WORD_REV = "uni_not_word_rev" +UNI_WHITESPACE = "uni_whitespace" +UNI_WHITESPACE_REV = "uni_whitespace_rev" +UNI_WORD = "uni_word" +UNI_WORD_REV = "uni_word_rev" +WHITESPACE = "whitespace" +WHITESPACE_REV = "whitespace_rev" +WORD = "word" +WORD_REV = "word_rev" OPCODES = [ - # failure=0 success=1 (just because it looks better that way :-) - FAILURE, SUCCESS, + FAILURE, + SUCCESS, - ANY, ANY_ALL, - ASSERT, ASSERT_NOT, - AT, + ANY, + ANY_ALL, + ANY_ALL_REV, + ANY_REV, + ASSERT, + ASSERT_NOT, + ATOMIC, + BIGCHARSET, + BIGCHARSET_IGNORE, + BIGCHARSET_IGNORE_REV, + BIGCHARSET_REV, + BOUNDARY, BRANCH, CALL, - CATEGORY, - CHARSET, BIGCHARSET, - GROUPREF, GROUPREF_EXISTS, GROUPREF_IGNORE, - IN, IN_IGNORE, - INFO, + CHARSET, + CHARSET_IGNORE, + CHARSET_IGNORE_REV, + CHARSET_REV, + DIGIT, + DIGIT_REV, + END_ASSERT, + END_ASSERT_NOT, + END_ATOMIC, + END_OF_LINE, + END_OF_STRING, + END_OF_STRING_2, + END_REPEAT_MAX, + END_REPEAT_MAX_REV, + END_REPEAT_MIN, + END_REPEAT_MIN_REV, + END_REPEAT_ONE_MAX, + END_REPEAT_ONE_MAX_REV, + END_REPEAT_ONE_MIN, + END_REPEAT_ONE_MIN_REV, + END_REPEAT_ONE_POSS, + END_REPEAT_ONE_POSS_REV, + END_REPEAT_POSS, + END_REPEAT_POSS_REV, + GROUPREF, + GROUPREF_EXISTS, + GROUPREF_IGNORE, + GROUPREF_IGNORE_REV, + GROUPREF_REV, + IN, + IN_IGNORE, + IN_IGNORE_REV, + IN_REV, JUMP, - LITERAL, LITERAL_IGNORE, + LITERAL, + LITERAL_IGNORE, + LITERAL_IGNORE_REV, + LITERAL_REV, + LITERAL_STRING, + LITERAL_STRING_IGNORE, + LITERAL_STRING_IGNORE_REV, + LITERAL_STRING_REV, + LOC_BOUNDARY, + LOC_NOT_BOUNDARY, + LOC_NOT_WORD, + LOC_NOT_WORD_REV, + LOC_WORD, + LOC_WORD_REV, MARK, - MAX_UNTIL, - MIN_UNTIL, - NOT_LITERAL, NOT_LITERAL_IGNORE, - NEGATE, + NOT_BIGCHARSET, + NOT_BIGCHARSET_IGNORE, + NOT_BIGCHARSET_IGNORE_REV, + NOT_BIGCHARSET_REV, + NOT_BOUNDARY, + NOT_CHARSET, + NOT_CHARSET_IGNORE, + NOT_CHARSET_IGNORE_REV, + NOT_CHARSET_REV, + NOT_DIGIT, + NOT_DIGIT_REV, + NOT_IN, + NOT_IN_IGNORE, + NOT_IN_IGNORE_REV, + NOT_IN_REV, + NOT_LITERAL, + NOT_LITERAL_IGNORE, + NOT_LITERAL_IGNORE_REV, + NOT_LITERAL_REV, + NOT_RANGE, + NOT_RANGE_IGNORE, + NOT_RANGE_IGNORE_REV, + NOT_RANGE_REV, + NOT_WHITESPACE, + NOT_WHITESPACE_REV, + NOT_WORD, + NOT_WORD_REV, RANGE, - REPEAT, - REPEAT_ONE, + RANGE_IGNORE, + RANGE_IGNORE_REV, + RANGE_REV, + REPEAT_MAX, + REPEAT_MAX_REV, + REPEAT_MIN, + REPEAT_MIN_REV, + REPEAT_ONE_MAX, + REPEAT_ONE_MAX_REV, + REPEAT_ONE_MIN, + REPEAT_ONE_MIN_REV, + REPEAT_ONE_POSS, + REPEAT_ONE_POSS_REV, + REPEAT_POSS, + REPEAT_POSS_REV, + START_OF_LINE, + START_OF_STRING, SUBPATTERN, - MIN_REPEAT_ONE - -] - -ATCODES = [ - AT_BEGINNING, AT_BEGINNING_LINE, AT_BEGINNING_STRING, AT_BOUNDARY, - AT_NON_BOUNDARY, AT_END, AT_END_LINE, AT_END_STRING, - AT_LOC_BOUNDARY, AT_LOC_NON_BOUNDARY, AT_UNI_BOUNDARY, - AT_UNI_NON_BOUNDARY -] - -CHCODES = [ - CATEGORY_DIGIT, CATEGORY_NOT_DIGIT, CATEGORY_SPACE, - CATEGORY_NOT_SPACE, CATEGORY_WORD, CATEGORY_NOT_WORD, - CATEGORY_LINEBREAK, CATEGORY_NOT_LINEBREAK, CATEGORY_LOC_WORD, - CATEGORY_LOC_NOT_WORD, CATEGORY_UNI_DIGIT, CATEGORY_UNI_NOT_DIGIT, - CATEGORY_UNI_SPACE, CATEGORY_UNI_NOT_SPACE, CATEGORY_UNI_WORD, - CATEGORY_UNI_NOT_WORD, CATEGORY_UNI_LINEBREAK, - CATEGORY_UNI_NOT_LINEBREAK + UNI_BOUNDARY, + UNI_DIGIT, + UNI_DIGIT_REV, + UNI_NOT_BOUNDARY, + UNI_NOT_DIGIT, + UNI_NOT_DIGIT_REV, + UNI_NOT_WHITESPACE, + UNI_NOT_WHITESPACE_REV, + UNI_NOT_WORD, + UNI_NOT_WORD_REV, + UNI_WHITESPACE, + UNI_WHITESPACE_REV, + UNI_WORD, + UNI_WORD_REV, + WHITESPACE, + WHITESPACE_REV, + WORD, + WORD_REV, ] def makedict(list): @@ -153,53 +300,6 @@ return d OPCODES = makedict(OPCODES) -ATCODES = makedict(ATCODES) -CHCODES = makedict(CHCODES) - -# replacement operations for "ignore case" mode -OP_IGNORE = { - GROUPREF: GROUPREF_IGNORE, - IN: IN_IGNORE, - LITERAL: LITERAL_IGNORE, - NOT_LITERAL: NOT_LITERAL_IGNORE -} - -AT_MULTILINE = { - AT_BEGINNING: AT_BEGINNING_LINE, - AT_END: AT_END_LINE -} - -AT_LOCALE = { - AT_BOUNDARY: AT_LOC_BOUNDARY, - AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY -} - -AT_UNICODE = { - AT_BOUNDARY: AT_UNI_BOUNDARY, - AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY -} - -CH_LOCALE = { - CATEGORY_DIGIT: CATEGORY_DIGIT, - CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT, - CATEGORY_SPACE: CATEGORY_SPACE, - CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE, - CATEGORY_WORD: CATEGORY_LOC_WORD, - CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD, - CATEGORY_LINEBREAK: CATEGORY_LINEBREAK, - CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK -} - -CH_UNICODE = { - CATEGORY_DIGIT: CATEGORY_UNI_DIGIT, - CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT, - CATEGORY_SPACE: CATEGORY_UNI_SPACE, - CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE, - CATEGORY_WORD: CATEGORY_UNI_WORD, - CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD, - CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK, - CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK -} # flags SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking) @@ -242,8 +342,6 @@ f.write("#define SRE_MAGIC %d\n" % MAGIC) dump(f, OPCODES, "SRE_OP") - dump(f, ATCODES, "SRE") - dump(f, CHCODES, "SRE") f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE) f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE) Index: Lib/sre_parse.py =================================================================== --- sre_parse.py Wed Sep 14 09:54:39 2005 +++ sre_parse.py Mon Sep 08 16:31:06 2008 @@ -16,12 +16,6 @@ from sre_constants import * -def set(seq): - s = {} - for elem in seq: - s[elem] = 1 - return s - SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -30,7 +24,7 @@ OCTDIGITS = set("01234567") HEXDIGITS = set("0123456789abcdefABCDEF") -WHITESPACE = set(" \t\n\r\v\f") +WHITESPACE_CHARS = set(" \t\n\r\v\f") ESCAPES = { r"\a": (LITERAL, ord("\a")), @@ -43,17 +37,20 @@ r"\\": (LITERAL, ord("\\")) } -CATEGORIES = { - r"\A": (AT, AT_BEGINNING_STRING), # start of string - r"\b": (AT, AT_BOUNDARY), - r"\B": (AT, AT_NON_BOUNDARY), - r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), - r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), - r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), - r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), - r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), - r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), - r"\Z": (AT, AT_END_STRING), # end of string +POSITIONS = { + r"\A": (START_OF_STRING, None), + r"\b": (BOUNDARY, None), + r"\B": (NOT_BOUNDARY, None), + r"\Z": (END_OF_STRING, None) +} + +CHARSETS = { + r"\d": (DIGIT, None), + r"\D": (NOT_DIGIT, None), + r"\s": (WHITESPACE, None), + r"\S": (NOT_WHITESPACE, None), + r"\w": (WORD, None), + r"\W": (NOT_WORD, None) } FLAGS = { @@ -91,6 +88,12 @@ def checkgroup(self, gid): return gid < self.groups and gid not in self.open +UNIT_CODES = set([ANY, DIGIT, IN, LITERAL, NOT_DIGIT, + NOT_IN, NOT_LITERAL, NOT_WHITESPACE, NOT_WORD, RANGE, + WHITESPACE, WORD]) + +REPEAT_CODES = set([REPEAT_MIN, REPEAT_MAX, REPEAT_POSS]) + class SubPattern: # a subpattern, in intermediate form def __init__(self, pattern, data=None): @@ -104,12 +107,17 @@ seqtypes = type(()), type([]) for op, av in self.data: print level*" " + op,; nl = 0 - if op == "in": + if op == IN: + # member sublanguage + print; nl = 1 + for op, a in av: + print (level+1)*" " + op, a + elif op == NOT_IN: # member sublanguage print; nl = 1 for op, a in av: print (level+1)*" " + op, a - elif op == "branch": + elif op == BRANCH: print; nl = 1 i = 0 for a in av[1]: @@ -148,8 +156,6 @@ if self.width: return self.width lo = hi = 0L - UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY) - REPEATCODES = (MIN_REPEAT, MAX_REPEAT) for op, av in self.data: if op is BRANCH: i = sys.maxint @@ -164,19 +170,17 @@ i, j = av.getwidth() lo = lo + i hi = hi + j - elif op is SUBPATTERN: + elif op in [SUBPATTERN, ATOMIC]: i, j = av[1].getwidth() lo = lo + i hi = hi + j - elif op in REPEATCODES: + elif op in REPEAT_CODES: i, j = av[2].getwidth() lo = lo + long(i) * av[0] hi = hi + long(j) * av[1] - elif op in UNITCODES: + elif op in UNIT_CODES: lo = lo + 1 hi = hi + 1 - elif op == SUCCESS: - break self.width = int(min(lo, sys.maxint)), int(min(hi, sys.maxint)) return self.width @@ -230,10 +234,10 @@ def _class_escape(source, escape): # handle escape code inside character class - code = ESCAPES.get(escape) + code = CHARSETS.get(escape) if code: return code - code = CATEGORIES.get(escape) + code = ESCAPES.get(escape) if code: return code try: @@ -262,7 +266,10 @@ def _escape(source, escape, state): # handle escape code in expression - code = CATEGORIES.get(escape) + code = POSITIONS.get(escape) + if code: + return code + code = CHARSETS.get(escape) if code: return code code = ESCAPES.get(escape) @@ -325,40 +332,6 @@ return items[0] subpattern = SubPattern(state) - subpatternappend = subpattern.append - - # check if all items share a common prefix - while 1: - prefix = None - for item in items: - if not item: - break - if prefix is None: - prefix = item[0] - elif item[0] != prefix: - break - else: - # all subitems start with a common "prefix". - # move it out of the branch - for item in items: - del item[0] - subpatternappend(prefix) - continue # check next one - break - - # check if the branch can be replaced by a character set - for item in items: - if len(item) != 1 or item[0][0] != LITERAL: - break - else: - # we can store this as a character set instead of a - # branch (the compiler may optimize this even more) - set = [] - setappend = set.append - for item in items: - setappend(item[0]) - subpatternappend((IN, set)) - return subpattern subpattern.append((BRANCH, (None, items))) return subpattern @@ -377,10 +350,15 @@ subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern -_PATTERNENDERS = set("|)") -_ASSERTCHARS = set("=!<") -_LOOKBEHINDASSERTCHARS = set("=!") -_REPEATCODES = set([MIN_REPEAT, MAX_REPEAT]) +PATTERN_ENDERS = set("|)") +ASSERT_CHARS = set("=!<") +LOOKBEHIND_ASSERT_CHARS = set("=!") +POSITION_CODES = set([BOUNDARY, END_OF_LINE, END_OF_STRING, NOT_BOUNDARY, START_OF_LINE, START_OF_STRING]) +REPEAT_CODES = set([REPEAT_MIN, REPEAT_MAX, REPEAT_POSS]) +QUERY_GROUP = 0 +CAPTURE_GROUP = 1 +NONCAPTURE_GROUP = 2 +ATOMIC_GROUP = 3 def _parse(source, state): # parse a simple pattern @@ -390,15 +368,9 @@ subpatternappend = subpattern.append sourceget = source.get sourcematch = source.match - _len = len - PATTERNENDERS = _PATTERNENDERS - ASSERTCHARS = _ASSERTCHARS - LOOKBEHINDASSERTCHARS = _LOOKBEHINDASSERTCHARS - REPEATCODES = _REPEATCODES while 1: - - if source.next in PATTERNENDERS: + if source.next in PATTERN_ENDERS: break # end of subpattern this = sourceget() if this is None: @@ -406,7 +378,7 @@ if state.flags & SRE_FLAG_VERBOSE: # skip whitespace and comments - if this in WHITESPACE: + if this in WHITESPACE_CHARS: continue if this == "#": while 1: @@ -417,20 +389,17 @@ if this and this[0] not in SPECIAL_CHARS: subpatternappend((LITERAL, ord(this))) - elif this == "[": # character set - set = [] - setappend = set.append + charset = [] + setappend = charset.append ## if sourcematch(":"): ## pass # handle character classes - if sourcematch("^"): - setappend((NEGATE, None)) + negate = sourcematch("^") # check remaining characters - start = set[:] while 1: this = sourceget() - if this == "]" and set != start: + if this == "]" and charset: break elif this and this[0] == "\\": code1 = _class_escape(source, this) @@ -442,8 +411,6 @@ # potential range this = sourceget() if this == "]": - if code1[0] is IN: - code1 = code1[1][0] setappend(code1) setappend((LITERAL, ord("-"))) break @@ -462,34 +429,25 @@ else: raise error, "unexpected end of regular expression" else: - if code1[0] is IN: - code1 = code1[1][0] setappend(code1) - - # XXX: should move set optimization to compiler! - if _len(set)==1 and set[0][0] is LITERAL: - subpatternappend(set[0]) # optimization - elif _len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL: - subpatternappend((NOT_LITERAL, set[1][1])) # optimization + if negate: + subpatternappend((NOT_IN, charset)) else: - # XXX: should add charmap optimization here - subpatternappend((IN, set)) - + subpatternappend((IN, charset)) elif this and this[0] in REPEAT_CHARS: # repeat previous item if this == "?": min, max = 0, 1 elif this == "*": - min, max = 0, MAXREPEAT - + min, max = 0, MAX_REPEATS elif this == "+": - min, max = 1, MAXREPEAT + min, max = 1, MAX_REPEATS elif this == "{": if source.next == "}": subpatternappend((LITERAL, ord(this))) continue here = source.tell() - min, max = 0, MAXREPEAT + min, max = 0, MAX_REPEATS lo = hi = "" while source.next in DIGITS: lo = lo + source.get() @@ -511,28 +469,25 @@ else: raise error, "not supported" # figure out which item to repeat - if subpattern: - item = subpattern[-1:] - else: - item = None - if not item or (_len(item) == 1 and item[0][0] == AT): + item = subpattern[-1:] + if not item or len(item) == 1 and item[0][0] in POSITION_CODES: raise error, "nothing to repeat" - if item[0][0] in REPEATCODES: + if item[0][0] in REPEAT_CODES: raise error, "multiple repeat" if sourcematch("?"): - subpattern[-1] = (MIN_REPEAT, (min, max, item)) + subpattern[-1] = (REPEAT_MIN, (min, max, item)) + elif sourcematch("+"): + subpattern[-1] = (REPEAT_POSS, (min, max, item)) else: - subpattern[-1] = (MAX_REPEAT, (min, max, item)) - + subpattern[-1] = (REPEAT_MAX, (min, max, item)) elif this == ".": subpatternappend((ANY, None)) - elif this == "(": - group = 1 + group = CAPTURE_GROUP name = None condgroup = None if sourcematch("?"): - group = 0 + group = QUERY_GROUP # options if sourcematch("P"): # python extensions @@ -546,7 +501,7 @@ if char == ">": break name = name + char - group = 1 + group = CAPTURE_GROUP if not isname(name): raise error, "bad character in group name" elif sourcematch("="): @@ -573,7 +528,10 @@ raise error, "unknown specifier: ?P%s" % char elif sourcematch(":"): # non-capturing group - group = 2 + group = NONCAPTURE_GROUP + elif sourcematch(">"): + # atomic group + group = ATOMIC_GROUP elif sourcematch("#"): # comment while 1: @@ -583,12 +541,12 @@ if not sourcematch(")"): raise error, "unbalanced parenthesis" continue - elif source.next in ASSERTCHARS: + elif source.next in ASSERT_CHARS: # lookahead assertions char = sourceget() dir = 1 if char == "<": - if source.next not in LOOKBEHINDASSERTCHARS: + if source.next not in LOOKBEHIND_ASSERT_CHARS: raise error, "syntax error" dir = -1 # lookbehind char = sourceget() @@ -610,7 +568,7 @@ if char == ")": break condname = condname + char - group = 2 + group = NONCAPTURE_GROUP if isname(condname): condgroup = state.groupdict.get(condname) if condgroup is None: @@ -627,8 +585,9 @@ while source.next in FLAGS: state.flags = state.flags | FLAGS[sourceget()] if group: + atomic = group == ATOMIC_GROUP # parse group contents - if group == 2: + if group in [NONCAPTURE_GROUP, ATOMIC_GROUP]: # anonymous group group = None else: @@ -641,7 +600,10 @@ raise error, "unbalanced parenthesis" if group is not None: state.closegroup(group) - subpatternappend((SUBPATTERN, (group, p))) + if atomic: + subpatternappend((ATOMIC, (group, p))) + else: + subpatternappend((SUBPATTERN, (group, p))) else: while 1: char = sourceget() @@ -650,17 +612,13 @@ if char == ")": break raise error, "unknown extension" - elif this == "^": - subpatternappend((AT, AT_BEGINNING)) - + subpatternappend((START_OF_LINE, None)) elif this == "$": - subpattern.append((AT, AT_END)) - + subpattern.append((END_OF_LINE, None)) elif this and this[0] == "\\": code = _escape(source, this, state) subpatternappend(code) - else: raise error, "parser error" Index: Modules/_sre.c =================================================================== --- _sre.c Mon Feb 04 23:00:35 2008 +++ _sre.c Wed Sep 10 16:25:12 2008 @@ -4,24 +4,26 @@ * regular expression matching engine * * partial history: - * 1999-10-24 fl created (based on existing template matcher code) - * 2000-03-06 fl first alpha, sort of - * 2000-08-01 fl fixes for 1.6b1 - * 2000-08-07 fl use PyOS_CheckStack() if available - * 2000-09-20 fl added expand method - * 2001-03-20 fl lots of fixes for 2.1b2 - * 2001-04-15 fl export copyright as Python attribute, not global - * 2001-04-28 fl added __copy__ methods (work in progress) - * 2001-05-14 fl fixes for 1.5.2 compatibility - * 2001-07-01 fl added BIGCHARSET support (from Martin von Loewis) - * 2001-10-18 fl fixed group reset issue (from Matthew Mueller) - * 2001-10-20 fl added split primitive; reenable unicode for 1.6/2.0/2.1 - * 2001-10-21 fl added sub/subn primitive - * 2001-10-24 fl added finditer primitive (for 2.2 only) - * 2001-12-07 fl fixed memory leak in sub/subn (Guido van Rossum) - * 2002-11-09 fl fixed empty sub/subn return type - * 2003-04-18 mvl fully support 4-byte codes - * 2003-10-17 gn implemented non recursive scheme + * 1999-10-24 fl created (based on existing template matcher code) + * 2000-03-06 fl first alpha, sort of + * 2000-08-01 fl fixes for 1.6b1 + * 2000-08-07 fl use PyOS_CheckStack() if available + * 2000-09-20 fl added expand method + * 2001-03-20 fl lots of fixes for 2.1b2 + * 2001-04-15 fl export copyright as Python attribute, not global + * 2001-04-28 fl added __copy__ methods (work in progress) + * 2001-05-14 fl fixes for 1.5.2 compatibility + * 2001-07-01 fl added BIGCHARSET support (from Martin von Loewis) + * 2001-10-18 fl fixed group reset issue (from Matthew Mueller) + * 2001-10-20 fl added split primitive; reenable unicode for 1.6/2.0/2.1 + * 2001-10-21 fl added sub/subn primitive + * 2001-10-24 fl added finditer primitive (for 2.2 only) + * 2001-12-07 fl fixed memory leak in sub/subn (Guido van Rossum) + * 2002-11-09 fl fixed empty sub/subn return type + * 2003-04-18 mvl fully support 4-byte codes + * 2003-10-17 gn implemented non recursive scheme + * 2008-09-09 mrab major reworking; added atomic groups, possessive quantifiers, + * let lookbehinds be variable length; faster * * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. * @@ -48,6 +50,8 @@ #include +#define UNLIMITED_REPEATS 0xFFFF + /* name of this module, minus the leading underscore */ #if !defined(SRE_MODULE) #define SRE_MODULE "sre" @@ -68,9 +72,6 @@ /* -------------------------------------------------------------------- */ /* optional features */ -/* enables fast searching */ -#define USE_FAST_SEARCH - /* enables aggressive inlining (always on for Visual C) */ #undef USE_INLINE @@ -94,6 +95,52 @@ #define LOCAL(type) static type #endif +#ifdef Py_UNICODE_WIDE +LOCAL(int) in_charset(SRE_CODE* charset, SRE_CODE ch) { + return charset[ch / 32] & (1 << (ch % 32)); +} + +LOCAL(SRE_CODE*) skip_charset(SRE_CODE* charset) { + return charset + 256 / 32; +} + +LOCAL(int) in_bigcharset(SRE_CODE* charset, SRE_CODE ch) { + int hi_byte = ch / 256; + int lo_byte = ch % 256; + int index; + charset++; + index = (charset[hi_byte / 4] >> ((hi_byte % 4) * 8)) & 0xFF; + charset += 256 / 4; + return charset[lo_byte / 32] & (1 << (lo_byte % 32)); +} + +LOCAL(SRE_CODE*) skip_bigcharset(SRE_CODE* charset) { + return charset + charset[0]; +} +#else +LOCAL(int) in_charset(SRE_CODE* charset, SRE_CODE ch) { + return charset[ch / 16] & (1 << (ch % 16)); +} + +LOCAL(SRE_CODE*) skip_charset(SRE_CODE* charset) { + return charset + 256 / 16; +} + +LOCAL(int) in_bigcharset(SRE_CODE* charset, SRE_CODE ch) { + int hi_byte = ch / 256; + int lo_byte = ch % 256; + int index; + charset++; + index = (charset[hi_byte / 2] >> ((hi_byte % 2) * 8)) & 0xFF; + charset += 256 / 2; + return charset[lo_byte / 16] & (1 << (lo_byte % 16)); +} + +LOCAL(SRE_CODE*) skip_bigcharset(SRE_CODE* charset) { + return charset + charset[0]; +} +#endif + /* error codes */ #define SRE_ERROR_ILLEGAL -1 /* illegal opcode */ #define SRE_ERROR_STATE -2 /* illegal state */ @@ -113,7 +160,7 @@ /* default character predicates (run sre_chars.py to regenerate tables) */ #define SRE_DIGIT_MASK 1 -#define SRE_SPACE_MASK 2 +#define SRE_WHITESPACE_MASK 2 #define SRE_LINEBREAK_MASK 4 #define SRE_ALNUM_MASK 8 #define SRE_WORD_MASK 16 @@ -140,8 +187,8 @@ #define SRE_IS_DIGIT(ch)\ ((ch) < 128 ? (sre_char_info[(ch)] & SRE_DIGIT_MASK) : 0) -#define SRE_IS_SPACE(ch)\ - ((ch) < 128 ? (sre_char_info[(ch)] & SRE_SPACE_MASK) : 0) +#define SRE_IS_WHITESPACE(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_WHITESPACE_MASK) : 0) #define SRE_IS_LINEBREAK(ch)\ ((ch) < 128 ? (sre_char_info[(ch)] & SRE_LINEBREAK_MASK) : 0) #define SRE_IS_ALNUM(ch)\ @@ -158,7 +205,7 @@ /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids * warnings when c's type supports only numbers < N+1 */ #define SRE_LOC_IS_DIGIT(ch) (!((ch) & ~255) ? isdigit((ch)) : 0) -#define SRE_LOC_IS_SPACE(ch) (!((ch) & ~255) ? isspace((ch)) : 0) +#define SRE_LOC_IS_WHITESPACE(ch) (!((ch) & ~255) ? isspace((ch)) : 0) #define SRE_LOC_IS_LINEBREAK(ch) ((ch) == '\n') #define SRE_LOC_IS_ALNUM(ch) (!((ch) & ~255) ? isalnum((ch)) : 0) #define SRE_LOC_IS_WORD(ch) (SRE_LOC_IS_ALNUM((ch)) || (ch) == '_') @@ -173,7 +220,7 @@ #if defined(HAVE_UNICODE) #define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDIGIT((Py_UNICODE)(ch)) -#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch)) +#define SRE_UNI_IS_WHITESPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch)) #define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK((Py_UNICODE)(ch)) #define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM((Py_UNICODE)(ch)) #define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM((ch)) || (ch) == '_') @@ -185,144 +232,41 @@ #endif -LOCAL(int) -sre_category(SRE_CODE category, unsigned int ch) -{ - switch (category) { - - case SRE_CATEGORY_DIGIT: - return SRE_IS_DIGIT(ch); - case SRE_CATEGORY_NOT_DIGIT: - return !SRE_IS_DIGIT(ch); - case SRE_CATEGORY_SPACE: - return SRE_IS_SPACE(ch); - case SRE_CATEGORY_NOT_SPACE: - return !SRE_IS_SPACE(ch); - case SRE_CATEGORY_WORD: - return SRE_IS_WORD(ch); - case SRE_CATEGORY_NOT_WORD: - return !SRE_IS_WORD(ch); - case SRE_CATEGORY_LINEBREAK: - return SRE_IS_LINEBREAK(ch); - case SRE_CATEGORY_NOT_LINEBREAK: - return !SRE_IS_LINEBREAK(ch); - - case SRE_CATEGORY_LOC_WORD: - return SRE_LOC_IS_WORD(ch); - case SRE_CATEGORY_LOC_NOT_WORD: - return !SRE_LOC_IS_WORD(ch); - -#if defined(HAVE_UNICODE) - case SRE_CATEGORY_UNI_DIGIT: - return SRE_UNI_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_NOT_DIGIT: - return !SRE_UNI_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_SPACE: - return SRE_UNI_IS_SPACE(ch); - case SRE_CATEGORY_UNI_NOT_SPACE: - return !SRE_UNI_IS_SPACE(ch); - case SRE_CATEGORY_UNI_WORD: - return SRE_UNI_IS_WORD(ch); - case SRE_CATEGORY_UNI_NOT_WORD: - return !SRE_UNI_IS_WORD(ch); - case SRE_CATEGORY_UNI_LINEBREAK: - return SRE_UNI_IS_LINEBREAK(ch); - case SRE_CATEGORY_UNI_NOT_LINEBREAK: - return !SRE_UNI_IS_LINEBREAK(ch); -#else - case SRE_CATEGORY_UNI_DIGIT: - return SRE_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_NOT_DIGIT: - return !SRE_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_SPACE: - return SRE_IS_SPACE(ch); - case SRE_CATEGORY_UNI_NOT_SPACE: - return !SRE_IS_SPACE(ch); - case SRE_CATEGORY_UNI_WORD: - return SRE_LOC_IS_WORD(ch); - case SRE_CATEGORY_UNI_NOT_WORD: - return !SRE_LOC_IS_WORD(ch); - case SRE_CATEGORY_UNI_LINEBREAK: - return SRE_IS_LINEBREAK(ch); - case SRE_CATEGORY_UNI_NOT_LINEBREAK: - return !SRE_IS_LINEBREAK(ch); -#endif - } - return 0; -} - -/* helpers */ - -static void -data_stack_dealloc(SRE_STATE* state) -{ - if (state->data_stack) { - PyMem_FREE(state->data_stack); - state->data_stack = NULL; - } - state->data_stack_size = state->data_stack_base = 0; -} - -static int -data_stack_grow(SRE_STATE* state, Py_ssize_t size) -{ - Py_ssize_t minsize, cursize; - minsize = state->data_stack_base+size; - cursize = state->data_stack_size; - if (cursize < minsize) { - void* stack; - cursize = minsize+minsize/4+1024; - TRACE(("allocate/grow stack %d\n", cursize)); - stack = PyMem_REALLOC(state->data_stack, cursize); - if (!stack) { - data_stack_dealloc(state); - return SRE_ERROR_MEMORY; - } - state->data_stack = (char *)stack; - state->data_stack_size = cursize; - } - return 0; +LOCAL(int) sre_repeat_limit(int repeats, int limit) { + if (repeats > limit || repeats == UNLIMITED_REPEATS) + return limit; + return repeats; } /* generate 8-bit version */ #define SRE_CHAR unsigned char -#define SRE_AT sre_at -#define SRE_COUNT sre_count -#define SRE_CHARSET sre_charset -#define SRE_INFO sre_info +#define SRE_IN sre_in #define SRE_MATCH sre_match -#define SRE_MATCH_CONTEXT sre_match_context #define SRE_SEARCH sre_search #define SRE_LITERAL_TEMPLATE sre_literal_template +#define SRE_IN_RANGE sre_in_range #if defined(HAVE_UNICODE) #define SRE_RECURSIVE #include "_sre.c" #undef SRE_RECURSIVE - +#undef SRE_IN_RANGE #undef SRE_LITERAL_TEMPLATE #undef SRE_SEARCH #undef SRE_MATCH -#undef SRE_MATCH_CONTEXT -#undef SRE_INFO -#undef SRE_CHARSET -#undef SRE_COUNT -#undef SRE_AT +#undef SRE_IN #undef SRE_CHAR /* generate 16-bit unicode version */ #define SRE_CHAR Py_UNICODE -#define SRE_AT sre_uat -#define SRE_COUNT sre_ucount -#define SRE_CHARSET sre_ucharset -#define SRE_INFO sre_uinfo +#define SRE_IN sre_uin #define SRE_MATCH sre_umatch -#define SRE_MATCH_CONTEXT sre_umatch_context #define SRE_SEARCH sre_usearch #define SRE_LITERAL_TEMPLATE sre_uliteral_template +#define SRE_IN_RANGE sre_uin_range #endif #endif /* SRE_RECURSIVE */ @@ -333,1156 +277,3416 @@ /* the following section is compiled twice, with different character settings */ -LOCAL(int) -SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at) -{ - /* check if pointer is at given position */ - - Py_ssize_t thisp, thatp; - - switch (at) { - - case SRE_AT_BEGINNING: - case SRE_AT_BEGINNING_STRING: - return ((void*) ptr == state->beginning); - - case SRE_AT_BEGINNING_LINE: - return ((void*) ptr == state->beginning || - SRE_IS_LINEBREAK((int) ptr[-1])); - - case SRE_AT_END: - return (((void*) (ptr+1) == state->end && - SRE_IS_LINEBREAK((int) ptr[0])) || - ((void*) ptr == state->end)); - - case SRE_AT_END_LINE: - return ((void*) ptr == state->end || - SRE_IS_LINEBREAK((int) ptr[0])); - - case SRE_AT_END_STRING: - return ((void*) ptr == state->end); - - case SRE_AT_BOUNDARY: - if (state->beginning == state->end) - return 0; - thatp = ((void*) ptr > state->beginning) ? - SRE_IS_WORD((int) ptr[-1]) : 0; - thisp = ((void*) ptr < state->end) ? - SRE_IS_WORD((int) ptr[0]) : 0; - return thisp != thatp; - - case SRE_AT_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; - thatp = ((void*) ptr > state->beginning) ? - SRE_IS_WORD((int) ptr[-1]) : 0; - thisp = ((void*) ptr < state->end) ? - SRE_IS_WORD((int) ptr[0]) : 0; - return thisp == thatp; - - case SRE_AT_LOC_BOUNDARY: - if (state->beginning == state->end) - return 0; - thatp = ((void*) ptr > state->beginning) ? - SRE_LOC_IS_WORD((int) ptr[-1]) : 0; - thisp = ((void*) ptr < state->end) ? - SRE_LOC_IS_WORD((int) ptr[0]) : 0; - return thisp != thatp; - - case SRE_AT_LOC_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; - thatp = ((void*) ptr > state->beginning) ? - SRE_LOC_IS_WORD((int) ptr[-1]) : 0; - thisp = ((void*) ptr < state->end) ? - SRE_LOC_IS_WORD((int) ptr[0]) : 0; - return thisp == thatp; - -#if defined(HAVE_UNICODE) - case SRE_AT_UNI_BOUNDARY: - if (state->beginning == state->end) - return 0; - thatp = ((void*) ptr > state->beginning) ? - SRE_UNI_IS_WORD((int) ptr[-1]) : 0; - thisp = ((void*) ptr < state->end) ? - SRE_UNI_IS_WORD((int) ptr[0]) : 0; - return thisp != thatp; - - case SRE_AT_UNI_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; - thatp = ((void*) ptr > state->beginning) ? - SRE_UNI_IS_WORD((int) ptr[-1]) : 0; - thisp = ((void*) ptr < state->end) ? - SRE_UNI_IS_WORD((int) ptr[0]) : 0; - return thisp == thatp; -#endif - - } - - return 0; +LOCAL(int) SRE_IN_RANGE(SRE_CODE ch, SRE_CODE lower, SRE_CODE upper) { + return lower <= ch && ch <= upper; } LOCAL(int) -SRE_CHARSET(SRE_CODE* set, SRE_CODE ch) -{ - /* check if character is a member of the given set */ - - int ok = 1; - - for (;;) { - switch (*set++) { +SRE_IN(SRE_CODE* charset, SRE_CODE ch) { + // Check if character is a member of the given set. + SRE_CODE* charset_end = charset + charset[0]; - case SRE_OP_FAILURE: - return !ok; + charset++; + do { + switch (*charset++) { + case SRE_OP_BIGCHARSET: + // + if (in_bigcharset(charset, ch)) + return 1; + charset = skip_bigcharset(charset); + break; + case SRE_OP_CHARSET: + // + if (in_charset(charset, ch)) + return 1; + charset = skip_charset(charset); + break; + case SRE_OP_DIGIT: + // + if (SRE_IS_DIGIT(ch)) + return 1; + break; case SRE_OP_LITERAL: - /* */ - if (ch == set[0]) - return ok; - set++; + // + if (ch == charset[0]) + return 1; + charset++; break; - - case SRE_OP_CATEGORY: - /* */ - if (sre_category(set[0], (int) ch)) - return ok; - set += 1; + case SRE_OP_LOC_NOT_WORD: + // + if (! SRE_LOC_IS_WORD(ch)) + return 1; break; - - case SRE_OP_CHARSET: - if (sizeof(SRE_CODE) == 2) { - /* (16 bits per code word) */ - if (ch < 256 && (set[ch >> 4] & (1 << (ch & 15)))) - return ok; - set += 16; - } - else { - /* (32 bits per code word) */ - if (ch < 256 && (set[ch >> 5] & (1 << (ch & 31)))) - return ok; - set += 8; - } + case SRE_OP_LOC_WORD: + // + if (SRE_LOC_IS_WORD(ch)) + return 1; + break; + case SRE_OP_NOT_DIGIT: + // + if (!SRE_IS_DIGIT(ch)) + return 1; + break; + case SRE_OP_NOT_LITERAL: + // + if (ch != charset[0]) + return 1; + charset++; + break; + case SRE_OP_NOT_WHITESPACE: + // + if (!SRE_IS_WHITESPACE(ch)) + return 1; + break; + case SRE_OP_NOT_WORD: + // + if (! SRE_IS_WORD(ch)) + return 1; break; - case SRE_OP_RANGE: - /* */ - if (set[0] <= ch && ch <= set[1]) - return ok; - set += 2; + // + if (SRE_IN_RANGE(ch, charset[0], charset[1])) + return 1; + charset += 2; break; - - case SRE_OP_NEGATE: - ok = !ok; + case SRE_OP_UNI_DIGIT: + // + if (SRE_UNI_IS_DIGIT(ch)) + return 1; break; - - case SRE_OP_BIGCHARSET: - /* <256 blockindices> */ - { - Py_ssize_t count, block; - count = *(set++); - - if (sizeof(SRE_CODE) == 2) { - block = ((unsigned char*)set)[ch >> 8]; - set += 128; - if (set[block*16 + ((ch & 255)>>4)] & (1 << (ch & 15))) - return ok; - set += count*16; - } - else { - /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids - * warnings when c's type supports only numbers < N+1 */ - if (!(ch & ~65535)) - block = ((unsigned char*)set)[ch >> 8]; - else - block = -1; - set += 64; - if (block >=0 && - (set[block*8 + ((ch & 255)>>5)] & (1 << (ch & 31)))) - return ok; - set += count*8; - } + case SRE_OP_UNI_NOT_DIGIT: + // + if (!SRE_UNI_IS_DIGIT(ch)) + return 1; + break; + case SRE_OP_UNI_NOT_WHITESPACE: + // + if (! SRE_UNI_IS_WHITESPACE(ch)) + return 1; + break; + case SRE_OP_UNI_NOT_WORD: + // + if (! SRE_UNI_IS_WORD(ch)) + return 1; + break; + case SRE_OP_UNI_WHITESPACE: + // + if (SRE_UNI_IS_WHITESPACE(ch)) + return 1; + break; + case SRE_OP_UNI_WORD: + // + if (SRE_UNI_IS_WORD(ch)) + return 1; + break; + case SRE_OP_WHITESPACE: + // + if (SRE_IS_WHITESPACE(ch)) + return 1; + break; + case SRE_OP_WORD: + // + if (SRE_IS_WORD(ch)) + return 1; break; - } - default: /* internal error -- there's not much we can do about it here, so let's just pretend it didn't match... */ return 0; } - } -} + } while (charset < charset_end); -LOCAL(Py_ssize_t) SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern); + return 0; +} -LOCAL(Py_ssize_t) -SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) -{ - SRE_CODE chr; - SRE_CHAR* ptr = (SRE_CHAR *)state->ptr; - SRE_CHAR* end = (SRE_CHAR *)state->end; - Py_ssize_t i; +#define SRE_RETURN(result) { \ + while (backtrack_chunk->previous != NULL) { \ + SRE_BACKTRACK_CHUNK* previous = backtrack_chunk->previous; \ + PyMem_FREE(backtrack_chunk); \ + backtrack_chunk = previous; \ + } \ + backtrack_chunk->count = 0; \ + state->backtrack_chunk = backtrack_chunk; \ +\ + while (saved_marks_chunk->previous != NULL) { \ + SRE_SAVED_MARKS_CHUNK* previous = saved_marks_chunk->previous; \ + PyMem_FREE(saved_marks_chunk); \ + saved_marks_chunk = previous; \ + } \ + saved_marks_chunk->count = 0; \ + state->saved_marks_chunk = saved_marks_chunk; \ +\ + return result; \ +} + +#define SAVE_BACKTRACK(OP, PATTERN_PTR, INDEX) { \ + SRE_BACKTRACK_ITEM* backtrack_item; \ +\ + if (backtrack_chunk->count >= SRE_BACKTRACK_CHUNK_SIZE) { \ + SRE_BACKTRACK_CHUNK* new_backtrack_chunk = (SRE_BACKTRACK_CHUNK*)PyMem_MALLOC(sizeof(SRE_BACKTRACK_CHUNK)); \ + if (new_backtrack_chunk == NULL) \ + SRE_RETURN(SRE_ERROR_MEMORY) \ +\ + new_backtrack_chunk->previous = backtrack_chunk; \ + new_backtrack_chunk->count = 0; \ + backtrack_chunk = new_backtrack_chunk; \ + state->backtrack_chunk = backtrack_chunk; \ + } \ +\ + backtrack_item = &backtrack_chunk->items[backtrack_chunk->count++]; \ + backtrack_item->op = OP; \ + backtrack_item->pattern_ptr = PATTERN_PTR; \ + backtrack_item->text_ptr = text_ptr; \ + backtrack_item->index = INDEX; \ + backtrack_item->repeat_count = repeat_counts[INDEX]; \ + backtrack_item->repeat_start = repeat_starts[INDEX]; \ +} - /* adjust end */ - if (maxcount < end - ptr && maxcount != 65535) - end = ptr + maxcount; - - switch (pattern[0]) { - - case SRE_OP_IN: - /* repeated set */ - TRACE(("|%p|%p|COUNT IN\n", pattern, ptr)); - while (ptr < end && SRE_CHARSET(pattern + 2, *ptr)) - ptr++; - break; +#define DISCARD_BACKTRACK(REPEAT_COUNT) { \ + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[--backtrack_chunk->count]; \ + if (REPEAT_COUNT) { \ + repeat_counts[backtrack_item->index] = backtrack_item->repeat_count; \ + repeat_starts[backtrack_item->index] = backtrack_item->repeat_start; \ + } \ +\ + if (backtrack_chunk->count == 0 && backtrack_chunk->previous != NULL) { \ + SRE_BACKTRACK_CHUNK* previous = backtrack_chunk->previous; \ + PyMem_FREE(backtrack_chunk); \ + backtrack_chunk = previous; \ + state->backtrack_chunk = backtrack_chunk; \ + } \ +} - case SRE_OP_ANY: - /* repeated dot wildcard. */ - TRACE(("|%p|%p|COUNT ANY\n", pattern, ptr)); - while (ptr < end && !SRE_IS_LINEBREAK(*ptr)) - ptr++; - break; +#define SAVE_MARKS() { \ + if (saved_marks_chunk->count + mark_count > SRE_SAVED_MARKS_CHUNK_SIZE) { \ + SRE_SAVED_MARKS_CHUNK* new_marks_chunk = (SRE_SAVED_MARKS_CHUNK*)PyMem_MALLOC(sizeof(SRE_SAVED_MARKS_CHUNK)); \ + if (new_marks_chunk == NULL) \ + SRE_RETURN(SRE_ERROR_MEMORY); \ +\ + new_marks_chunk->previous = saved_marks_chunk; \ + new_marks_chunk->count = 0; \ + saved_marks_chunk = new_marks_chunk; \ + state->saved_marks_chunk = saved_marks_chunk; \ + } \ +\ + memmove(&saved_marks_chunk->marks[saved_marks_chunk->count], mark, mark_count * sizeof(SRE_CHAR*)); \ + saved_marks_chunk->count += mark_count; \ +} - case SRE_OP_ANY_ALL: - /* repeated dot wildcard. skip to the end of the target - string, and backtrack from there */ - TRACE(("|%p|%p|COUNT ANY_ALL\n", pattern, ptr)); - ptr = end; - break; +#define RESTORE_MARKS() { \ + saved_marks_chunk->count -= mark_count; \ + memmove(mark, &saved_marks_chunk->marks[saved_marks_chunk->count], mark_count * sizeof(SRE_CHAR*)); \ +\ + if (saved_marks_chunk->count == 0 && saved_marks_chunk->previous != NULL) { \ + SRE_SAVED_MARKS_CHUNK* previous = saved_marks_chunk->previous; \ + PyMem_FREE(saved_marks_chunk); \ + saved_marks_chunk = previous; \ + state->saved_marks_chunk = saved_marks_chunk; \ + } \ +} - case SRE_OP_LITERAL: - /* repeated literal */ - chr = pattern[1]; - TRACE(("|%p|%p|COUNT LITERAL %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) *ptr == chr) - ptr++; - break; +#define DISCARD_SAVED_MARKS() { \ + saved_marks_chunk->count -= mark_count; \ +\ + if (saved_marks_chunk->count == 0 && saved_marks_chunk->previous != NULL) { \ + SRE_SAVED_MARKS_CHUNK* previous = saved_marks_chunk->previous; \ + PyMem_FREE(saved_marks_chunk); \ + saved_marks_chunk = previous; \ + state->saved_marks_chunk = saved_marks_chunk; \ + } \ +} - case SRE_OP_LITERAL_IGNORE: - /* repeated literal */ - chr = pattern[1]; - TRACE(("|%p|%p|COUNT LITERAL_IGNORE %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) state->lower(*ptr) == chr) - ptr++; - break; +#define REPEAT_ONE_MAX_MACRO(TEST, LABEL) { \ + /* Match up to the maximum. */ \ + while (text_ptr < end_ptr && (TEST)) \ + text_ptr++; \ + start_ptr += repeat_min; \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_max; \ + case SRE_OP_LITERAL_IGNORE: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_max_ignore; \ + case SRE_OP_LITERAL_STRING: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_max: \ + while (text_ptr >= start_ptr && text_ptr[0] != following_char) \ + text_ptr--; \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_max_ignore: \ + while (text_ptr >= start_ptr && state->lower(text_ptr[0]) != following_char) \ + text_ptr--; \ + break; \ + } \ +} - case SRE_OP_NOT_LITERAL: - /* repeated non-literal */ - chr = pattern[1]; - TRACE(("|%p|%p|COUNT NOT_LITERAL %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) *ptr != chr) - ptr++; - break; +#define REPEAT_ONE_MAX_REV_MACRO(TEST, LABEL) { \ + /* Match up to the maximum. */ \ + while (text_ptr > end_ptr && (TEST)) \ + text_ptr--; \ + start_ptr -= repeat_min; \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_max_rev; \ + case SRE_OP_LITERAL_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_max_ignore_rev; \ + case SRE_OP_LITERAL_STRING_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_max_rev: \ + while (text_ptr <= start_ptr && text_ptr[-1] != following_char) \ + text_ptr++; \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_max_ignore_rev: \ + while (text_ptr <= start_ptr && state->lower(text_ptr[-1]) != following_char) \ + text_ptr++; \ + break; \ + } \ +} - case SRE_OP_NOT_LITERAL_IGNORE: - /* repeated non-literal */ - chr = pattern[1]; - TRACE(("|%p|%p|COUNT NOT_LITERAL_IGNORE %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) state->lower(*ptr) != chr) - ptr++; - break; +#define REPEAT_ONE_MIN_MACRO(TEST, LABEL) { \ + /* First match the minimum. */ \ + while (text_ptr < end_ptr && (TEST)) \ + text_ptr++; \ + /* Matched at least the minimum? */ \ + start_ptr += repeat_min; \ + if (text_ptr < start_ptr) \ + goto backtrack; \ + /* Now match up to the maximum. */ \ + end_ptr = start_ptr - repeat_min + repeat_max; \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_min; \ + case SRE_OP_LITERAL_IGNORE: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_min_ignore; \ + case SRE_OP_LITERAL_STRING: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_min: \ + while (text_ptr[0] != following_char) { \ + if (text_ptr >= end_ptr || !(TEST)) \ + goto backtrack; \ + text_ptr++; \ + } \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_min_ignore: \ + while (state->lower(text_ptr[0]) != following_char) { \ + if (text_ptr >= end_ptr || !(TEST)) \ + goto backtrack; \ + text_ptr++; \ + } \ + break; \ + } \ +} - default: - /* repeated single character pattern */ - TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr)); - while ((SRE_CHAR*) state->ptr < end) { - i = SRE_MATCH(state, pattern); - if (i < 0) - return i; - if (!i) - break; - } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, - (SRE_CHAR*) state->ptr - ptr)); - return (SRE_CHAR*) state->ptr - ptr; - } +#define REPEAT_ONE_MIN_REV_MACRO(TEST, LABEL) { \ + /* First match the minimum. */ \ + while (text_ptr > end_ptr && (TEST)) \ + text_ptr--; \ + /* Matched at least the minimum? */ \ + start_ptr -= repeat_min; \ + if (text_ptr > start_ptr) \ + goto backtrack; \ + /* Now match up to the maximum. */ \ + end_ptr = start_ptr + repeat_min - repeat_max; \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_min_rev; \ + case SRE_OP_LITERAL_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_min_ignore_rev; \ + case SRE_OP_LITERAL_STRING_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_min_rev: \ + while (text_ptr[-1] != following_char) { \ + if (text_ptr <= end_ptr || !(TEST)) \ + goto backtrack; \ + text_ptr--; \ + } \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_min_ignore_rev: \ + while (state->lower(text_ptr[-1]) != following_char) { \ + if (text_ptr <= end_ptr || !(TEST)) \ + goto backtrack; \ + text_ptr--; \ + } \ + break; \ + } \ +} - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); - return ptr - (SRE_CHAR*) state->ptr; +#define REPEAT_ONE_MIN_BACKTRACK_MACRO(TEST, LABEL) { \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto end_##LABEL##_min; \ + case SRE_OP_LITERAL_IGNORE: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto end_##LABEL##_min_ignore; \ + case SRE_OP_LITERAL_STRING: \ + following_char = (SRE_CHAR)look_literal[2]; \ +end_##LABEL##_min: \ + do { \ + if (text_ptr >= end_ptr || !(TEST)) { \ + DISCARD_BACKTRACK(1); \ + goto backtrack; \ + } \ + text_ptr++; \ + } while (text_ptr[0] != following_char); \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE: \ + following_char = (SRE_CHAR)look_literal[2]; \ +end_##LABEL##_min_ignore: \ + do { \ + if (text_ptr >= end_ptr || !(TEST)) { \ + DISCARD_BACKTRACK(1); \ + goto backtrack; \ + } \ + text_ptr++; \ + } while (state->lower(text_ptr[0]) != following_char); \ + break; \ + default: \ + if (text_ptr >= end_ptr || !(TEST)) { \ + DISCARD_BACKTRACK(1); \ + goto backtrack; \ + } \ + text_ptr++; \ + } \ } -#if 0 /* not used in this release */ -LOCAL(int) -SRE_INFO(SRE_STATE* state, SRE_CODE* pattern) -{ - /* check if an SRE_OP_INFO block matches at the current position. - returns the number of SRE_CODE objects to skip if successful, 0 - if no match */ +#define REPEAT_ONE_MIN_BACKTRACK_REV_MACRO(TEST, LABEL) { \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto end_##LABEL##_min_rev; \ + case SRE_OP_LITERAL_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto end_##LABEL##_min_ignore_rev; \ + case SRE_OP_LITERAL_STRING_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +end_##LABEL##_min_rev: \ + do { \ + if (text_ptr <= end_ptr || !(TEST)) { \ + DISCARD_BACKTRACK(1); \ + goto backtrack; \ + } \ + text_ptr--; \ + } while (text_ptr[-1] != following_char); \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +end_##LABEL##_min_ignore_rev: \ + do { \ + if (text_ptr <= end_ptr || !(TEST)) { \ + DISCARD_BACKTRACK(1); \ + goto backtrack; \ + } \ + text_ptr--; \ + } while (state->lower(text_ptr[-1]) != following_char); \ + break; \ + default: \ + if (text_ptr <= end_ptr || !(TEST)) { \ + DISCARD_BACKTRACK(1); \ + goto backtrack; \ + } \ + text_ptr--; \ + } \ +} - SRE_CHAR* end = state->end; - SRE_CHAR* ptr = state->ptr; - Py_ssize_t i; +#define REPEAT_ONE_POSS_MACRO(TEST, LABEL) { \ + /* Match up to the maximum. */ \ + while (text_ptr < end_ptr && (TEST)) \ + text_ptr++; \ + start_ptr += repeat_min; \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_poss; \ + case SRE_OP_LITERAL_IGNORE: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_poss_ignore; \ + case SRE_OP_LITERAL_STRING: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_poss: \ + if (text_ptr[0] != following_char) \ + goto backtrack; \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_poss_ignore: \ + if (state->lower(text_ptr[0]) != following_char) \ + goto backtrack; \ + break; \ + } \ +} - /* check minimal length */ - if (pattern[3] && (end - ptr) < pattern[3]) - return 0; +#define REPEAT_ONE_POSS_REV_MACRO(TEST, LABEL) { \ + /* Match up to the maximum. */ \ + while (text_ptr > end_ptr && (TEST)) \ + text_ptr--; \ + start_ptr -= repeat_min; \ + /* Look at what follows to avoid unnecessary backtracking. */ \ + switch (look_literal[0]) { \ + case SRE_OP_LITERAL_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_poss_rev; \ + case SRE_OP_LITERAL_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[1]; \ + goto LABEL##_poss_ignore_rev; \ + case SRE_OP_LITERAL_STRING_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_poss_rev: \ + if (text_ptr[-1] != following_char) \ + goto backtrack; \ + break; \ + case SRE_OP_LITERAL_STRING_IGNORE_REV: \ + following_char = (SRE_CHAR)look_literal[2]; \ +LABEL##_poss_ignore_rev: \ + if (state->lower(text_ptr[-1]) != following_char) \ + goto backtrack; \ + break; \ + } \ +} - /* check known prefix */ - if (pattern[2] & SRE_INFO_PREFIX && pattern[5] > 1) { - /* */ - for (i = 0; i < pattern[5]; i++) - if ((SRE_CODE) ptr[i] != pattern[7 + i]) - return 0; - return pattern[0] + 2 * pattern[6]; - } - return pattern[0]; +#define REPEAT_ONE_MACRO(MACRO) { \ + switch (*body++) { \ + case SRE_OP_ANY: \ + { \ + /* Match anything (except a newline). */ \ + /* */ \ + TRACE(("|%p|%p|ANY\n", body, text_ptr)); \ + MACRO(!SRE_IS_LINEBREAK(text_ptr[0]), any) \ + break; \ + } \ + case SRE_OP_ANY_ALL: \ + { \ + /* Match anything. */ \ + /* */ \ + TRACE(("|%p|%p|ANY_ALL\n", body, text_ptr)); \ + MACRO(1, any_all) \ + break; \ + } \ + case SRE_OP_BIGCHARSET: \ + { \ + /* Match character in charset. */ \ + /* */ \ + TRACE(("|%p|%p|BIGCHARSET\n", body, text_ptr)); \ + MACRO(in_bigcharset(body, text_ptr[0]), bigcharset) \ + break; \ + } \ + case SRE_OP_BIGCHARSET_IGNORE: \ + { \ + /* Match character not in charset. */ \ + /* */ \ + TRACE(("|%p|%p|BIGCHARSET_IGNORE\n", body, text_ptr)); \ + MACRO(in_bigcharset(body, state->lower(text_ptr[0])), bigcharset_ignore) \ + break; \ + } \ + case SRE_OP_CHARSET: \ + { \ + /* Match character in charset. */ \ + /* */ \ + TRACE(("|%p|%p|CHARSET\n", body, text_ptr)); \ + MACRO(in_charset(body, text_ptr[0]), charset) \ + break; \ + } \ + case SRE_OP_CHARSET_IGNORE: \ + { \ + /* Match character in charset, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|CHARSET_IGNORE\n", body, text_ptr)); \ + MACRO(in_charset(body, state->lower(text_ptr[0])), charset_ignore) \ + break; \ + } \ + case SRE_OP_DIGIT: \ + { \ + /* Digit. */ \ + /* */ \ + TRACE(("|%p|%p|DIGIT\n", body, text_ptr)); \ + MACRO(SRE_IS_DIGIT(text_ptr[0]), digit) \ + break; \ + } \ + case SRE_OP_IN: \ + { \ + /* Match set member. */ \ + /* */ \ + TRACE(("|%p|%p|IN\n", body, text_ptr)); \ + MACRO(SRE_IN(body, text_ptr[0]), in) \ + break; \ + } \ + case SRE_OP_IN_IGNORE: \ + { \ + /* Match set member, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|IN\n", body, text_ptr)); \ + MACRO(SRE_IN(body, state->lower(text_ptr[0])), in_ignore) \ + break; \ + } \ + case SRE_OP_LITERAL: \ + { \ + /* Match literal character. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|LITERAL %d\n", body, text_ptr, body[0])); \ + MACRO(text_ptr[0] == chr, literal) \ + break; \ + } \ + case SRE_OP_LITERAL_IGNORE: \ + { \ + /* Match literal character, ignoring case. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|LITERAL %d\n", body, text_ptr, body[0])); \ + MACRO(state->lower(text_ptr[0]) == chr, literal_ignore) \ + break; \ + } \ + case SRE_OP_LOC_NOT_WORD: \ + { \ + /* Not word. */ \ + /* */ \ + TRACE(("|%p|%p|LOC_NOT_WORD\n", body, text_ptr)); \ + MACRO(!SRE_LOC_IS_WORD(text_ptr[0]), loc_not_word) \ + break; \ + } \ + case SRE_OP_LOC_WORD: \ + { \ + /* Word. */ \ + /* */ \ + TRACE(("|%p|%p|LOC_WORD\n", body, text_ptr)); \ + MACRO(SRE_LOC_IS_WORD(text_ptr[0]), loc_word) \ + break; \ + } \ + case SRE_OP_NOT_BIGCHARSET: \ + { \ + /* Match character not in charset. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_BIGCHARSET\n", body, text_ptr)); \ + MACRO(!in_bigcharset(body, text_ptr[0]), not_bigcharset) \ + break; \ + } \ + case SRE_OP_NOT_BIGCHARSET_IGNORE: \ + { \ + /* Match character not in charset, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_BIGCHARSET_IGNORE\n", body, text_ptr)); \ + MACRO(!in_bigcharset(body, state->lower(text_ptr[0])), not_bigcharset_ignore) \ + break; \ + } \ + case SRE_OP_NOT_CHARSET: \ + { \ + /* Match character not in charset. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_CHARSET\n", body, text_ptr)); \ + MACRO(!in_charset(body, text_ptr[0]), not_charset) \ + break; \ + } \ + case SRE_OP_NOT_CHARSET_IGNORE: \ + { \ + /* Match character not in charset, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_CHARSET_IGNORE\n", body, text_ptr)); \ + MACRO(!in_charset(body, state->lower(text_ptr[0])), not_charset_ignore) \ + break; \ + } \ + case SRE_OP_NOT_DIGIT: \ + { \ + /* Not digit. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_DIGIT\n", body, text_ptr)); \ + MACRO(!SRE_IS_DIGIT(text_ptr[0]), not_digit) \ + break; \ + } \ + case SRE_OP_NOT_IN: \ + { \ + /* Match not set member. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_IN\n", body, text_ptr)); \ + MACRO(!SRE_IN(body, text_ptr[0]), not_in) \ + break; \ + } \ + case SRE_OP_NOT_IN_IGNORE: \ + { \ + /* Match not set member, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_IN\n", body, text_ptr)); \ + MACRO(!SRE_IN(body, state->lower(text_ptr[0])), not_in_ignore) \ + break; \ + } \ + case SRE_OP_NOT_LITERAL: \ + { \ + /* Match not literal character. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|NOT_LITERAL %d\n", body, text_ptr, body[0])); \ + MACRO(text_ptr[0] != chr, not_literal) \ + break; \ + } \ + case SRE_OP_NOT_LITERAL_IGNORE: \ + { \ + /* Match not literal character, ignoring case. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|NOT_LITERAL %d\n", body, text_ptr, body[0])); \ + MACRO(state->lower(text_ptr[0]) != chr, not_literal_ignore) \ + break; \ + } \ + case SRE_OP_NOT_RANGE: \ + { \ + /* Match not range character. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_RANGE %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(!SRE_IN_RANGE(text_ptr[0], body[0], body[1]), not_range) \ + break; \ + } \ + case SRE_OP_NOT_RANGE_IGNORE: \ + { \ + /* Match not range character, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_RANGE_IGNORE %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(!SRE_IN_RANGE(state->lower(text_ptr[0]), body[0], body[1]), not_range_ignore) \ + break; \ + } \ + case SRE_OP_NOT_WHITESPACE: \ + { \ + /* Not whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_WHITESPACE\n", body, text_ptr)); \ + MACRO(!SRE_IS_WHITESPACE(text_ptr[0]), not_whitespace) \ + break; \ + } \ + case SRE_OP_NOT_WORD: \ + { \ + /* Not word. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_WORD\n", body, text_ptr)); \ + MACRO(!SRE_IS_WORD(text_ptr[0]), not_word) \ + break; \ + } \ + case SRE_OP_RANGE: \ + { \ + /* Match range character. */ \ + /* */ \ + TRACE(("|%p|%p|RANGE %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(SRE_IN_RANGE(text_ptr[0], body[0], body[1]), range) \ + break; \ + } \ + case SRE_OP_RANGE_IGNORE: \ + { \ + /* Match range character, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|RANGE_IGNORE %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(SRE_IN_RANGE(state->lower(text_ptr[0]), body[0], body[1]), range_ignore) \ + break; \ + } \ + case SRE_OP_UNI_DIGIT: \ + { \ + /* Digit. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_DIGIT\n", body, text_ptr)); \ + MACRO(SRE_UNI_IS_DIGIT(text_ptr[0]), uni_digit) \ + break; \ + } \ + case SRE_OP_UNI_NOT_DIGIT: \ + { \ + /* Not digit. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_NOT_DIGIT\n", body, text_ptr)); \ + MACRO(!SRE_UNI_IS_DIGIT(text_ptr[0]), uni_not_digit) \ + break; \ + } \ + case SRE_OP_UNI_NOT_WHITESPACE: \ + { \ + /* Not whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_NOT_WHITESPACE\n", body, text_ptr)); \ + MACRO(!SRE_UNI_IS_WHITESPACE(text_ptr[0]), uni_not_whitespace) \ + break; \ + } \ + case SRE_OP_UNI_NOT_WORD: \ + { \ + /* Not word. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_NOT_WORD\n", body, text_ptr)); \ + MACRO(!SRE_UNI_IS_WORD(text_ptr[0]), uni_not_word) \ + break; \ + } \ + case SRE_OP_UNI_WHITESPACE: \ + { \ + /* Whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_WHITESPACE\n", body, text_ptr)); \ + MACRO(SRE_UNI_IS_WHITESPACE(text_ptr[0]), uni_whitespace) \ + break; \ + } \ + case SRE_OP_UNI_WORD: \ + { \ + /* Word. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_WORD\n", body, text_ptr)); \ + MACRO(SRE_UNI_IS_WORD(text_ptr[0]), uni_word) \ + break; \ + } \ + case SRE_OP_WHITESPACE: \ + { \ + /* Whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|WHITESPACE\n", body, text_ptr)); \ + MACRO(SRE_IS_WHITESPACE(text_ptr[0]), whitespace) \ + break; \ + } \ + case SRE_OP_WORD: \ + { \ + /* Word. */ \ + /* */ \ + TRACE(("|%p|%p|WORD\n", body, text_ptr)); \ + MACRO(SRE_IS_WORD(text_ptr[0]), word) \ + break; \ + } \ + } \ } -#endif -/* The macros below should be used to protect recursive SRE_MATCH() - * calls that *failed* and do *not* return immediately (IOW, those - * that will backtrack). Explaining: - * - * - Recursive SRE_MATCH() returned true: that's usually a success - * (besides atypical cases like ASSERT_NOT), therefore there's no - * reason to restore lastmark; - * - * - Recursive SRE_MATCH() returned false but the current SRE_MATCH() - * is returning to the caller: If the current SRE_MATCH() is the - * top function of the recursion, returning false will be a matching - * failure, and it doesn't matter where lastmark is pointing to. - * If it's *not* the top function, it will be a recursive SRE_MATCH() - * failure by itself, and the calling SRE_MATCH() will have to deal - * with the failure by the same rules explained here (it will restore - * lastmark by itself if necessary); - * - * - Recursive SRE_MATCH() returned false, and will continue the - * outside 'for' loop: must be protected when breaking, since the next - * OP could potentially depend on lastmark; - * - * - Recursive SRE_MATCH() returned false, and will be called again - * inside a local for/while loop: must be protected between each - * loop iteration, since the recursive SRE_MATCH() could do anything, - * and could potentially depend on lastmark. - * - * For more information, check the discussion at SF patch #712900. - */ -#define LASTMARK_SAVE() \ - do { \ - ctx->lastmark = state->lastmark; \ - ctx->lastindex = state->lastindex; \ - } while (0) -#define LASTMARK_RESTORE() \ - do { \ - state->lastmark = ctx->lastmark; \ - state->lastindex = ctx->lastindex; \ - } while (0) - -#define RETURN_ERROR(i) do { return i; } while(0) -#define RETURN_FAILURE do { ret = 0; goto exit; } while(0) -#define RETURN_SUCCESS do { ret = 1; goto exit; } while(0) - -#define RETURN_ON_ERROR(i) \ - do { if (i < 0) RETURN_ERROR(i); } while (0) -#define RETURN_ON_SUCCESS(i) \ - do { RETURN_ON_ERROR(i); if (i > 0) RETURN_SUCCESS; } while (0) -#define RETURN_ON_FAILURE(i) \ - do { RETURN_ON_ERROR(i); if (i == 0) RETURN_FAILURE; } while (0) - -#define SFY(x) #x - -#define DATA_STACK_ALLOC(state, type, ptr) \ -do { \ - alloc_pos = state->data_stack_base; \ - TRACE(("allocating %s in %d (%d)\n", \ - SFY(type), alloc_pos, sizeof(type))); \ - if (state->data_stack_size < alloc_pos+sizeof(type)) { \ - int j = data_stack_grow(state, sizeof(type)); \ - if (j < 0) return j; \ - if (ctx_pos != -1) \ - DATA_STACK_LOOKUP_AT(state, SRE_MATCH_CONTEXT, ctx, ctx_pos); \ - } \ - ptr = (type*)(state->data_stack+alloc_pos); \ - state->data_stack_base += sizeof(type); \ -} while (0) - -#define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ -do { \ - TRACE(("looking up %s at %d\n", SFY(type), pos)); \ - ptr = (type*)(state->data_stack+pos); \ -} while (0) - -#define DATA_STACK_PUSH(state, data, size) \ -do { \ - TRACE(("copy data in %p to %d (%d)\n", \ - data, state->data_stack_base, size)); \ - if (state->data_stack_size < state->data_stack_base+size) { \ - int j = data_stack_grow(state, size); \ - if (j < 0) return j; \ - if (ctx_pos != -1) \ - DATA_STACK_LOOKUP_AT(state, SRE_MATCH_CONTEXT, ctx, ctx_pos); \ - } \ - memcpy(state->data_stack+state->data_stack_base, data, size); \ - state->data_stack_base += size; \ -} while (0) - -#define DATA_STACK_POP(state, data, size, discard) \ -do { \ - TRACE(("copy data to %p from %d (%d)\n", \ - data, state->data_stack_base-size, size)); \ - memcpy(data, state->data_stack+state->data_stack_base-size, size); \ - if (discard) \ - state->data_stack_base -= size; \ -} while (0) - -#define DATA_STACK_POP_DISCARD(state, size) \ -do { \ - TRACE(("discard data from %d (%d)\n", \ - state->data_stack_base-size, size)); \ - state->data_stack_base -= size; \ -} while(0) - -#define DATA_PUSH(x) \ - DATA_STACK_PUSH(state, (x), sizeof(*(x))) -#define DATA_POP(x) \ - DATA_STACK_POP(state, (x), sizeof(*(x)), 1) -#define DATA_POP_DISCARD(x) \ - DATA_STACK_POP_DISCARD(state, sizeof(*(x))) -#define DATA_ALLOC(t,p) \ - DATA_STACK_ALLOC(state, t, p) -#define DATA_LOOKUP_AT(t,p,pos) \ - DATA_STACK_LOOKUP_AT(state,t,p,pos) - -#define MARK_PUSH(lastmark) \ - do if (lastmark > 0) { \ - i = lastmark; /* ctx->lastmark may change if reallocated */ \ - DATA_STACK_PUSH(state, state->mark, (i+1)*sizeof(void*)); \ - } while (0) -#define MARK_POP(lastmark) \ - do if (lastmark > 0) { \ - DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 1); \ - } while (0) -#define MARK_POP_KEEP(lastmark) \ - do if (lastmark > 0) { \ - DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 0); \ - } while (0) -#define MARK_POP_DISCARD(lastmark) \ - do if (lastmark > 0) { \ - DATA_STACK_POP_DISCARD(state, (lastmark+1)*sizeof(void*)); \ - } while (0) - -#define JUMP_NONE 0 -#define JUMP_MAX_UNTIL_1 1 -#define JUMP_MAX_UNTIL_2 2 -#define JUMP_MAX_UNTIL_3 3 -#define JUMP_MIN_UNTIL_1 4 -#define JUMP_MIN_UNTIL_2 5 -#define JUMP_MIN_UNTIL_3 6 -#define JUMP_REPEAT 7 -#define JUMP_REPEAT_ONE_1 8 -#define JUMP_REPEAT_ONE_2 9 -#define JUMP_MIN_REPEAT_ONE 10 -#define JUMP_BRANCH 11 -#define JUMP_ASSERT 12 -#define JUMP_ASSERT_NOT 13 - -#define DO_JUMP(jumpvalue, jumplabel, nextpattern) \ - DATA_ALLOC(SRE_MATCH_CONTEXT, nextctx); \ - nextctx->last_ctx_pos = ctx_pos; \ - nextctx->jump = jumpvalue; \ - nextctx->pattern = nextpattern; \ - ctx_pos = alloc_pos; \ - ctx = nextctx; \ - goto entrance; \ - jumplabel: \ - while (0) /* gcc doesn't like labels at end of scopes */ \ - -typedef struct { - Py_ssize_t last_ctx_pos; - Py_ssize_t jump; - SRE_CHAR* ptr; - SRE_CODE* pattern; - Py_ssize_t count; - Py_ssize_t lastmark; - Py_ssize_t lastindex; - union { - SRE_CODE chr; - SRE_REPEAT* rep; - } u; -} SRE_MATCH_CONTEXT; +#define REPEAT_ONE_REV_MACRO(MACRO) { \ + switch (*body++) { \ + case SRE_OP_ANY_REV: \ + { \ + /* Match anything (except a newline). */ \ + /* */ \ + TRACE(("|%p|%p|ANY_REV\n", body, text_ptr)); \ + MACRO(!SRE_IS_LINEBREAK(text_ptr[-1]), any) \ + break; \ + } \ + case SRE_OP_ANY_ALL_REV: \ + { \ + /* Match anything. */ \ + /* */ \ + TRACE(("|%p|%p|ANY_ALL_REV\n", body, text_ptr)); \ + MACRO(1, any_all) \ + break; \ + } \ + case SRE_OP_BIGCHARSET_REV: \ + { \ + /* Match character in charset. */ \ + /* */ \ + TRACE(("|%p|%p|BIGCHARSET_REV\n", body, text_ptr)); \ + MACRO(in_bigcharset(body, text_ptr[-1]), bigcharset) \ + break; \ + } \ + case SRE_OP_BIGCHARSET_IGNORE_REV: \ + { \ + /* Match character not in charset. */ \ + /* */ \ + TRACE(("|%p|%p|BIGCHARSET_IGNORE_REV\n", body, text_ptr)); \ + MACRO(in_bigcharset(body, state->lower(text_ptr[-1])), bigcharset_ignore) \ + break; \ + } \ + case SRE_OP_CHARSET_REV: \ + { \ + /* Match character in charset. */ \ + /* */ \ + TRACE(("|%p|%p|CHARSET_REV\n", body, text_ptr)); \ + MACRO(in_charset(body, text_ptr[-1]), charset) \ + break; \ + } \ + case SRE_OP_CHARSET_IGNORE_REV: \ + { \ + /* Match character in charset, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|CHARSET_IGNORE_REV\n", body, text_ptr)); \ + MACRO(in_charset(body, state->lower(text_ptr[-1])), charset_ignore) \ + break; \ + } \ + case SRE_OP_DIGIT_REV: \ + { \ + /* Digit. */ \ + /* */ \ + TRACE(("|%p|%p|DIGIT_REV\n", body, text_ptr)); \ + MACRO(SRE_IS_DIGIT(text_ptr[-1]), digit) \ + break; \ + } \ + case SRE_OP_IN_REV: \ + { \ + /* Match set member. */ \ + /* */ \ + TRACE(("|%p|%p|IN_REV\n", body, text_ptr)); \ + MACRO(SRE_IN(body, text_ptr[-1]), in) \ + break; \ + } \ + case SRE_OP_IN_IGNORE_REV: \ + { \ + /* Match set member, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|IN_REV\n", body, text_ptr)); \ + MACRO(SRE_IN(body, state->lower(text_ptr[-1])), in_ignore) \ + break; \ + } \ + case SRE_OP_LITERAL_REV: \ + { \ + /* Match literal character. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|LITERAL_REV %d\n", body, text_ptr, body[0])); \ + MACRO(text_ptr[-1] == chr, literal) \ + break; \ + } \ + case SRE_OP_LITERAL_IGNORE_REV: \ + { \ + /* Match literal character, ignoring case. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|LITERAL_REV %d\n", body, text_ptr, body[0])); \ + MACRO(state->lower(text_ptr[-1]) == chr, literal_ignore) \ + break; \ + } \ + case SRE_OP_LOC_NOT_WORD_REV: \ + { \ + /* Not word. */ \ + /* */ \ + TRACE(("|%p|%p|LOC_NOT_WORD_REV\n", body, text_ptr)); \ + MACRO(!SRE_LOC_IS_WORD(text_ptr[-1]), loc_not_word) \ + break; \ + } \ + case SRE_OP_LOC_WORD_REV: \ + { \ + /* Word. */ \ + /* */ \ + TRACE(("|%p|%p|LOC_WORD_REV\n", body, text_ptr)); \ + MACRO(SRE_LOC_IS_WORD(text_ptr[-1]), loc_word) \ + break; \ + } \ + case SRE_OP_NOT_BIGCHARSET_REV: \ + { \ + /* Match character not in charset. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_BIGCHARSET_REV\n", body, text_ptr)); \ + MACRO(!in_bigcharset(body, text_ptr[-1]), not_bigcharset) \ + break; \ + } \ + case SRE_OP_NOT_BIGCHARSET_IGNORE_REV: \ + { \ + /* Match character not in charset, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_BIGCHARSET_IGNORE_REV\n", body, text_ptr)); \ + MACRO(!in_bigcharset(body, state->lower(text_ptr[-1])), not_bigcharset_ignore) \ + break; \ + } \ + case SRE_OP_NOT_CHARSET_REV: \ + { \ + /* Match character not in charset. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_CHARSET_REV\n", body, text_ptr)); \ + MACRO(!in_charset(body, text_ptr[-1]), not_charset) \ + break; \ + } \ + case SRE_OP_NOT_CHARSET_IGNORE_REV: \ + { \ + /* Match character not in charset, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_CHARSET_IGNORE_REV\n", body, text_ptr)); \ + MACRO(!in_charset(body, state->lower(text_ptr[-1])), not_charset_ignore) \ + break; \ + } \ + case SRE_OP_NOT_DIGIT_REV: \ + { \ + /* Not digit. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_DIGIT_REV\n", body, text_ptr)); \ + MACRO(!SRE_IS_DIGIT(text_ptr[-1]), not_digit) \ + break; \ + } \ + case SRE_OP_NOT_IN_REV: \ + { \ + /* Match not set member. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_IN_REV\n", body, text_ptr)); \ + MACRO(!SRE_IN(body, text_ptr[-1]), not_in) \ + break; \ + } \ + case SRE_OP_NOT_IN_IGNORE_REV: \ + { \ + /* Match not set member, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_IN_REV\n", body, text_ptr)); \ + MACRO(!SRE_IN(body, state->lower(text_ptr[-1])), not_in_ignore) \ + break; \ + } \ + case SRE_OP_NOT_LITERAL_REV: \ + { \ + /* Match not literal character. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|NOT_LITERAL_REV %d\n", body, text_ptr, body[0])); \ + MACRO(text_ptr[-1] != chr, not_literal) \ + break; \ + } \ + case SRE_OP_NOT_LITERAL_IGNORE_REV: \ + { \ + /* Match not literal character, ignoring case. */ \ + /* */ \ + SRE_CHAR chr = (SRE_CHAR)body[0]; \ + TRACE(("|%p|%p|NOT_LITERAL_REV %d\n", body, text_ptr, body[0])); \ + MACRO(state->lower(text_ptr[-1]) != chr, not_literal_ignore) \ + break; \ + } \ + case SRE_OP_NOT_RANGE_REV: \ + { \ + /* Match not range character. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_RANGE_REV %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(!SRE_IN_RANGE(text_ptr[-1], body[0], body[1]), not_range) \ + break; \ + } \ + case SRE_OP_NOT_RANGE_IGNORE_REV: \ + { \ + /* Match not range character, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_RANGE_IGNORE_REV %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(!SRE_IN_RANGE(state->lower(text_ptr[-1]), body[0], body[1]), not_range_ignore) \ + break; \ + } \ + case SRE_OP_NOT_WHITESPACE_REV: \ + { \ + /* Not whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_WHITESPACE_REV\n", body, text_ptr)); \ + MACRO(!SRE_IS_WHITESPACE(text_ptr[-1]), not_whitespace) \ + break; \ + } \ + case SRE_OP_NOT_WORD_REV: \ + { \ + /* Not word. */ \ + /* */ \ + TRACE(("|%p|%p|NOT_WORD_REV\n", body, text_ptr)); \ + MACRO(!SRE_IS_WORD(text_ptr[-1]), not_word) \ + break; \ + } \ + case SRE_OP_RANGE_REV: \ + { \ + /* Match range character. */ \ + /* */ \ + TRACE(("|%p|%p|RANGE_REV %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(SRE_IN_RANGE(text_ptr[-1], body[0], body[1]), range) \ + break; \ + } \ + case SRE_OP_RANGE_IGNORE_REV: \ + { \ + /* Match range character, ignoring case. */ \ + /* */ \ + TRACE(("|%p|%p|RANGE_IGNORE_REV %d %d\n", body, text_ptr, body[0], body[1])); \ + MACRO(SRE_IN_RANGE(state->lower(text_ptr[-1]), body[0], body[1]), range_ignore) \ + break; \ + } \ + case SRE_OP_UNI_DIGIT_REV: \ + { \ + /* Digit. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_DIGIT_REV\n", body, text_ptr)); \ + MACRO(SRE_UNI_IS_DIGIT(text_ptr[-1]), uni_digit) \ + break; \ + } \ + case SRE_OP_UNI_NOT_DIGIT_REV: \ + { \ + /* Not digit. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_NOT_DIGIT_REV\n", body, text_ptr)); \ + MACRO(!SRE_UNI_IS_DIGIT(text_ptr[-1]), uni_not_digit) \ + break; \ + } \ + case SRE_OP_UNI_NOT_WHITESPACE_REV: \ + { \ + /* Not whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_NOT_WHITESPACE_REV\n", body, text_ptr)); \ + MACRO(!SRE_UNI_IS_WHITESPACE(text_ptr[-1]), uni_not_whitespace) \ + break; \ + } \ + case SRE_OP_UNI_NOT_WORD_REV: \ + { \ + /* Not word. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_NOT_WORD_REV\n", body, text_ptr)); \ + MACRO(!SRE_UNI_IS_WORD(text_ptr[-1]), uni_not_word) \ + break; \ + } \ + case SRE_OP_UNI_WHITESPACE_REV: \ + { \ + /* Whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_WHITESPACE_REV\n", body, text_ptr)); \ + MACRO(SRE_UNI_IS_WHITESPACE(text_ptr[-1]), uni_whitespace) \ + break; \ + } \ + case SRE_OP_UNI_WORD_REV: \ + { \ + /* Word. */ \ + /* */ \ + TRACE(("|%p|%p|UNI_WORD_REV\n", body, text_ptr)); \ + MACRO(SRE_UNI_IS_WORD(text_ptr[-1]), uni_word) \ + break; \ + } \ + case SRE_OP_WHITESPACE_REV: \ + { \ + /* Whitespace. */ \ + /* */ \ + TRACE(("|%p|%p|WHITESPACE_REV\n", body, text_ptr)); \ + MACRO(SRE_IS_WHITESPACE(text_ptr[-1]), whitespace) \ + break; \ + } \ + case SRE_OP_WORD_REV: \ + { \ + /* Word. */ \ + /* */ \ + TRACE(("|%p|%p|WORD_REV\n", body, text_ptr)); \ + MACRO(SRE_IS_WORD(text_ptr[-1]), word) \ + break; \ + } \ + } \ +} /* check if string matches the given pattern. returns <0 for error, 0 for failure, and 1 for success */ LOCAL(Py_ssize_t) -SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) -{ - SRE_CHAR* end = (SRE_CHAR *)state->end; - Py_ssize_t alloc_pos, ctx_pos = -1; - Py_ssize_t i, ret = 0; - Py_ssize_t jump; - unsigned int sigcount=0; +SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern_ptr) { + SRE_CHAR* text_start = (SRE_CHAR *)state->beginning; + SRE_CHAR* text_end = (SRE_CHAR *)state->end; + SRE_CHAR* text_ptr = (SRE_CHAR *)state->ptr; + int mark_count = state->pattern_code[0]; + SRE_CHAR** mark = (SRE_CHAR**)state->mark; + int* repeat_counts = state->repeat_counts; + SRE_CHAR** repeat_starts = (SRE_CHAR**)state->repeat_starts; + SRE_BACKTRACK_CHUNK* backtrack_chunk = state->backtrack_chunk; + SRE_SAVED_MARKS_CHUNK* saved_marks_chunk = state->saved_marks_chunk; + SRE_CHAR* final_linebreak; + unsigned int sigcount = 0; - SRE_MATCH_CONTEXT* ctx; - SRE_MATCH_CONTEXT* nextctx; + // Point to the final newline if it's the final character. + final_linebreak = text_start < text_end && SRE_IS_LINEBREAK(text_end[-1]) ? text_end - 1 : NULL; TRACE(("|%p|%p|ENTER\n", pattern, state->ptr)); + SAVE_BACKTRACK(SRE_OP_FAILURE, NULL, 0) + memset(mark, 0, mark_count * sizeof(SRE_CHAR*)); - DATA_ALLOC(SRE_MATCH_CONTEXT, ctx); - ctx->last_ctx_pos = -1; - ctx->jump = JUMP_NONE; - ctx->pattern = pattern; - ctx_pos = alloc_pos; - -entrance: - - ctx->ptr = (SRE_CHAR *)state->ptr; - - if (ctx->pattern[0] == SRE_OP_INFO) { - /* optimization info block */ - /* <1=skip> <2=flags> <3=min> ... */ - if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { - TRACE(("reject (got %d chars, need %d)\n", - (end - ctx->ptr), ctx->pattern[3])); - RETURN_FAILURE; +advance: + for (;;) { + ++sigcount; + if ((0 == (sigcount & 0xfff)) && PyErr_CheckSignals()) + SRE_RETURN(SRE_ERROR_INTERRUPTED) + + switch (*pattern_ptr++) { + case SRE_OP_ANY: + // Match anything (except a newline). + // + TRACE(("|%p|%p|ANY\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_IS_LINEBREAK(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_ANY_ALL: + // Match anything. + // + TRACE(("|%p|%p|ANY_ALL\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end) + goto backtrack; + text_ptr++; + break; + case SRE_OP_ANY_ALL_REV: + // Match anything. + // + TRACE(("|%p|%p|ANY_ALL_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start) + goto backtrack; + text_ptr--; + break; + case SRE_OP_ANY_REV: + // Match anything (except a newline). + // + TRACE(("|%p|%p|ANY_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_IS_LINEBREAK(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_ASSERT: + // Assert subpattern. + // ... + TRACE(("|%p|%p|ASSERT\n", pattern_ptr, text_ptr)); + SAVE_BACKTRACK(SRE_OP_ASSERT, pattern_ptr, 0) + SAVE_MARKS() + pattern_ptr++; + break; + case SRE_OP_ASSERT_NOT: + // Assert not subpattern. + // ... + TRACE(("|%p|%p|ASSERT_NOT\n", pattern_ptr, text_ptr)); + SAVE_BACKTRACK(SRE_OP_ASSERT_NOT, pattern_ptr, 0) + SAVE_MARKS() + pattern_ptr++; + break; + case SRE_OP_ATOMIC: + // Atomic subpattern. + // ... + TRACE(("|%p|%p|ATOMIC\n", pattern_ptr, text_ptr)); + SAVE_BACKTRACK(SRE_OP_ATOMIC, pattern_ptr, 0) + SAVE_MARKS() + pattern_ptr++; + break; + case SRE_OP_BIGCHARSET: + // Match character in charset. + // + TRACE(("|%p|%p|BIGCHARSET\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !in_bigcharset(pattern_ptr, text_ptr[0])) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr++; + break; + case SRE_OP_BIGCHARSET_IGNORE: + // Match character in charset, ignoring case. + // + TRACE(("|%p|%p|BIGCHARSET\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !in_bigcharset(pattern_ptr, state->lower(text_ptr[0]))) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr++; + break; + case SRE_OP_BIGCHARSET_IGNORE_REV: + // Match character in charset, ignoring case. + // + TRACE(("|%p|%p|BIGCHARSET_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !in_bigcharset(pattern_ptr, state->lower(text_ptr[-1]))) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr--; + break; + case SRE_OP_BIGCHARSET_REV: + // Match character in charset. + // + TRACE(("|%p|%p|BIGCHARSET_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !in_bigcharset(pattern_ptr, text_ptr[-1])) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr--; + break; + case SRE_OP_BOUNDARY: + { + // Boundary between word and non-word. + // + int before = text_ptr > text_start && SRE_IS_WORD(text_ptr[-1]); + int after = text_ptr < text_end && SRE_IS_WORD(text_ptr[0]); + TRACE(("|%p|%p|BOUNDARY\n", pattern_ptr, text_ptr)); + if (before == after) + goto backtrack; + break; + } + case SRE_OP_BRANCH: + // Alternation. + // ... ... 0 + TRACE(("|%p|%p|BRANCH\n", pattern_ptr, text_ptr)); + SAVE_BACKTRACK(SRE_OP_BRANCH, pattern_ptr, 0) + SAVE_MARKS() + pattern_ptr++; + break; + case SRE_OP_CHARSET: + // Match character in charset. + // + TRACE(("|%p|%p|CHARSET\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !in_charset(pattern_ptr, text_ptr[0])) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr++; + break; + case SRE_OP_CHARSET_IGNORE: + // Match character in charset, ignoring case. + // + TRACE(("|%p|%p|CHARSET\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !in_charset(pattern_ptr, state->lower(text_ptr[0]))) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr++; + break; + case SRE_OP_CHARSET_IGNORE_REV: + // Match character in charset, ignoring case. + // + TRACE(("|%p|%p|CHARSET_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !in_charset(pattern_ptr, state->lower(text_ptr[-1]))) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr--; + break; + case SRE_OP_CHARSET_REV: + // Match character in charset. + // + TRACE(("|%p|%p|CHARSET_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !in_charset(pattern_ptr, text_ptr[-1])) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr--; + break; + case SRE_OP_DIGIT: + // Digit. + // + TRACE(("|%p|%p|DIGIT\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_IS_DIGIT(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_DIGIT_REV: + // Digit. + // + TRACE(("|%p|%p|DIGIT_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_IS_DIGIT(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_END_ASSERT: + { + // Assert subpattern. + // ... + int done = 0; + TRACE(("|%p|%p|END_ASSERT\n", pattern_ptr, text_ptr)); + do { + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[-- backtrack_chunk->count]; + switch(backtrack_item->op) { + case SRE_OP_ASSERT: + // Assert subpattern. + // ... + text_ptr = backtrack_item->text_ptr; + done = 1; + break; + case SRE_OP_BRANCH: + case SRE_OP_END_REPEAT_MAX: + case SRE_OP_END_REPEAT_MAX_REV: + case SRE_OP_END_REPEAT_MIN: + case SRE_OP_END_REPEAT_MIN_REV: + case SRE_OP_END_REPEAT_POSS: + case SRE_OP_END_REPEAT_POSS_REV: + case SRE_OP_REPEAT_MAX: + case SRE_OP_REPEAT_MAX_REV: + case SRE_OP_REPEAT_MIN: + case SRE_OP_REPEAT_MIN_REV: + case SRE_OP_REPEAT_POSS: + case SRE_OP_REPEAT_POSS_REV: + { + DISCARD_SAVED_MARKS() + break; + } + } + } while (!done); + RESTORE_MARKS() + break; + } + case SRE_OP_END_ASSERT_NOT: + { + // Assert not subpattern. + // ... + int done = 0; + TRACE(("|%p|%p|END_ASSERT_NOT\n", pattern_ptr, text_ptr)); + do { + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[-- backtrack_chunk->count]; + switch(backtrack_item->op) { + case SRE_OP_ASSERT_NOT: + // Assert not subpattern. + // ... + done = 1; + break; + case SRE_OP_BRANCH: + case SRE_OP_END_REPEAT_MAX: + case SRE_OP_END_REPEAT_MAX_REV: + case SRE_OP_END_REPEAT_MIN: + case SRE_OP_END_REPEAT_MIN_REV: + case SRE_OP_END_REPEAT_POSS: + case SRE_OP_END_REPEAT_POSS_REV: + case SRE_OP_REPEAT_MAX: + case SRE_OP_REPEAT_MAX_REV: + case SRE_OP_REPEAT_MIN: + case SRE_OP_REPEAT_MIN_REV: + case SRE_OP_REPEAT_POSS: + case SRE_OP_REPEAT_POSS_REV: + { + DISCARD_SAVED_MARKS() + break; + } + } + } while (!done); + RESTORE_MARKS() + goto backtrack; + } + case SRE_OP_END_ATOMIC: + { + // Atomic subpattern. + // ... + int done = 0; + do { + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + switch(backtrack_item->op) { + case SRE_OP_ATOMIC: + done = 1; + break; + case SRE_OP_BRANCH: + case SRE_OP_END_REPEAT_MAX: + case SRE_OP_END_REPEAT_MAX_REV: + case SRE_OP_END_REPEAT_MIN: + case SRE_OP_END_REPEAT_MIN_REV: + case SRE_OP_END_REPEAT_POSS: + case SRE_OP_END_REPEAT_POSS_REV: + case SRE_OP_REPEAT_MAX: + case SRE_OP_REPEAT_MAX_REV: + case SRE_OP_REPEAT_MIN: + case SRE_OP_REPEAT_MIN_REV: + case SRE_OP_REPEAT_POSS: + case SRE_OP_REPEAT_POSS_REV: + { + DISCARD_SAVED_MARKS() + DISCARD_BACKTRACK(0) + break; + } + } + } while (!done); + break; + } + case SRE_OP_END_OF_LINE: + // End of line. + // + TRACE(("|%p|%p|END_OF_LINE\n", pattern_ptr, text_ptr)); + if (text_ptr < text_end && !SRE_IS_LINEBREAK(text_ptr[0])) + goto backtrack; + break; + case SRE_OP_END_OF_STRING: + // End of string. + // + TRACE(("|%p|%p|END_OF_STRING\n", pattern_ptr, text_ptr)); + if (text_ptr < text_end) + goto backtrack; + break; + case SRE_OP_END_OF_STRING_2: + // End of string or final line. + // + TRACE(("|%p|%p|END_OF_STRING_2\n", pattern_ptr, text_ptr)); + if (text_ptr < text_end && text_ptr != final_linebreak) + goto backtrack; + break; + case SRE_OP_END_REPEAT_MAX: + { + // End of greedy repeat. + // ... + SRE_CODE* repeat = pattern_ptr - pattern_ptr[0]; + SRE_CODE* end_repeat = pattern_ptr; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_end - text_ptr; + int repeat_count = ++repeat_counts[index]; + TRACE(("|%p|%p|END_REPEAT_MAX\n", pattern_ptr, text_ptr)); + if (text_ptr == repeat_starts[index]) + pattern_ptr = tail; + else if (repeat_count < repeat_min) { + if (repeat_min - repeat_count > limit) + goto backtrack; + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else { + repeat_max = sre_repeat_limit(repeat_max, repeat_count + limit); + if (repeat_count < repeat_max && limit > 0) { + SRE_CODE* look_literal = tail; + int save; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL: + save = text_ptr[0] == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING: + save = text_ptr[0] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; + } + if (save) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MAX, end_repeat, 0) + SAVE_MARKS() + } + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else + pattern_ptr = tail; + } + break; + } + case SRE_OP_END_REPEAT_MAX_REV: + { + // End of greedy repeat. + // ... + SRE_CODE* repeat = pattern_ptr - pattern_ptr[0]; + SRE_CODE* end_repeat = pattern_ptr; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_ptr - text_start; + int repeat_count = ++repeat_counts[index]; + TRACE(("|%p|%p|END_REPEAT_MAX_REV\n", pattern_ptr, text_ptr)); + if (text_ptr == repeat_starts[index]) + pattern_ptr = tail; + else if (repeat_count < repeat_min) { + if (repeat_min - repeat_count > limit) + goto backtrack; + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else { + repeat_max = sre_repeat_limit(repeat_max, repeat_count + limit); + if (repeat_count < repeat_max && limit > 0) { + SRE_CODE* look_literal = tail; + int save; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL_REV: + save = text_ptr[-1] == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_IGNORE_REV: + save = state->lower(text_ptr[-1]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING_REV: + save = text_ptr[-1] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE_REV: + save = state->lower(text_ptr[-1])== (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; + } + if (save) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MAX_REV, end_repeat, 0) + SAVE_MARKS() + } + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else + pattern_ptr = tail; + } + break; + } + case SRE_OP_END_REPEAT_MIN: + { + // End of lazy repeat. + // ... + SRE_CODE* repeat = pattern_ptr - pattern_ptr[0]; + SRE_CODE* end_repeat = pattern_ptr; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_end - text_ptr; + int repeat_count = ++repeat_counts[index]; + TRACE(("|%p|%p|END_REPEAT_MIN\n", pattern_ptr, text_ptr)); + if (text_ptr == repeat_starts[index]) + pattern_ptr = tail; + else if (repeat_count < repeat_min) { + if (repeat_min - repeat_count > limit) + goto backtrack; + RESTORE_MARKS() + SAVE_MARKS() + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else { + int save; + if (limit > 0) { + SRE_CODE* look_literal = tail; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL: + save = text_ptr[0] == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING: + save = text_ptr[0] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; + } + } else + save = 1; + repeat_max = sre_repeat_limit(repeat_max, repeat_count + limit); + if (repeat_count < repeat_max) { + if (save) { + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + backtrack_item->text_ptr = text_ptr; + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MIN, end_repeat, 0) + SAVE_MARKS() + pattern_ptr = tail; + } else { + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } + } else { + if (save) + pattern_ptr = tail; + else + goto backtrack; + } + } + break; + } + case SRE_OP_END_REPEAT_MIN_REV: + { + // End of lazy repeat. + // ... + SRE_CODE* repeat = pattern_ptr - pattern_ptr[0]; + SRE_CODE* end_repeat = pattern_ptr; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_ptr - text_start; + int repeat_count = ++repeat_counts[index]; + TRACE(("|%p|%p|END_REPEAT_MIN_REV\n", pattern_ptr, text_ptr)); + if (text_ptr == repeat_starts[index]) + pattern_ptr = tail; + else if (repeat_count < repeat_min) { + if (repeat_min - repeat_count > limit) + goto backtrack; + RESTORE_MARKS() + SAVE_MARKS() + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else { + int save; + if (limit > 0) { + SRE_CODE* look_literal = tail; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL_REV: + save = text_ptr[-1] == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_IGNORE_REV: + save = state->lower(text_ptr[-1]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING_REV: + save = text_ptr[-1] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE_REV: + save = state->lower(text_ptr[-1]) == (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; + } + } else + save = 1; + repeat_max = sre_repeat_limit(repeat_max, repeat_count + limit); + if (repeat_count < repeat_max) { + if (save) { + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + backtrack_item->text_ptr = text_ptr; + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MIN, end_repeat, 0) + SAVE_MARKS() + pattern_ptr = tail; + } else { + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } + } else { + if (save) + pattern_ptr = tail; + else + goto backtrack; + } + } + break; + } + case SRE_OP_END_REPEAT_POSS: + { + // End of possessive repeat. + // ... + SRE_CODE* repeat = pattern_ptr - pattern_ptr[0]; + SRE_CODE* end_repeat = pattern_ptr; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_end - text_ptr; + int repeat_count = ++repeat_counts[index]; + int done = 0; + TRACE(("|%p|%p|END_REPEAT_POSS\n", pattern_ptr, text_ptr)); + do { + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + switch(backtrack_item->op) { + case SRE_OP_BRANCH: + case SRE_OP_END_REPEAT_MAX: + case SRE_OP_END_REPEAT_MAX_REV: + case SRE_OP_END_REPEAT_MIN: + case SRE_OP_END_REPEAT_MIN_REV: + case SRE_OP_END_REPEAT_POSS: + case SRE_OP_END_REPEAT_POSS_REV: + case SRE_OP_REPEAT_MAX: + case SRE_OP_REPEAT_MAX_REV: + case SRE_OP_REPEAT_MIN: + case SRE_OP_REPEAT_MIN_REV: + case SRE_OP_REPEAT_POSS_REV: + { + DISCARD_SAVED_MARKS() + DISCARD_BACKTRACK(0) + break; + } + case SRE_OP_REPEAT_POSS: + { + done = 1; + break; + } + } + } while (!done); + if (repeat_starts[index] == text_ptr) + pattern_ptr = tail; + else if (repeat_count < repeat_min) { + if (repeat_min - repeat_count > limit) + goto backtrack; + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else { + repeat_max = sre_repeat_limit(repeat_max, repeat_count + limit); + if (repeat_count < repeat_max && limit > 0) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_POSS, end_repeat, 0) + SAVE_MARKS() + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else + pattern_ptr = tail; + } + break; + } + case SRE_OP_END_REPEAT_POSS_REV: + { + // End of possessive repeat. + // ... + SRE_CODE* repeat = pattern_ptr - pattern_ptr[0]; + SRE_CODE* end_repeat = pattern_ptr; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_ptr - text_start; + int repeat_count = ++repeat_counts[index]; + int done = 0; + TRACE(("|%p|%p|END_REPEAT_POSS_REV\n", pattern_ptr, text_ptr)); + do { + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + switch(backtrack_item->op) { + case SRE_OP_BRANCH: + case SRE_OP_END_REPEAT_MAX: + case SRE_OP_END_REPEAT_MAX_REV: + case SRE_OP_END_REPEAT_MIN: + case SRE_OP_END_REPEAT_MIN_REV: + case SRE_OP_END_REPEAT_POSS: + case SRE_OP_END_REPEAT_POSS_REV: + case SRE_OP_REPEAT_MAX: + case SRE_OP_REPEAT_MAX_REV: + case SRE_OP_REPEAT_MIN: + case SRE_OP_REPEAT_MIN_REV: + case SRE_OP_REPEAT_POSS: + { + DISCARD_SAVED_MARKS() + DISCARD_BACKTRACK(0) + break; + } + case SRE_OP_REPEAT_POSS_REV: + { + done = 1; + break; + } + } + } while (!done); + if (repeat_starts[index] == text_ptr) + pattern_ptr = tail; + else if (repeat_count < repeat_min) { + if (repeat_min - repeat_count > limit) + goto backtrack; + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else { + repeat_max = sre_repeat_limit(repeat_max, repeat_count + limit); + if (repeat_count < repeat_max && limit > 0) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_POSS_REV, end_repeat, 0) + SAVE_MARKS() + pattern_ptr = body; + repeat_starts[index] = text_ptr; + } else + pattern_ptr = tail; + } + break; + } + case SRE_OP_GROUPREF: + { + // Match backreference. + // + Py_ssize_t group; + SRE_CHAR* group_start; + SRE_CHAR* group_end; + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|GROUPREF %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + group = pattern_ptr[0]; // Zero-based index. Note that externally group 0 is the entire matched string. + group_start = mark[group * 2]; + group_end = mark[group * 2 + 1]; + if (group_start == NULL || group_end == NULL) + goto backtrack; + length = group_end - group_start; + if (length > text_end - text_ptr) + goto backtrack; + i = 0; + while (i < length) { + if (text_ptr[i] != group_start[i]) + goto backtrack; + i++; + } + pattern_ptr++; + text_ptr += length; + break; + } + case SRE_OP_GROUPREF_EXISTS: + { + // Whether backreference exists. + // code_yes code_no + Py_ssize_t group; + SRE_CHAR* group_start; + SRE_CHAR* group_end; + TRACE(("|%p|%p|GROUPREF_EXISTS %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + group = pattern_ptr[0]; // Zero-based index. Note that externally group 0 is the entire matched string. + group_start = mark[group * 2]; + group_end = mark[group * 2 + 1]; + if (group_start == NULL || group_end == NULL) + pattern_ptr += pattern_ptr[1]; + else + pattern_ptr += 2; + break; + } + case SRE_OP_GROUPREF_IGNORE: + { + // Match backreference, ignoring case. + // + Py_ssize_t group; + SRE_CHAR* group_start; + SRE_CHAR* group_end; + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|GROUPREF_IGNORE %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + group = pattern_ptr[0]; // Zero-based index. Note that externally group 0 is the entire matched string. + group_start = mark[group * 2]; + group_end = mark[group * 2 + 1]; + if (group_start == NULL || group_end == NULL) + goto backtrack; + length = group_end - group_start; + if (length > text_end - text_ptr) + goto backtrack; + i = 0; + while (i < length) { + if (state->lower(text_ptr[i]) != state->lower(group_start[i])) + goto backtrack; + i++; + } + pattern_ptr++; + text_ptr += length; + break; + } + case SRE_OP_GROUPREF_IGNORE_REV: + { + // Match backreference, ignoring case. + // + Py_ssize_t group; + SRE_CHAR* group_start; + SRE_CHAR* group_end; + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|GROUPREF_IGNORE_REV %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + group = pattern_ptr[0]; // Zero-based index. Note that externally group 0 is the entire matched string. + group_start = mark[group * 2]; + group_end = mark[group * 2 + 1]; + if (group_start == NULL || group_end == NULL) + goto backtrack; + length = group_end - group_start; + if (length > text_ptr - text_start) + goto backtrack; + text_ptr -= length; + i = 0; + while (i < length) { + if (state->lower(text_ptr[i]) != state->lower(group_start[i])) + goto backtrack; + i++; + } + pattern_ptr++; + break; + } + case SRE_OP_GROUPREF_REV: + { + // Match backreference. + // + Py_ssize_t group; + SRE_CHAR* group_start; + SRE_CHAR* group_end; + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|GROUPREF_REV %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + group = pattern_ptr[0]; // Zero-based index. Note that externally group 0 is the entire matched string. + group_start = mark[group * 2]; + group_end = mark[group * 2 + 1]; + if (group_start == NULL || group_end == NULL) + goto backtrack; + length = group_end - group_start; + if (length > text_ptr - text_start) + goto backtrack; + text_ptr -= length; + i = 0; + while (i < length) { + if (text_ptr[i] != group_start[i]) + goto backtrack; + i++; + } + pattern_ptr++; + break; + } + case SRE_OP_IN: + // Match set member. + // + TRACE(("|%p|%p|IN\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_IN(pattern_ptr, text_ptr[0])) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr++; + break; + case SRE_OP_IN_IGNORE: + // Match set member, ignoring case. + // + TRACE(("|%p|%p|IN_IGNORE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_IN(pattern_ptr, state->lower(text_ptr[0]))) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr++; + break; + case SRE_OP_IN_IGNORE_REV: + // Match set member, ignoring case. + // + TRACE(("|%p|%p|IN_IGNORE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_IN(pattern_ptr, state->lower(text_ptr[-1]))) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr--; + break; + case SRE_OP_IN_REV: + // Match set member. + // + TRACE(("|%p|%p|IN_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_IN(pattern_ptr, text_ptr[-1])) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr--; + break; + case SRE_OP_JUMP: + // Jump forward. + // + TRACE(("|%p|%p|JUMP %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + pattern_ptr += pattern_ptr[0]; + break; + case SRE_OP_LITERAL: + // Match literal character. + // + TRACE(("|%p|%p|LITERAL %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || (SRE_CODE) text_ptr[0] != pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr++; + break; + case SRE_OP_LITERAL_IGNORE: + // Match literal character, ignoring case. + // + TRACE(("|%p|%p|LITERAL_IGNORE %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || state->lower(text_ptr[0]) != pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr++; + break; + case SRE_OP_LITERAL_IGNORE_REV: + // Match literal character, ignoring case. + // + TRACE(("|%p|%p|LITERAL_IGNORE_REV %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || state->lower(text_ptr[-1]) != pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr--; + break; + case SRE_OP_LITERAL_REV: + // Match literal character. + // + TRACE(("|%p|%p|LITERAL_REV %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || (SRE_CODE) text_ptr[-1] != pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr--; + break; + case SRE_OP_LITERAL_STRING: + { + // Match literal string. + // ... + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|LITERAL_STRING %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + length = *pattern_ptr++; + if (text_ptr + length > text_end) + goto backtrack; + i = 0; + do { + if ((SRE_CODE) text_ptr[i] != pattern_ptr[i]) + goto backtrack; + i++; + } + while (i < length); + pattern_ptr += length; + text_ptr += length; + break; + } + case SRE_OP_LITERAL_STRING_IGNORE: + { + // Match literal string, ignoring case. + // ... + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|LITERAL_STRING_IGNORE %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + length = *pattern_ptr++; + if (length > text_end - text_ptr) + goto backtrack; + i = 0; + do { + if (state->lower(text_ptr[i]) != pattern_ptr[i]) + goto backtrack; + i++; + } + while (i < length); + pattern_ptr += length; + text_ptr += length; + break; + } + case SRE_OP_LITERAL_STRING_IGNORE_REV: + { + // Match literal string, ignoring case. + // ... + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|LITERAL_STRING_IGNORE_REV %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + length = *pattern_ptr++; + if (length > text_ptr - text_start) + goto backtrack; + text_ptr -= length; + i = 0; + do { + if (state->lower(text_ptr[i]) != pattern_ptr[i]) + goto backtrack; + i++; + } + while (i < length); + pattern_ptr += length; + break; + } + case SRE_OP_LITERAL_STRING_REV: + { + // Match literal string. + // ... + Py_ssize_t length; + Py_ssize_t i; + TRACE(("|%p|%p|LITERAL_STRING_REV %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + length = *pattern_ptr++; + if (text_ptr - length < text_start) + goto backtrack; + text_ptr -= length; + i = 0; + do { + if ((SRE_CODE) text_ptr[i] != pattern_ptr[i]) + goto backtrack; + i++; + } + while (i < length); + pattern_ptr += length; + break; + } + case SRE_OP_LOC_BOUNDARY: + { + // Boundary between word and non-word. + // + int before = text_ptr > text_start && SRE_LOC_IS_WORD(text_ptr[-1]); + int after = text_ptr < text_end && SRE_LOC_IS_WORD(text_ptr[0]); + TRACE(("|%p|%p|LOC_BOUNDARY\n", pattern_ptr, text_ptr)); + if (before == after) + goto backtrack; + break; + } + case SRE_OP_LOC_NOT_BOUNDARY: + { + // Not boundary between word and non-word. + // + int before = text_ptr > text_start && SRE_LOC_IS_WORD(text_ptr[-1]); + int after = text_ptr < text_end && SRE_LOC_IS_WORD(text_ptr[0]); + TRACE(("|%p|%p|LOC_NOT_BOUNDARY\n", pattern_ptr, text_ptr)); + if (before != after) + goto backtrack; + break; } - ctx->pattern += ctx->pattern[1] + 1; - } - - for (;;) { - ++sigcount; - if ((0 == (sigcount & 0xfff)) && PyErr_CheckSignals()) - RETURN_ERROR(SRE_ERROR_INTERRUPTED); - - switch (*ctx->pattern++) { - + case SRE_OP_LOC_NOT_WORD: + // Not word. + // + TRACE(("|%p|%p|LOC_NOT_WORD\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_LOC_IS_WORD(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_LOC_NOT_WORD_REV: + // Not word. + // + TRACE(("|%p|%p|LOC_NOT_WORD_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_LOC_IS_WORD(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_LOC_WORD: + // Word. + // + TRACE(("|%p|%p|LOC_WORD\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_LOC_IS_WORD(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_LOC_WORD_REV: + // Word. + // + TRACE(("|%p|%p|LOC_WORD_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_LOC_IS_WORD(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; case SRE_OP_MARK: - /* set mark */ - /* */ - TRACE(("|%p|%p|MARK %d\n", ctx->pattern, - ctx->ptr, ctx->pattern[0])); - i = ctx->pattern[0]; - if (i & 1) - state->lastindex = i/2 + 1; - if (i > state->lastmark) { - /* state->lastmark is the highest valid index in the - state->mark array. If it is increased by more than 1, - the intervening marks must be set to NULL to signal - that these marks have not been encountered. */ - Py_ssize_t j = state->lastmark + 1; - while (j < i) - state->mark[j++] = NULL; - state->lastmark = i; - } - state->mark[i] = ctx->ptr; - ctx->pattern++; + { + // Set mark. + // + TRACE(("|%p|%p|MARK %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + mark[pattern_ptr[0]] = text_ptr; + pattern_ptr++; break; - - case SRE_OP_LITERAL: - /* match literal string */ - /* */ - TRACE(("|%p|%p|LITERAL %d\n", ctx->pattern, - ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] != ctx->pattern[0]) - RETURN_FAILURE; - ctx->pattern++; - ctx->ptr++; + } + case SRE_OP_NOT_BIGCHARSET: + // Match character not in charset. + // + TRACE(("|%p|%p|NOT_BIGCHARSET\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || in_bigcharset(pattern_ptr, text_ptr[0])) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr++; break; - - case SRE_OP_NOT_LITERAL: - /* match anything that is not literal character */ - /* */ - TRACE(("|%p|%p|NOT_LITERAL %d\n", ctx->pattern, - ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] == ctx->pattern[0]) - RETURN_FAILURE; - ctx->pattern++; - ctx->ptr++; + case SRE_OP_NOT_BIGCHARSET_IGNORE: + // Match character not in charset, ignoring case. + // + TRACE(("|%p|%p|NOT_BIGCHARSET_IGNORE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || in_bigcharset(pattern_ptr, state->lower(text_ptr[0]))) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr++; break; - - case SRE_OP_SUCCESS: - /* end of pattern */ - TRACE(("|%p|%p|SUCCESS\n", ctx->pattern, ctx->ptr)); - state->ptr = ctx->ptr; - RETURN_SUCCESS; - - case SRE_OP_AT: - /* match at given position */ - /* */ - TRACE(("|%p|%p|AT %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); - if (!SRE_AT(state, ctx->ptr, *ctx->pattern)) - RETURN_FAILURE; - ctx->pattern++; - break; - - case SRE_OP_CATEGORY: - /* match at given category */ - /* */ - TRACE(("|%p|%p|CATEGORY %d\n", ctx->pattern, - ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || !sre_category(ctx->pattern[0], ctx->ptr[0])) - RETURN_FAILURE; - ctx->pattern++; - ctx->ptr++; + case SRE_OP_NOT_BIGCHARSET_IGNORE_REV: + // Match character not in charset, ignoring case. + // + TRACE(("|%p|%p|NOT_BIGCHARSET_IGNORE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || in_bigcharset(pattern_ptr, state->lower(text_ptr[-1]))) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr--; break; - - case SRE_OP_ANY: - /* match anything (except a newline) */ - /* */ - TRACE(("|%p|%p|ANY\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end || SRE_IS_LINEBREAK(ctx->ptr[0])) - RETURN_FAILURE; - ctx->ptr++; + case SRE_OP_NOT_BIGCHARSET_REV: + // Match character not in charset. + // + TRACE(("|%p|%p|NOT_BIGCHARSET_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || in_bigcharset(pattern_ptr, text_ptr[-1])) + goto backtrack; + pattern_ptr = skip_bigcharset(pattern_ptr); + text_ptr--; break; - - case SRE_OP_ANY_ALL: - /* match anything */ - /* */ - TRACE(("|%p|%p|ANY_ALL\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end) - RETURN_FAILURE; - ctx->ptr++; + case SRE_OP_NOT_BOUNDARY: + { + // Not boundary between word and non-word. + // + int before = text_ptr > text_start && SRE_IS_WORD(text_ptr[-1]); + int after = text_ptr < text_end && SRE_IS_WORD(text_ptr[0]); + TRACE(("|%p|%p|NOT_BOUNDARY\n", pattern_ptr, text_ptr)); + if (before != after) + goto backtrack; break; - - case SRE_OP_IN: - /* match set member (or non_member) */ - /* */ - TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, *ctx->ptr)) - RETURN_FAILURE; - ctx->pattern += ctx->pattern[0]; - ctx->ptr++; + } + case SRE_OP_NOT_CHARSET: + { + // Match character not in charset. + // + TRACE(("|%p|%p|NOT_CHARSET\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || in_charset(pattern_ptr, text_ptr[0])) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr++; break; - - case SRE_OP_LITERAL_IGNORE: - TRACE(("|%p|%p|LITERAL_IGNORE %d\n", - ctx->pattern, ctx->ptr, ctx->pattern[0])); - if (ctx->ptr >= end || - state->lower(*ctx->ptr) != state->lower(*ctx->pattern)) - RETURN_FAILURE; - ctx->pattern++; - ctx->ptr++; + } + case SRE_OP_NOT_CHARSET_IGNORE: + { + // Match character not in charset, ignoring case. + // + TRACE(("|%p|%p|NOT_CHARSET_IGNORE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || in_charset(pattern_ptr, state->lower(text_ptr[0]))) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr++; + break; + } + case SRE_OP_NOT_CHARSET_IGNORE_REV: + { + // Match character not in charset, ignoring case. + // + TRACE(("|%p|%p|NOT_CHARSET_IGNORE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || in_charset(pattern_ptr, state->lower(text_ptr[-1]))) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr--; + break; + } + case SRE_OP_NOT_CHARSET_REV: + { + // Match character not in charset. + // + TRACE(("|%p|%p|NOT_CHARSET_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || in_charset(pattern_ptr, text_ptr[-1])) + goto backtrack; + pattern_ptr = skip_charset(pattern_ptr); + text_ptr--; + break; + } + case SRE_OP_NOT_DIGIT: + // Not digit. + // + TRACE(("|%p|%p|NOT_DIGIT\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_IS_DIGIT(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_NOT_DIGIT_REV: + // Not digit. + // + TRACE(("|%p|%p|NOT_DIGIT_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_IS_DIGIT(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_NOT_IN: + // Match not set member. + // + TRACE(("|%p|%p|NOT_IN\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_IN(pattern_ptr, text_ptr[0])) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr++; + break; + case SRE_OP_NOT_IN_IGNORE: + // Match not set member, ignoring case. + // + TRACE(("|%p|%p|NOT_IN_IGNORE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_IN(pattern_ptr, state->lower(text_ptr[0]))) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr++; + break; + case SRE_OP_NOT_IN_IGNORE_REV: + // Match not set member, ignoring case. + // + TRACE(("|%p|%p|NOT_IN_IGNORE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_IN(pattern_ptr, state->lower(text_ptr[-1]))) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr--; + break; + case SRE_OP_NOT_IN_REV: + // Match not set member. + // + TRACE(("|%p|%p|NOT_IN_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_IN(pattern_ptr, text_ptr[-1])) + goto backtrack; + pattern_ptr += pattern_ptr[0]; + text_ptr--; + break; + case SRE_OP_NOT_LITERAL: + // Match a character that is not this character. + // + TRACE(("|%p|%p|NOT_LITERAL %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || (SRE_CODE) text_ptr[0] == pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr++; break; - case SRE_OP_NOT_LITERAL_IGNORE: - TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n", - ctx->pattern, ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || - state->lower(*ctx->ptr) == state->lower(*ctx->pattern)) - RETURN_FAILURE; - ctx->pattern++; - ctx->ptr++; + // Match a character that is not this character, ignoring case. + // + TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || state->lower(text_ptr[0]) == pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr++; break; - - case SRE_OP_IN_IGNORE: - TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end - || !SRE_CHARSET(ctx->pattern+1, - (SRE_CODE)state->lower(*ctx->ptr))) - RETURN_FAILURE; - ctx->pattern += ctx->pattern[0]; - ctx->ptr++; + case SRE_OP_NOT_LITERAL_IGNORE_REV: + // Match a character that is not this character, ignoring case. + // + TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || state->lower(text_ptr[-1]) == pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr--; break; - - case SRE_OP_JUMP: - case SRE_OP_INFO: - /* jump forward */ - /* */ - TRACE(("|%p|%p|JUMP %d\n", ctx->pattern, - ctx->ptr, ctx->pattern[0])); - ctx->pattern += ctx->pattern[0]; + case SRE_OP_NOT_LITERAL_REV: + // Match a character that is not this character. + // + TRACE(("|%p|%p|NOT_LITERAL %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || (SRE_CODE) text_ptr[-1] == pattern_ptr[0]) + goto backtrack; + pattern_ptr++; + text_ptr--; break; - - case SRE_OP_BRANCH: - /* alternation */ - /* <0=skip> code ... */ - TRACE(("|%p|%p|BRANCH\n", ctx->pattern, ctx->ptr)); - LASTMARK_SAVE(); - ctx->u.rep = state->repeat; - if (ctx->u.rep) - MARK_PUSH(ctx->lastmark); - for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { - if (ctx->pattern[1] == SRE_OP_LITERAL && - (ctx->ptr >= end || - (SRE_CODE) *ctx->ptr != ctx->pattern[2])) - continue; - if (ctx->pattern[1] == SRE_OP_IN && - (ctx->ptr >= end || - !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) *ctx->ptr))) - continue; - state->ptr = ctx->ptr; - DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); - if (ret) { - if (ctx->u.rep) - MARK_POP_DISCARD(ctx->lastmark); - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - if (ctx->u.rep) - MARK_POP_KEEP(ctx->lastmark); - LASTMARK_RESTORE(); - } - if (ctx->u.rep) - MARK_POP_DISCARD(ctx->lastmark); - RETURN_FAILURE; - - case SRE_OP_REPEAT_ONE: - /* match repeated sequence (maximizing regexp) */ - - /* this operator only works if the repeated item is - exactly one character wide, and we're not already - collecting backtracking points. for other cases, - use the MAX_REPEAT operator */ - - /* <1=min> <2=max> item tail */ - - TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, - ctx->pattern[1], ctx->pattern[2])); - - if (ctx->ptr + ctx->pattern[1] > end) - RETURN_FAILURE; /* cannot match */ - - state->ptr = ctx->ptr; - - ret = SRE_COUNT(state, ctx->pattern+3, ctx->pattern[2]); - RETURN_ON_ERROR(ret); - DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); - ctx->count = ret; - ctx->ptr += ctx->count; - - /* when we arrive here, count contains the number of - matches, and ctx->ptr points to the tail of the target - string. check if the rest of the pattern matches, - and backtrack if not. */ - - if (ctx->count < (Py_ssize_t) ctx->pattern[1]) - RETURN_FAILURE; - - if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { - /* tail is empty. we're finished */ - state->ptr = ctx->ptr; - RETURN_SUCCESS; - } - - LASTMARK_SAVE(); - - if (ctx->pattern[ctx->pattern[0]] == SRE_OP_LITERAL) { - /* tail starts with a literal. skip positions where - the rest of the pattern cannot possibly match */ - ctx->u.chr = ctx->pattern[ctx->pattern[0]+1]; - for (;;) { - while (ctx->count >= (Py_ssize_t) ctx->pattern[1] && - (ctx->ptr >= end || *ctx->ptr != ctx->u.chr)) { - ctx->ptr--; - ctx->count--; - } - if (ctx->count < (Py_ssize_t) ctx->pattern[1]) + case SRE_OP_NOT_RANGE: + // Match not range character. + // + TRACE(("|%p|%p|NOT_RANGE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || SRE_IN_RANGE(text_ptr[0], pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr++; + break; + case SRE_OP_NOT_RANGE_IGNORE: + // Match not range character, ignoring case. + // + TRACE(("|%p|%p|NOT_RANGE_IGNORE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || SRE_IN_RANGE(state->lower(text_ptr[0]), pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr++; + break; + case SRE_OP_NOT_RANGE_IGNORE_REV: + // Match not range character, ignoring case. + // + TRACE(("|%p|%p|NOT_RANGE_IGNORE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || SRE_IN_RANGE(state->lower(text_ptr[-1]), pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr--; + break; + case SRE_OP_NOT_RANGE_REV: + // Match not range character. + // + TRACE(("|%p|%p|NOT_RANGE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || SRE_IN_RANGE(text_ptr[-1], pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr--; + break; + case SRE_OP_NOT_WHITESPACE: + // Not whitespace. + // + TRACE(("|%p|%p|NOT_WHITESPACE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_IS_WHITESPACE(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_NOT_WHITESPACE_REV: + // Not whitespace. + // + TRACE(("|%p|%p|NOT_WHITESPACE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_IS_WHITESPACE(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_NOT_WORD: + // Not word. + // + TRACE(("|%p|%p|NOT_WORD\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_IS_WORD(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_NOT_WORD_REV: + // Not word. + // + TRACE(("|%p|%p|NOT_WORD_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_IS_WORD(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_RANGE: + // Match range character. + // + TRACE(("|%p|%p|RANGE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || !SRE_IN_RANGE(text_ptr[0], pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr++; + break; + case SRE_OP_RANGE_IGNORE: + // Match range character, ignoring case. + // + TRACE(("|%p|%p|RANGE_IGNORE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr >= text_end || !SRE_IN_RANGE(state->lower(text_ptr[0]), pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr++; + break; + case SRE_OP_RANGE_IGNORE_REV: + // Match range character, ignoring case. + // + TRACE(("|%p|%p|RANGE_IGNORE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || !SRE_IN_RANGE(state->lower(text_ptr[-1]), pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr--; + break; + case SRE_OP_RANGE_REV: + // Match range character. + // + TRACE(("|%p|%p|RANGE %d %d\n", pattern_ptr, text_ptr, pattern_ptr[0])); + if (text_ptr <= text_start || !SRE_IN_RANGE(text_ptr[-1], pattern_ptr[0], pattern_ptr[1])) + goto backtrack; + pattern_ptr += 2; + text_ptr--; + break; + case SRE_OP_REPEAT_MAX: + { + // Greedy repeat. + // ... + SRE_CODE* repeat = pattern_ptr; + SRE_CODE* end_repeat = pattern_ptr + pattern_ptr[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_end - text_ptr; + TRACE(("|%p|%p|REPEAT_MAX %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_MAX, repeat, index) + SAVE_MARKS() + repeat_counts[index] = 0; + if (repeat_min == 0) { + int save; + // Look at what follows to avoid unnecessary backtracking. + if (limit > 0) { + SRE_CODE* look_literal = tail; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + switch (look_literal[0]) { + case SRE_OP_LITERAL: + save = text_ptr[0] == (SRE_CHAR)look_literal[1]; break; - state->ptr = ctx->ptr; - DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, - ctx->pattern+ctx->pattern[0]); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - - LASTMARK_RESTORE(); - - ctx->ptr--; - ctx->count--; - } - - } else { - /* general case */ - while (ctx->count >= (Py_ssize_t) ctx->pattern[1]) { - state->ptr = ctx->ptr; - DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, - ctx->pattern+ctx->pattern[0]); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - ctx->ptr--; - ctx->count--; - LASTMARK_RESTORE(); - } - } - RETURN_FAILURE; - - case SRE_OP_MIN_REPEAT_ONE: - /* match repeated sequence (minimizing regexp) */ - - /* this operator only works if the repeated item is - exactly one character wide, and we're not already - collecting backtracking points. for other cases, - use the MIN_REPEAT operator */ - - /* <1=min> <2=max> item tail */ - - TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, - ctx->pattern[1], ctx->pattern[2])); - - if (ctx->ptr + ctx->pattern[1] > end) - RETURN_FAILURE; /* cannot match */ - - state->ptr = ctx->ptr; - - if (ctx->pattern[1] == 0) - ctx->count = 0; - else { - /* count using pattern min as the maximum */ - ret = SRE_COUNT(state, ctx->pattern+3, ctx->pattern[1]); - RETURN_ON_ERROR(ret); - DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); - if (ret < (Py_ssize_t) ctx->pattern[1]) - /* didn't match minimum number of times */ - RETURN_FAILURE; - /* advance past minimum matches of repeat */ - ctx->count = ret; - ctx->ptr += ctx->count; - } - - if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { - /* tail is empty. we're finished */ - state->ptr = ctx->ptr; - RETURN_SUCCESS; - - } else { - /* general case */ - LASTMARK_SAVE(); - while ((Py_ssize_t)ctx->pattern[2] == 65535 - || ctx->count <= (Py_ssize_t)ctx->pattern[2]) { - state->ptr = ctx->ptr; - DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, - ctx->pattern+ctx->pattern[0]); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; + case SRE_OP_LITERAL_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING: + save = text_ptr[0] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; } - state->ptr = ctx->ptr; - ret = SRE_COUNT(state, ctx->pattern+3, 1); - RETURN_ON_ERROR(ret); - DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); - if (ret == 0) - break; - assert(ret == 1); - ctx->ptr++; - ctx->count++; - LASTMARK_RESTORE(); + } else + save = 1; + if (save) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MAX, end_repeat, 0) + SAVE_MARKS() } } - RETURN_FAILURE; - - case SRE_OP_REPEAT: - /* create repeat context. all the hard work is done - by the UNTIL operator (MAX_UNTIL, MIN_UNTIL) */ - /* <1=min> <2=max> item tail */ - TRACE(("|%p|%p|REPEAT %d %d\n", ctx->pattern, ctx->ptr, - ctx->pattern[1], ctx->pattern[2])); - - /* install new repeat context */ - ctx->u.rep = (SRE_REPEAT*) PyObject_MALLOC(sizeof(*ctx->u.rep)); - if (!ctx->u.rep) { - PyErr_NoMemory(); - RETURN_FAILURE; - } - ctx->u.rep->count = -1; - ctx->u.rep->pattern = ctx->pattern; - ctx->u.rep->prev = state->repeat; - ctx->u.rep->last_ptr = NULL; - state->repeat = ctx->u.rep; - - state->ptr = ctx->ptr; - DO_JUMP(JUMP_REPEAT, jump_repeat, ctx->pattern+ctx->pattern[0]); - state->repeat = ctx->u.rep->prev; - PyObject_FREE(ctx->u.rep); - - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - RETURN_FAILURE; - - case SRE_OP_MAX_UNTIL: - /* maximizing repeat */ - /* <1=min> <2=max> item tail */ - - /* FIXME: we probably need to deal with zero-width - matches in here... */ - - ctx->u.rep = state->repeat; - if (!ctx->u.rep) - RETURN_ERROR(SRE_ERROR_STATE); - - state->ptr = ctx->ptr; - - ctx->count = ctx->u.rep->count+1; - - TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, - ctx->ptr, ctx->count)); - - if (ctx->count < ctx->u.rep->pattern[1]) { - /* not enough matches */ - ctx->u.rep->count = ctx->count; - DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, - ctx->u.rep->pattern+3); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - ctx->u.rep->count = ctx->count-1; - state->ptr = ctx->ptr; - RETURN_FAILURE; - } - - if ((ctx->count < ctx->u.rep->pattern[2] || - ctx->u.rep->pattern[2] == 65535) && - state->ptr != ctx->u.rep->last_ptr) { - /* we may have enough matches, but if we can - match another item, do so */ - ctx->u.rep->count = ctx->count; - LASTMARK_SAVE(); - MARK_PUSH(ctx->lastmark); - /* zero-width match protection */ - DATA_PUSH(&ctx->u.rep->last_ptr); - ctx->u.rep->last_ptr = state->ptr; - DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2, - ctx->u.rep->pattern+3); - DATA_POP(&ctx->u.rep->last_ptr); - if (ret) { - MARK_POP_DISCARD(ctx->lastmark); - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - MARK_POP(ctx->lastmark); - LASTMARK_RESTORE(); - ctx->u.rep->count = ctx->count-1; - state->ptr = ctx->ptr; - } - - /* cannot match more repeated items here. make sure the - tail matches */ - state->repeat = ctx->u.rep->prev; - DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, ctx->pattern); - RETURN_ON_SUCCESS(ret); - state->repeat = ctx->u.rep; - state->ptr = ctx->ptr; - RETURN_FAILURE; - - case SRE_OP_MIN_UNTIL: - /* minimizing repeat */ - /* <1=min> <2=max> item tail */ - - ctx->u.rep = state->repeat; - if (!ctx->u.rep) - RETURN_ERROR(SRE_ERROR_STATE); - - state->ptr = ctx->ptr; - - ctx->count = ctx->u.rep->count+1; - - TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, - ctx->ptr, ctx->count, ctx->u.rep->pattern)); - - if (ctx->count < ctx->u.rep->pattern[1]) { - /* not enough matches */ - ctx->u.rep->count = ctx->count; - DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, - ctx->u.rep->pattern+3); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - ctx->u.rep->count = ctx->count-1; - state->ptr = ctx->ptr; - RETURN_FAILURE; - } - - LASTMARK_SAVE(); - - /* see if the tail matches */ - state->repeat = ctx->u.rep->prev; - DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, ctx->pattern); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - - state->repeat = ctx->u.rep; - state->ptr = ctx->ptr; - - LASTMARK_RESTORE(); - - if (ctx->count >= ctx->u.rep->pattern[2] - && ctx->u.rep->pattern[2] != 65535) - RETURN_FAILURE; - - ctx->u.rep->count = ctx->count; - DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3, - ctx->u.rep->pattern+3); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_SUCCESS; - } - ctx->u.rep->count = ctx->count-1; - state->ptr = ctx->ptr; - RETURN_FAILURE; - - case SRE_OP_GROUPREF: - /* match backreference */ - TRACE(("|%p|%p|GROUPREF %d\n", ctx->pattern, - ctx->ptr, ctx->pattern[0])); - i = ctx->pattern[0]; - { - Py_ssize_t groupref = i+i; - if (groupref >= state->lastmark) { - RETURN_FAILURE; - } else { - SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; - SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; - if (!p || !e || e < p) - RETURN_FAILURE; - while (p < e) { - if (ctx->ptr >= end || *ctx->ptr != *p) - RETURN_FAILURE; - p++; ctx->ptr++; + pattern_ptr = body; + repeat_starts[index] = text_ptr; + break; + } + case SRE_OP_REPEAT_MAX_REV: + { + // Greedy repeat. + // ... + SRE_CODE* repeat = pattern_ptr; + SRE_CODE* end_repeat = pattern_ptr + pattern_ptr[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_ptr - text_start; + TRACE(("|%p|%p|REPEAT_MAX_REV %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_MAX_REV, repeat, index) + SAVE_MARKS() + repeat_counts[index] = 0; + if (repeat_min == 0) { + int save; + // Look at what follows to avoid unnecessary backtracking. + if (limit > 0) { + SRE_CODE* look_literal = tail; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + switch (look_literal[0]) { + case SRE_OP_LITERAL: + save = text_ptr[-1] == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_IGNORE: + save = state->lower(text_ptr[-1]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING: + save = text_ptr[-1] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE: + save = state->lower(text_ptr[-1]) == (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; } + } else + save = 1; + if (save) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MAX_REV, end_repeat, 0) + SAVE_MARKS() } } - ctx->pattern++; + pattern_ptr = body; + repeat_starts[index] = text_ptr; break; - - case SRE_OP_GROUPREF_IGNORE: - /* match backreference */ - TRACE(("|%p|%p|GROUPREF_IGNORE %d\n", ctx->pattern, - ctx->ptr, ctx->pattern[0])); - i = ctx->pattern[0]; - { - Py_ssize_t groupref = i+i; - if (groupref >= state->lastmark) { - RETURN_FAILURE; - } else { - SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; - SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; - if (!p || !e || e < p) - RETURN_FAILURE; - while (p < e) { - if (ctx->ptr >= end || - state->lower(*ctx->ptr) != state->lower(*p)) - RETURN_FAILURE; - p++; ctx->ptr++; + } + case SRE_OP_REPEAT_MIN: + { + // Lazy repeat. + // ... + SRE_CODE* repeat = pattern_ptr; + SRE_CODE* end_repeat = pattern_ptr + pattern_ptr[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_end - text_ptr; + TRACE(("|%p|%p|REPEAT_MIN %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_MIN, repeat, index) + SAVE_MARKS() + repeat_counts[index] = 0; + if (repeat_min == 0) { + int save; + if (limit > 0) { + SRE_CODE* look_literal = tail; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL: + save = text_ptr[0] == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING: + save = text_ptr[0] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE: + save = state->lower(text_ptr[0]) == (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; } + } else + save = 1; + if (save) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MIN, end_repeat, 0) + SAVE_MARKS() + pattern_ptr = tail; + } else { + pattern_ptr = body; + repeat_starts[index] = text_ptr; } + } else { + pattern_ptr = body; + repeat_starts[index] = text_ptr; } - ctx->pattern++; break; - - case SRE_OP_GROUPREF_EXISTS: - TRACE(("|%p|%p|GROUPREF_EXISTS %d\n", ctx->pattern, - ctx->ptr, ctx->pattern[0])); - /* codeyes codeno ... */ - i = ctx->pattern[0]; - { - Py_ssize_t groupref = i+i; - if (groupref >= state->lastmark) { - ctx->pattern += ctx->pattern[1]; - break; - } else { - SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; - SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; - if (!p || !e || e < p) { - ctx->pattern += ctx->pattern[1]; + } + case SRE_OP_REPEAT_MIN_REV: + { + // Lazy repeat. + // ... + SRE_CODE* repeat = pattern_ptr; + SRE_CODE* end_repeat = pattern_ptr + pattern_ptr[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_ptr - text_start; + TRACE(("|%p|%p|REPEAT_MIN_REV %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_MIN_REV, repeat, index) + SAVE_MARKS() + repeat_counts[index] = 0; + if (repeat_min == 0) { + int save; + if (limit > 0) { + SRE_CODE* look_literal = tail; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL_REV: + save = text_ptr[-1] == (SRE_CHAR)look_literal[1]; break; + case SRE_OP_LITERAL_IGNORE_REV: + save = state->lower(text_ptr[-1]) == (SRE_CHAR)look_literal[1]; + break; + case SRE_OP_LITERAL_STRING_REV: + save = text_ptr[-1] == (SRE_CHAR)look_literal[2]; + break; + case SRE_OP_LITERAL_STRING_IGNORE_REV: + save = state->lower(text_ptr[-1]) == (SRE_CHAR)look_literal[2]; + break; + default: + save = 1; } + } else + save = 1; + if (save) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_MIN_REV, end_repeat, 0) + SAVE_MARKS() + pattern_ptr = tail; + } else { + pattern_ptr = body; + repeat_starts[index] = text_ptr; } + } else { + pattern_ptr = body; + repeat_starts[index] = text_ptr; } - ctx->pattern += 2; break; - - case SRE_OP_ASSERT: - /* assert subpattern */ - /* */ - TRACE(("|%p|%p|ASSERT %d\n", ctx->pattern, - ctx->ptr, ctx->pattern[1])); - state->ptr = ctx->ptr - ctx->pattern[1]; - if (state->ptr < state->beginning) - RETURN_FAILURE; - DO_JUMP(JUMP_ASSERT, jump_assert, ctx->pattern+2); - RETURN_ON_FAILURE(ret); - ctx->pattern += ctx->pattern[0]; + } + case SRE_OP_REPEAT_ONE_MAX: + { + // Greedy repeat. + // ... + int index = pattern_ptr[1]; + int repeat_min = pattern_ptr[2]; + int repeat_max = pattern_ptr[3]; + SRE_CODE* body = pattern_ptr + 4; + SRE_CODE* tail = pattern_ptr + pattern_ptr[0]; + Py_ssize_t limit = text_end - text_ptr; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr = text_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + TRACE(("|%p|%p|REPEAT_ONE_MAX %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + repeat_max = sre_repeat_limit(repeat_max, limit); + end_ptr = start_ptr + repeat_max; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_MACRO(REPEAT_ONE_MAX_MACRO) + if (text_ptr < start_ptr) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_ONE_MAX, pattern_ptr, index) + repeat_counts[index] = repeat_min + (text_ptr - start_ptr); + pattern_ptr = tail; break; - - case SRE_OP_ASSERT_NOT: - /* assert not subpattern */ - /* */ - TRACE(("|%p|%p|ASSERT_NOT %d\n", ctx->pattern, - ctx->ptr, ctx->pattern[1])); - state->ptr = ctx->ptr - ctx->pattern[1]; - if (state->ptr >= state->beginning) { - DO_JUMP(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); - if (ret) { - RETURN_ON_ERROR(ret); - RETURN_FAILURE; - } + } + case SRE_OP_REPEAT_ONE_MAX_REV: + { + // Greedy repeat. + // ... + int index = pattern_ptr[1]; + int repeat_min = pattern_ptr[2]; + int repeat_max = pattern_ptr[3]; + SRE_CODE* body = pattern_ptr + 4; + SRE_CODE* tail = pattern_ptr + pattern_ptr[0]; + Py_ssize_t limit = text_ptr - text_start; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr = text_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + TRACE(("|%p|%p|REPEAT_ONE_MAX_REV %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + repeat_max = sre_repeat_limit(repeat_max, limit); + end_ptr = start_ptr - repeat_max; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_REV_MACRO(REPEAT_ONE_MAX_REV_MACRO) + if (text_ptr > start_ptr) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_ONE_MAX_REV, pattern_ptr, index) + repeat_counts[index] = repeat_min + (start_ptr - text_ptr); + pattern_ptr = tail; + break; + } + case SRE_OP_REPEAT_ONE_MIN: + { + // Lazy repeat. + // ... + int index = pattern_ptr[1]; + int repeat_min = pattern_ptr[2]; + int repeat_max = pattern_ptr[3]; + SRE_CODE* body = pattern_ptr + 4; + SRE_CODE* tail = pattern_ptr + pattern_ptr[0]; + Py_ssize_t limit = text_end - text_ptr; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr = text_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + TRACE(("|%p|%p|REPEAT_ONE_MIN %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + repeat_max = sre_repeat_limit(repeat_max, limit); + end_ptr = start_ptr + repeat_min; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_MACRO(REPEAT_ONE_MIN_MACRO) + SAVE_BACKTRACK(SRE_OP_REPEAT_ONE_MIN, pattern_ptr, index); + repeat_counts[index] = repeat_min + (text_ptr - start_ptr); + pattern_ptr = tail; + break; + } + case SRE_OP_REPEAT_ONE_MIN_REV: + { + // Lazy repeat. + // ... + int index = pattern_ptr[1]; + int repeat_min = pattern_ptr[2]; + int repeat_max = pattern_ptr[3]; + SRE_CODE* body = pattern_ptr + 4; + SRE_CODE* tail = pattern_ptr + pattern_ptr[0]; + Py_ssize_t limit = text_ptr - text_start; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr = text_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + TRACE(("|%p|%p|REPEAT_ONE_MIN_REV %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + repeat_max = sre_repeat_limit(repeat_max, limit); + end_ptr = start_ptr - repeat_min; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_REV_MACRO(REPEAT_ONE_MIN_REV_MACRO) + SAVE_BACKTRACK(SRE_OP_REPEAT_ONE_MIN_REV, pattern_ptr, index); + repeat_counts[index] = repeat_min + (start_ptr - text_ptr); + pattern_ptr = tail; + break; + } + case SRE_OP_REPEAT_ONE_POSS: + { + // Possessive repeat + // ... + int index = pattern_ptr[1]; + int repeat_min = pattern_ptr[2]; + int repeat_max = pattern_ptr[3]; + SRE_CODE* body = pattern_ptr + 4; + SRE_CODE* tail = pattern_ptr + pattern_ptr[0]; + Py_ssize_t limit = text_end - text_ptr; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr = text_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + TRACE(("|%p|%p|REPEAT_ONE_POSS %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + repeat_max = sre_repeat_limit(repeat_max, limit); + end_ptr = start_ptr + repeat_max; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_MACRO(REPEAT_ONE_POSS_MACRO) + /* Matched at least the minimum? */ + if (text_ptr < start_ptr + repeat_min) + goto backtrack; + pattern_ptr = tail; + break; + } + case SRE_OP_REPEAT_ONE_POSS_REV: + { + // Possessive repeat + // ... + int index = pattern_ptr[1]; + int repeat_min = pattern_ptr[2]; + int repeat_max = pattern_ptr[3]; + SRE_CODE* body = pattern_ptr + 4; + SRE_CODE* tail = pattern_ptr + pattern_ptr[0]; + Py_ssize_t limit = text_ptr - text_start; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr = text_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + TRACE(("|%p|%p|REPEAT_ONE_POSS_REV %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + repeat_max = sre_repeat_limit(repeat_max, limit); + end_ptr = start_ptr - repeat_max; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_REV_MACRO(REPEAT_ONE_POSS_REV_MACRO) + /* Matched at least the minimum? */ + if (text_ptr > start_ptr - repeat_min) + goto backtrack; + pattern_ptr = tail; + break; + } + case SRE_OP_REPEAT_POSS: + { + // Possessive repeat. + // ... + SRE_CODE* repeat = pattern_ptr; + SRE_CODE* end_repeat = pattern_ptr + pattern_ptr[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_end - text_ptr; + TRACE(("|%p|%p|REPEAT_POSS %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_POSS, repeat, index) + SAVE_MARKS() + repeat_counts[index] = 0; + if (repeat_min == 0) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_POSS, end_repeat, 0) + SAVE_MARKS() } - ctx->pattern += ctx->pattern[0]; + pattern_ptr = body; + repeat_starts[index] = text_ptr; break; + } + case SRE_OP_REPEAT_POSS_REV: + { + // Possessive repeat. + // ... + SRE_CODE* repeat = pattern_ptr; + SRE_CODE* end_repeat = pattern_ptr + pattern_ptr[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_ptr - text_start; + TRACE(("|%p|%p|REPEAT_POSS_REV %d %d\n", pattern_ptr, text_ptr, repeat_min, repeat_max)); + if (repeat_min > limit) + goto backtrack; + SAVE_BACKTRACK(SRE_OP_REPEAT_POSS_REV, repeat, index) + SAVE_MARKS() + repeat_counts[index] = 0; + if (repeat_min == 0) { + SAVE_BACKTRACK(SRE_OP_END_REPEAT_POSS_REV, end_repeat, 0) + SAVE_MARKS() + } + pattern_ptr = body; + repeat_starts[index] = text_ptr; + break; + } + case SRE_OP_START_OF_LINE: + // Start of line. + // + TRACE(("|%p|%p|START_OF_LINE\n", pattern_ptr, text_ptr)); + if (text_ptr > text_start && !SRE_IS_LINEBREAK(text_ptr[-1])) + goto backtrack; + break; + case SRE_OP_START_OF_STRING: + // Start of string. + // + TRACE(("|%p|%p|START_OF_STRING\n", pattern_ptr, text_ptr)); + if (text_ptr > text_start) + goto backtrack; + break; + case SRE_OP_SUCCESS: + { + // End of pattern. + int m; + SRE_CHAR* end_ptr = NULL; + TRACE(("|%p|%p|SUCCESS\n", pattern_ptr, text_ptr)); + + // Find the mark which matched the furthest to the right. + for (m = 1; m < mark_count; m += 2) { + if (mark[m - 1] != NULL && mark[m] != NULL) { + state->lastmark = m; + if (end_ptr < mark[m]) { + state->lastindex = 1 + m / 2; + end_ptr = mark[m]; + } + } + } - case SRE_OP_FAILURE: - /* immediate failure */ - TRACE(("|%p|%p|FAILURE\n", ctx->pattern, ctx->ptr)); - RETURN_FAILURE; - + state->ptr = text_ptr; + SRE_RETURN(1); + } + case SRE_OP_UNI_BOUNDARY: + { + // Boundary between word and non-word. + // + int before = text_ptr > text_start && SRE_UNI_IS_WORD(text_ptr[-1]); + int after = text_ptr < text_end && SRE_UNI_IS_WORD(text_ptr[0]); + TRACE(("|%p|%p|UNI_BOUNDARY\n", pattern_ptr, text_ptr)); + if (before == after) + goto backtrack; + break; + } + case SRE_OP_UNI_DIGIT: + // Digit. + // + TRACE(("|%p|%p|UNI_DIGIT\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_UNI_IS_DIGIT(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_UNI_DIGIT_REV: + // Digit. + // + TRACE(("|%p|%p|UNI_DIGIT_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_UNI_IS_DIGIT(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_UNI_NOT_BOUNDARY: + { + // Not boundary between word and non-word. + // + int before = text_ptr > text_start && SRE_UNI_IS_WORD(text_ptr[-1]); + int after = text_ptr < text_end && SRE_UNI_IS_WORD(text_ptr[0]); + TRACE(("|%p|%p|UNI_NOT_BOUNDARY\n", pattern_ptr, text_ptr)); + if (before != after) + goto backtrack; + break; + } + case SRE_OP_UNI_NOT_DIGIT: + // Not digit. + // + TRACE(("|%p|%p|UNI_NOT_DIGIT\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_UNI_IS_DIGIT(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_UNI_NOT_DIGIT_REV: + // Not digit. + // + TRACE(("|%p|%p|UNI_NOT_DIGIT_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_UNI_IS_DIGIT(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_UNI_NOT_WHITESPACE: + // Not whitespace. + // + TRACE(("|%p|%p|UNI_NOT_WHITESPACE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_UNI_IS_WHITESPACE(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_UNI_NOT_WHITESPACE_REV: + // Not whitespace. + // + TRACE(("|%p|%p|UNI_NOT_WHITESPACE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_UNI_IS_WHITESPACE(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_UNI_NOT_WORD: + // Not word. + // + TRACE(("|%p|%p|UNI_NOT_WORD\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || SRE_UNI_IS_WORD(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_UNI_NOT_WORD_REV: + // Not word. + // + TRACE(("|%p|%p|UNI_NOT_WORD_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || SRE_UNI_IS_WORD(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_UNI_WHITESPACE: + // Whitespace. + // + TRACE(("|%p|%p|UNI_WHITESPACE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_UNI_IS_WHITESPACE(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_UNI_WHITESPACE_REV: + // Whitespace. + // + TRACE(("|%p|%p|UNI_WHITESPACE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_UNI_IS_WHITESPACE(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_UNI_WORD: + // Word. + // + TRACE(("|%p|%p|UNI_WORD\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_UNI_IS_WORD(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_UNI_WORD_REV: + // Word. + // + TRACE(("|%p|%p|UNI_WORD_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_UNI_IS_WORD(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_WHITESPACE: + // Whitespace. + // + TRACE(("|%p|%p|WHITESPACE\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_IS_WHITESPACE(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_WHITESPACE_REV: + // Whitespace. + // + TRACE(("|%p|%p|WHITESPACE_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_IS_WHITESPACE(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; + case SRE_OP_WORD: + // Word. + // + TRACE(("|%p|%p|WORD\n", pattern_ptr, text_ptr)); + if (text_ptr >= text_end || !SRE_IS_WORD(text_ptr[0])) + goto backtrack; + text_ptr++; + break; + case SRE_OP_WORD_REV: + // Word. + // + TRACE(("|%p|%p|WORD_REV\n", pattern_ptr, text_ptr)); + if (text_ptr <= text_start || !SRE_IS_WORD(text_ptr[-1])) + goto backtrack; + text_ptr--; + break; default: - TRACE(("|%p|%p|UNKNOWN %d\n", ctx->pattern, ctx->ptr, - ctx->pattern[-1])); - RETURN_ERROR(SRE_ERROR_ILLEGAL); + TRACE(("|%p|%p|UNKNOWN %d\n", pattern_ptr, text_ptr, pattern_ptr[-1])); + SRE_RETURN(SRE_ERROR_ILLEGAL) } } -exit: - ctx_pos = ctx->last_ctx_pos; - jump = ctx->jump; - DATA_POP_DISCARD(ctx); - if (ctx_pos == -1) - return ret; - DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); - - switch (jump) { - case JUMP_MAX_UNTIL_2: - TRACE(("|%p|%p|JUMP_MAX_UNTIL_2\n", ctx->pattern, ctx->ptr)); - goto jump_max_until_2; - case JUMP_MAX_UNTIL_3: - TRACE(("|%p|%p|JUMP_MAX_UNTIL_3\n", ctx->pattern, ctx->ptr)); - goto jump_max_until_3; - case JUMP_MIN_UNTIL_2: - TRACE(("|%p|%p|JUMP_MIN_UNTIL_2\n", ctx->pattern, ctx->ptr)); - goto jump_min_until_2; - case JUMP_MIN_UNTIL_3: - TRACE(("|%p|%p|JUMP_MIN_UNTIL_3\n", ctx->pattern, ctx->ptr)); - goto jump_min_until_3; - case JUMP_BRANCH: - TRACE(("|%p|%p|JUMP_BRANCH\n", ctx->pattern, ctx->ptr)); - goto jump_branch; - case JUMP_MAX_UNTIL_1: - TRACE(("|%p|%p|JUMP_MAX_UNTIL_1\n", ctx->pattern, ctx->ptr)); - goto jump_max_until_1; - case JUMP_MIN_UNTIL_1: - TRACE(("|%p|%p|JUMP_MIN_UNTIL_1\n", ctx->pattern, ctx->ptr)); - goto jump_min_until_1; - case JUMP_REPEAT: - TRACE(("|%p|%p|JUMP_REPEAT\n", ctx->pattern, ctx->ptr)); - goto jump_repeat; - case JUMP_REPEAT_ONE_1: - TRACE(("|%p|%p|JUMP_REPEAT_ONE_1\n", ctx->pattern, ctx->ptr)); - goto jump_repeat_one_1; - case JUMP_REPEAT_ONE_2: - TRACE(("|%p|%p|JUMP_REPEAT_ONE_2\n", ctx->pattern, ctx->ptr)); - goto jump_repeat_one_2; - case JUMP_MIN_REPEAT_ONE: - TRACE(("|%p|%p|JUMP_MIN_REPEAT_ONE\n", ctx->pattern, ctx->ptr)); - goto jump_min_repeat_one; - case JUMP_ASSERT: - TRACE(("|%p|%p|JUMP_ASSERT\n", ctx->pattern, ctx->ptr)); - goto jump_assert; - case JUMP_ASSERT_NOT: - TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); - goto jump_assert_not; - case JUMP_NONE: - TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); +backtrack: + TRACE(("|%p|%p|BACKTRACK ", pattern_ptr, text_ptr)); + switch(backtrack_chunk->items[backtrack_chunk->count - 1].op) { + case SRE_OP_ASSERT: + // Assert subpattern. + // ... + TRACE(("ASSERT\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + goto backtrack; + case SRE_OP_ASSERT_NOT: + { + // Assert not subpattern. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + TRACE(("ASSERT_NOT\n")); + pattern_ptr = backtrack_item->pattern_ptr; + text_ptr = backtrack_item->text_ptr; + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + pattern_ptr += pattern_ptr[0]; + goto advance; + } + case SRE_OP_ATOMIC: + // Atomic subpattern. + // ... + TRACE(("ATOMIC\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + goto backtrack; + case SRE_OP_BRANCH: + { + // Alternation. + // ... ... 0 + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + TRACE(("BRANCH\n")); + pattern_ptr = backtrack_item->pattern_ptr; + pattern_ptr += pattern_ptr[0]; + RESTORE_MARKS() + if (pattern_ptr[0] == 0) { + DISCARD_BACKTRACK(0) + goto backtrack; + } + backtrack_item->pattern_ptr = pattern_ptr; + SAVE_MARKS() + text_ptr = backtrack_item->text_ptr; + pattern_ptr++; + goto advance; + } + case SRE_OP_END_REPEAT_MAX: + { + // End of greedy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* end_repeat = backtrack_item->pattern_ptr; + SRE_CODE* repeat = end_repeat - end_repeat[0]; + SRE_CODE* tail = end_repeat + 1; + TRACE(("END_REPEAT_MAX\n")); + text_ptr = backtrack_item->text_ptr; + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + pattern_ptr = tail; + goto advance; + } + case SRE_OP_END_REPEAT_MAX_REV: + { + // End of greedy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* end_repeat = backtrack_item->pattern_ptr; + SRE_CODE* repeat = end_repeat - end_repeat[0]; + SRE_CODE* tail = end_repeat + 1; + TRACE(("END_REPEAT_MAX_REV\n")); + text_ptr = backtrack_item->text_ptr; + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + pattern_ptr = tail; + goto advance; + } + case SRE_OP_END_REPEAT_MIN: + { + // End of lazy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* end_repeat = backtrack_item->pattern_ptr; + SRE_CODE* repeat = end_repeat - end_repeat[0]; + int index = repeat[1]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_end - text_ptr; + TRACE(("END_REPEAT_MIN\n")); + text_ptr = backtrack_item->text_ptr; + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + if (limit == 0) + goto backtrack; + pattern_ptr = body; + goto advance; + } + case SRE_OP_END_REPEAT_MIN_REV: + { + // End of lazy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* end_repeat = backtrack_item->pattern_ptr; + SRE_CODE* repeat = end_repeat - end_repeat[0]; + int index = repeat[1]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = end_repeat + 1; + Py_ssize_t limit = text_ptr - text_start; + TRACE(("END_REPEAT_MIN_REV\n")); + text_ptr = backtrack_item->text_ptr; + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + if (limit == 0) + goto backtrack; + pattern_ptr = body; + goto advance; + } + case SRE_OP_END_REPEAT_POSS: + { + // End of possessive repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* end_repeat = backtrack_item->pattern_ptr; + SRE_CODE* repeat = end_repeat - end_repeat[0]; + SRE_CODE* tail = end_repeat + 1; + TRACE(("END_REPEAT_POSS\n")); + text_ptr = backtrack_item->text_ptr; + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + pattern_ptr = tail; + goto advance; + } + case SRE_OP_END_REPEAT_POSS_REV: + { + // End of possessive repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* end_repeat = backtrack_item->pattern_ptr; + SRE_CODE* repeat = end_repeat - end_repeat[0]; + SRE_CODE* tail = end_repeat + 1; + TRACE(("END_REPEAT_POSS_REV\n")); + text_ptr = backtrack_item->text_ptr; + RESTORE_MARKS() + DISCARD_BACKTRACK(0) + pattern_ptr = tail; + goto advance; + } + case SRE_OP_FAILURE: + // Failed to match. + SRE_RETURN(0) + case SRE_OP_REPEAT_MAX: + { + // Greedy repeat. + // ... + TRACE(("REPEAT_MAX\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(1) + goto backtrack; + } + case SRE_OP_REPEAT_MAX_REV: + { + // Greedy repeat. + // ... + TRACE(("REPEAT_MAX\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(1) + goto backtrack; + } + case SRE_OP_REPEAT_MIN: + { + // Lazy repeat. + // ... + TRACE(("REPEAT_MIN\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(1) + goto backtrack; + } + case SRE_OP_REPEAT_MIN_REV: + { + // Lazy repeat. + // ... + TRACE(("REPEAT_MIN\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(1) + goto backtrack; + } + case SRE_OP_REPEAT_ONE_MAX: + { + // Greedy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* repeat = backtrack_item->pattern_ptr; + SRE_CODE* tail = repeat + repeat[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + SRE_CHAR* start_ptr; + SRE_CODE* look_literal = tail; + SRE_CHAR following_char; + text_ptr = backtrack_item->text_ptr; + start_ptr = text_ptr - repeat_counts[index] + repeat_min; + text_ptr--; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL: + following_char = (SRE_CHAR)look_literal[1]; + goto end_max; + case SRE_OP_LITERAL_IGNORE: + following_char = (SRE_CHAR)look_literal[1]; + goto end_max_ignore; + case SRE_OP_LITERAL_STRING: + following_char = (SRE_CHAR)look_literal[2]; +end_max: + while (text_ptr >= start_ptr && text_ptr[0] != following_char) + text_ptr--; + break; + case SRE_OP_LITERAL_STRING_IGNORE: + following_char = (SRE_CHAR)look_literal[2]; +end_max_ignore: + while (text_ptr >= start_ptr && state->lower(text_ptr[0]) != following_char) + text_ptr--; + break; + } + if (text_ptr < start_ptr) { + DISCARD_BACKTRACK(1) + goto backtrack; + } + backtrack_item->text_ptr = text_ptr; + repeat_counts[index] = repeat_min + (text_ptr - start_ptr); + pattern_ptr = tail; + goto advance; + } + case SRE_OP_REPEAT_ONE_MAX_REV: + { + // Greedy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* repeat = backtrack_item->pattern_ptr; + SRE_CODE* tail = repeat + repeat[0]; + int index = repeat[1]; + int repeat_min = repeat[2]; + SRE_CHAR* start_ptr; + SRE_CODE* look_literal = tail; + SRE_CHAR following_char; + text_ptr = backtrack_item->text_ptr; + start_ptr = text_ptr + repeat_counts[index] - repeat_min; + text_ptr++; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + // Look at what follows to avoid unnecessary backtracking. + switch (look_literal[0]) { + case SRE_OP_LITERAL_REV: + following_char = (SRE_CHAR)look_literal[1]; + goto end_max_rev; + case SRE_OP_LITERAL_IGNORE_REV: + following_char = (SRE_CHAR)look_literal[1]; + goto end_max_ignore_rev; + case SRE_OP_LITERAL_STRING_REV: + following_char = (SRE_CHAR)look_literal[2]; +end_max_rev: + while (text_ptr <= start_ptr && text_ptr[-1] != following_char) + text_ptr++; break; + case SRE_OP_LITERAL_STRING_IGNORE_REV: + following_char = (SRE_CHAR)look_literal[2]; +end_max_ignore_rev: + while (text_ptr <= start_ptr && state->lower(text_ptr[-1]) != following_char) + text_ptr++; + break; + } + if (text_ptr > start_ptr) { + DISCARD_BACKTRACK(1) + goto backtrack; + } + backtrack_item->text_ptr = text_ptr; + repeat_counts[index] = repeat_min + (start_ptr - text_ptr); + pattern_ptr = tail; + goto advance; + } + case SRE_OP_REPEAT_ONE_MIN: + { + // Lazy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* repeat = backtrack_item->pattern_ptr; + int index = repeat[1]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = repeat + repeat[0]; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + Py_ssize_t limit = text_end - text_ptr; + text_ptr = backtrack_item->text_ptr; + if (repeat_max > repeat_counts[index] + limit || repeat_max == UNLIMITED_REPEATS) + repeat_max = repeat_counts[index] + limit; + /* Now match up to the maximum. */ + start_ptr = text_ptr - repeat_counts[index]; + end_ptr = start_ptr + repeat_max; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_MACRO(REPEAT_ONE_MIN_BACKTRACK_MACRO) + backtrack_item->text_ptr = text_ptr; + repeat_counts[index] = text_ptr - start_ptr; + pattern_ptr = tail; + goto advance; + } + case SRE_OP_REPEAT_ONE_MIN_REV: + { + // Lazy repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + SRE_CODE* repeat = backtrack_item->pattern_ptr; + int index = repeat[1]; + int repeat_max = repeat[3]; + SRE_CODE* body = repeat + 4; + SRE_CODE* tail = repeat + repeat[0]; + SRE_CODE* look_literal = tail; + SRE_CHAR* start_ptr; + SRE_CHAR* end_ptr; + SRE_CHAR following_char; + Py_ssize_t limit = text_ptr - text_start; + text_ptr = backtrack_item->text_ptr; + if (repeat_max > repeat_counts[index] + limit || repeat_max == UNLIMITED_REPEATS) + repeat_max = repeat_counts[index] + limit; + /* Now match up to the maximum. */ + start_ptr = text_ptr + repeat_counts[index]; + end_ptr = start_ptr - repeat_max; + while (look_literal[0] == SRE_OP_MARK) + look_literal += 2; + REPEAT_ONE_REV_MACRO(REPEAT_ONE_MIN_BACKTRACK_REV_MACRO) + backtrack_item->text_ptr = text_ptr; + repeat_counts[index] = start_ptr - text_ptr; + pattern_ptr = tail; + goto advance; + } + case SRE_OP_REPEAT_POSS: + { + // Possessive repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + TRACE(("REPEAT_POSS\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(1) + goto backtrack; + } + case SRE_OP_REPEAT_POSS_REV: + { + // Possessive repeat. + // ... + SRE_BACKTRACK_ITEM* backtrack_item = &backtrack_chunk->items[backtrack_chunk->count - 1]; + TRACE(("REPEAT_POSS\n")); + RESTORE_MARKS() + DISCARD_BACKTRACK(1) + goto backtrack; + } + default: + TRACE(("UNKNOWN %d\n", backtrack_chunk->items[backtrack_chunk->count - 1].op)); + SRE_RETURN(SRE_ERROR_ILLEGAL) } - return ret; /* should never get here */ + return 0; } LOCAL(Py_ssize_t) @@ -1491,125 +3695,15 @@ SRE_CHAR* ptr = (SRE_CHAR *)state->start; SRE_CHAR* end = (SRE_CHAR *)state->end; Py_ssize_t status = 0; - Py_ssize_t prefix_len = 0; - Py_ssize_t prefix_skip = 0; - SRE_CODE* prefix = NULL; - SRE_CODE* charset = NULL; - SRE_CODE* overlap = NULL; - int flags = 0; - if (pattern[0] == SRE_OP_INFO) { - /* optimization info block */ - /* <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> */ - - flags = pattern[2]; - - if (pattern[3] > 1) { - /* adjust end point (but make sure we leave at least one - character in there, so literal search will work) */ - end -= pattern[3]-1; - if (end <= ptr) - end = ptr+1; - } - - if (flags & SRE_INFO_PREFIX) { - /* pattern starts with a known prefix */ - /* */ - prefix_len = pattern[5]; - prefix_skip = pattern[6]; - prefix = pattern + 7; - overlap = prefix + prefix_len - 1; - } else if (flags & SRE_INFO_CHARSET) - /* pattern starts with a character from a known set */ - /* */ - charset = pattern + 5; - - pattern += 1 + pattern[1]; - } - - TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); - TRACE(("charset = %p\n", charset)); - -#if defined(USE_FAST_SEARCH) - if (prefix_len > 1) { - /* pattern starts with a known prefix. use the overlap - table to skip forward as fast as we possibly can */ - Py_ssize_t i = 0; - end = (SRE_CHAR *)state->end; - while (ptr < end) { - for (;;) { - if ((SRE_CODE) ptr[0] != prefix[i]) { - if (!i) - break; - else - i = overlap[i]; - } else { - if (++i == prefix_len) { - /* found a potential match */ - TRACE(("|%p|%p|SEARCH SCAN\n", pattern, ptr)); - state->start = ptr + 1 - prefix_len; - state->ptr = ptr + 1 - prefix_len + prefix_skip; - if (flags & SRE_INFO_LITERAL) - return 1; /* we got all of it */ - status = SRE_MATCH(state, pattern + 2*prefix_skip); - if (status != 0) - return status; - /* close but no cigar -- try again */ - i = overlap[i]; - } - break; - } - } - ptr++; - } - return 0; + // General case. + while (ptr <= end) { + TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); + state->start = state->ptr = ptr++; + status = SRE_MATCH(state, state->pattern_ptr); + if (status != 0) + break; } -#endif - - if (pattern[0] == SRE_OP_LITERAL) { - /* pattern starts with a literal character. this is used - for short prefixes, and if fast search is disabled */ - SRE_CODE chr = pattern[1]; - end = (SRE_CHAR *)state->end; - for (;;) { - while (ptr < end && (SRE_CODE) ptr[0] != chr) - ptr++; - if (ptr >= end) - return 0; - TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr)); - state->start = ptr; - state->ptr = ++ptr; - if (flags & SRE_INFO_LITERAL) - return 1; /* we got all of it */ - status = SRE_MATCH(state, pattern + 2); - if (status != 0) - break; - } - } else if (charset) { - /* pattern starts with a character from a known set */ - end = (SRE_CHAR *)state->end; - for (;;) { - while (ptr < end && !SRE_CHARSET(charset, ptr[0])) - ptr++; - if (ptr >= end) - return 0; - TRACE(("|%p|%p|SEARCH CHARSET\n", pattern, ptr)); - state->start = ptr; - state->ptr = ptr; - status = SRE_MATCH(state, pattern); - if (status != 0) - break; - ptr++; - } - } else - /* general case */ - while (ptr <= end) { - TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); - state->start = state->ptr = ptr++; - status = SRE_MATCH(state, pattern); - if (status != 0) - break; - } return status; } @@ -1666,8 +3760,6 @@ state->lastindex = -1; state->repeat = NULL; - - data_stack_dealloc(state); } static void* @@ -1738,22 +3830,62 @@ LOCAL(PyObject*) state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, - Py_ssize_t start, Py_ssize_t end) + Py_ssize_t start, Py_ssize_t end, SRE_CODE* pattern_code) { /* prepare state object */ + SRE_BACKTRACK_CHUNK* backtrack_chunk; + SRE_SAVED_MARKS_CHUNK* saved_marks_chunk; + int repeat_count; + SRE_CODE* pattern_start; Py_ssize_t length; int charsize; void* ptr; memset(state, 0, sizeof(SRE_STATE)); + /* There are actually 2 versions of backtrack_chunk and saved_marks_chunk, + 8-bit and Unicode. This shouldn't be a problem because they have the same format + and contain pointers and an int, which are always the same size. */ + backtrack_chunk = (SRE_BACKTRACK_CHUNK*)PyMem_MALLOC(sizeof(SRE_BACKTRACK_CHUNK)); + if (backtrack_chunk == NULL) + goto error; + + backtrack_chunk->previous = NULL; + backtrack_chunk->count = 0; + state->backtrack_chunk = backtrack_chunk; + + saved_marks_chunk = (SRE_SAVED_MARKS_CHUNK*)PyMem_MALLOC(sizeof(SRE_SAVED_MARKS_CHUNK)); + if (saved_marks_chunk == NULL) + goto error; + + saved_marks_chunk->previous = NULL; + saved_marks_chunk->count = 0; + state->saved_marks_chunk = saved_marks_chunk; + + // pattern_code: + repeat_count = pattern_code[1]; + pattern_start = pattern_code + 3; + + state->pattern_code = pattern_code; + state->pattern_ptr = pattern_start; + + // Ensure that element 0 of the repeat_counts array always exists. + state->repeat_counts = (int*)PyMem_MALLOC((repeat_count + 1) * sizeof(int)); + if (state->repeat_counts == NULL) + goto error; + + // Ensure that element 0 of the repeat_starts array always exists. + state->repeat_starts = (void**)PyMem_MALLOC((repeat_count + 1) * sizeof(void*)); + if (state->repeat_starts == NULL) + goto error; + state->lastmark = -1; state->lastindex = -1; ptr = getstring(string, &length, &charsize); if (!ptr) - return NULL; + goto error; /* adjust boundaries */ if (start < 0) @@ -1790,13 +3922,41 @@ state->lower = sre_lower; return string; + +error: + PyMem_FREE(state->repeat_starts); + PyMem_FREE(state->repeat_counts); + PyMem_FREE(state->saved_marks_chunk); + PyMem_FREE(state->backtrack_chunk); + return NULL; } LOCAL(void) state_fini(SRE_STATE* state) { + /* There are actually 2 versions of backtrack_chunk and saved_marks_chunk, + 8-bit and Unicode. This shouldn't be a problem because they have the same format + and contain pointers and an int, which are always the same size. */ + SRE_SAVED_MARKS_CHUNK* saved_marks_chunk = state->saved_marks_chunk; + SRE_BACKTRACK_CHUNK* backtrack_chunk = state->backtrack_chunk; + + while (saved_marks_chunk != NULL) { + SRE_SAVED_MARKS_CHUNK* previous = saved_marks_chunk->previous; + PyMem_FREE(saved_marks_chunk); + saved_marks_chunk = previous; + } + state->saved_marks_chunk = NULL; + + while (backtrack_chunk != NULL) { + SRE_BACKTRACK_CHUNK* previous = backtrack_chunk->previous; + PyMem_FREE(backtrack_chunk); + backtrack_chunk = previous; + } + state->backtrack_chunk = NULL; + + PyMem_FREE(state->repeat_starts); + PyMem_FREE(state->repeat_counts); Py_XDECREF(state->string); - data_stack_dealloc(state); } /* calculate offset from start of string */ @@ -1867,7 +4027,7 @@ { SRE_STATE state; int status; - + SRE_CODE* pattern_code; PyObject* string; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; @@ -1876,7 +4036,9 @@ &string, &start, &end)) return NULL; - string = state_init(&state, self, string, start, end); + pattern_code = PatternObject_GetCode(self); + + string = state_init(&state, self, string, start, end, pattern_code); if (!string) return NULL; @@ -1885,10 +4047,10 @@ TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr)); if (state.charsize == 1) { - status = sre_match(&state, PatternObject_GetCode(self)); + status = sre_match(&state, state.pattern_ptr); } else { #if defined(HAVE_UNICODE) - status = sre_umatch(&state, PatternObject_GetCode(self)); + status = sre_umatch(&state, state.pattern_ptr); #endif } @@ -1906,7 +4068,7 @@ { SRE_STATE state; int status; - + SRE_CODE* pattern_code; PyObject* string; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; @@ -1915,17 +4077,19 @@ &string, &start, &end)) return NULL; - string = state_init(&state, self, string, start, end); + pattern_code = PatternObject_GetCode(self); + + string = state_init(&state, self, string, start, end, pattern_code); if (!string) return NULL; - TRACE(("|%p|%p|SEARCH\n", PatternObject_GetCode(self), state.ptr)); + TRACE(("|%p|%p|SEARCH\n", pattern, state.ptr)); if (state.charsize == 1) { - status = sre_search(&state, PatternObject_GetCode(self)); + status = sre_search(&state, state.pattern_ptr); } else { #if defined(HAVE_UNICODE) - status = sre_usearch(&state, PatternObject_GetCode(self)); + status = sre_usearch(&state, state.pattern_ptr); #endif } @@ -2041,7 +4205,7 @@ PyObject* list; int status; Py_ssize_t i, b, e; - + SRE_CODE* pattern_code; PyObject* string; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; @@ -2050,7 +4214,9 @@ &string, &start, &end)) return NULL; - string = state_init(&state, self, string, start, end); + pattern_code = PatternObject_GetCode(self); + + string = state_init(&state, self, string, start, end, pattern_code); if (!string) return NULL; @@ -2069,15 +4235,15 @@ state.ptr = state.start; if (state.charsize == 1) { - status = sre_search(&state, PatternObject_GetCode(self)); + status = sre_search(&state, state.pattern_ptr); } else { #if defined(HAVE_UNICODE) - status = sre_usearch(&state, PatternObject_GetCode(self)); + status = sre_usearch(&state, state.pattern_ptr); #endif } - if (PyErr_Occurred()) - goto error; + if (PyErr_Occurred()) + goto error; if (status <= 0) { if (status == 0) @@ -2168,6 +4334,7 @@ PyObject* list; PyObject* item; int status; + SRE_CODE* pattern_code; Py_ssize_t n; Py_ssize_t i; void* last; @@ -2179,7 +4346,9 @@ &string, &maxsplit)) return NULL; - string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX); + pattern_code = PatternObject_GetCode(self); + + string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX, pattern_code); if (!string) return NULL; @@ -2199,15 +4368,15 @@ state.ptr = state.start; if (state.charsize == 1) { - status = sre_search(&state, PatternObject_GetCode(self)); + status = sre_search(&state, state.pattern_ptr); } else { #if defined(HAVE_UNICODE) - status = sre_usearch(&state, PatternObject_GetCode(self)); + status = sre_usearch(&state, state.pattern_ptr); #endif } - if (PyErr_Occurred()) - goto error; + if (PyErr_Occurred()) + goto error; if (status <= 0) { if (status == 0) @@ -2290,6 +4459,7 @@ Py_ssize_t i, b, e; int bint; int filter_is_callable; + SRE_CODE* pattern_code; if (PyCallable_Check(ptemplate)) { /* sub/subn takes either a function or a template */ @@ -2329,7 +4499,9 @@ } } - string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX); + pattern_code = PatternObject_GetCode(self); + + string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX, pattern_code); if (!string) { Py_DECREF(filter); return NULL; @@ -2351,15 +4523,15 @@ state.ptr = state.start; if (state.charsize == 1) { - status = sre_search(&state, PatternObject_GetCode(self)); + status = sre_search(&state, state.pattern_ptr); } else { #if defined(HAVE_UNICODE) - status = sre_usearch(&state, PatternObject_GetCode(self)); + status = sre_usearch(&state, state.pattern_ptr); #endif } - if (PyErr_Occurred()) - goto error; + if (PyErr_Occurred()) + goto error; if (status <= 0) { if (status == 0) @@ -2658,6 +4830,8 @@ offsetof(PatternObject, weakreflist), /* tp_weaklistoffset */ }; +static int _validate(PatternObject *self); /* Forward */ + static PyObject * _compile(PyObject* self_, PyObject* args) { @@ -2678,7 +4852,7 @@ return NULL; n = PyList_GET_SIZE(code); - + /* coverity[ampersand_in_size] */ self = PyObject_NEW_VAR(PatternObject, &Pattern_Type, n); if (!self) return NULL; @@ -2717,10 +4891,482 @@ self->weakreflist = NULL; + if (!_validate(self)) { + Py_DECREF(self); + return NULL; + } + return (PyObject*) self; } /* -------------------------------------------------------------------- */ +/* Code validation */ + +/* To learn more about this code, have a look at the _compile() function in + Lib/sre_compile.py. The validation functions below checks the code array + for conformance with the code patterns generated there. + + The nice thing about the generated code is that it is position-independent: + all jumps are relative jumps forward. Also, jumps don't cross each other: + the target of a later jump is always earlier than the target of an earlier + jump. IOW, this is okay: + + J---------J-------T--------T + \ \_____/ / + \______________________/ + + but this is not: + + J---------J-------T--------T + \_________\_____/ / + \____________/ + + It also helps that SRE_CODE is always an unsigned type, either 2 bytes or 4 + bytes wide (the latter if Python is compiled for "wide" unicode support). +*/ + +/* Defining this one enables tracing of the validator */ +#undef VVERBOSE + +/* Trace macro for the validator */ +#if defined(VVERBOSE) +#define VTRACE(v) printf v +#else +#define VTRACE(v) +#endif + +/* Report failure */ +#define FAIL do { VTRACE(("FAIL: %d\n", __LINE__)); return 0; } while (0) + +/* Extract opcode, argument, or skip count from code array */ +#define GET_OP \ + do { \ + VTRACE(("%p: ", code)); \ + if (code >= end) FAIL; \ + op = *code++; \ + VTRACE(("%lu (op)\n", (unsigned long)op)); \ + } while (0) +#define GET_ARG \ + do { \ + VTRACE(("%p= ", code)); \ + if (code >= end) FAIL; \ + arg = *code++; \ + VTRACE(("%lu (arg)\n", (unsigned long)arg)); \ + } while (0) +#define GET_SKIP \ + do { \ + VTRACE(("%p= ", code)); \ + if (code >= end) FAIL; \ + skip = *code; \ + VTRACE(("%lu (skip to %p)\n", \ + (unsigned long)skip, code+skip)); \ + if (code+skip < code || code+skip > end) \ + FAIL; \ + code++; \ + } while (0) + +static int +_validate_charset(SRE_CODE *code, SRE_CODE *end) +{ + /* Some variables are manipulated by the macros above */ + SRE_CODE op; + SRE_CODE arg; + SRE_CODE offset; + int i; + + while (code < end) { + GET_OP; + switch (op) { + + case SRE_OP_NEGATE: + break; + + case SRE_OP_LITERAL: + GET_ARG; + break; + + case SRE_OP_RANGE: + GET_ARG; + GET_ARG; + break; + + case SRE_OP_CHARSET: + offset = 32/sizeof(SRE_CODE); /* 32-byte bitmap */ + if (code+offset < code || code+offset > end) + FAIL; + code += offset; + break; + + case SRE_OP_BIGCHARSET: + GET_ARG; /* Number of blocks */ + offset = 256/sizeof(SRE_CODE); /* 256-byte table */ + if (code+offset < code || code+offset > end) + FAIL; + /* Make sure that each byte points to a valid block */ + for (i = 0; i < 256; i++) { + if (((unsigned char *)code)[i] >= arg) + FAIL; + } + code += offset; + offset = arg * 32/sizeof(SRE_CODE); /* 32-byte bitmap times arg */ + if (code+offset < code || code+offset > end) + FAIL; + code += offset; + break; + + case SRE_OP_CATEGORY: + GET_ARG; + switch (arg) { + case SRE_CATEGORY_DIGIT: + case SRE_CATEGORY_NOT_DIGIT: + case SRE_CATEGORY_SPACE: + case SRE_CATEGORY_NOT_SPACE: + case SRE_CATEGORY_WORD: + case SRE_CATEGORY_NOT_WORD: + case SRE_CATEGORY_LINEBREAK: + case SRE_CATEGORY_NOT_LINEBREAK: + case SRE_CATEGORY_LOC_WORD: + case SRE_CATEGORY_LOC_NOT_WORD: + case SRE_CATEGORY_UNI_DIGIT: + case SRE_CATEGORY_UNI_NOT_DIGIT: + case SRE_CATEGORY_UNI_SPACE: + case SRE_CATEGORY_UNI_NOT_SPACE: + case SRE_CATEGORY_UNI_WORD: + case SRE_CATEGORY_UNI_NOT_WORD: + case SRE_CATEGORY_UNI_LINEBREAK: + case SRE_CATEGORY_UNI_NOT_LINEBREAK: + break; + default: + FAIL; + } + break; + + default: + FAIL; + + } + } + + return 1; +} + +static int +_validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +{ + /* Some variables are manipulated by the macros above */ + SRE_CODE op; + SRE_CODE arg; + SRE_CODE skip; + + VTRACE(("code=%p, end=%p\n", code, end)); + + if (code > end) + FAIL; + + while (code < end) { + GET_OP; + switch (op) { + + case SRE_OP_MARK: + /* We don't check whether marks are properly nested; the + sre_match() code is robust even if they don't, and the worst + you can get is nonsensical match results. */ + GET_ARG; + if (arg > 2*groups+1) { + VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); + FAIL; + } + break; + + case SRE_OP_LITERAL: + case SRE_OP_NOT_LITERAL: + case SRE_OP_LITERAL_IGNORE: + case SRE_OP_NOT_LITERAL_IGNORE: + GET_ARG; + /* The arg is just a character, nothing to check */ + break; + + case SRE_OP_SUCCESS: + case SRE_OP_FAILURE: + /* Nothing to check; these normally end the matching process */ + break; + + case SRE_OP_AT: + GET_ARG; + switch (arg) { + case SRE_AT_BEGINNING: + case SRE_AT_BEGINNING_STRING: + case SRE_AT_BEGINNING_LINE: + case SRE_AT_END: + case SRE_AT_END_LINE: + case SRE_AT_END_STRING: + case SRE_AT_BOUNDARY: + case SRE_AT_NON_BOUNDARY: + case SRE_AT_LOC_BOUNDARY: + case SRE_AT_LOC_NON_BOUNDARY: + case SRE_AT_UNI_BOUNDARY: + case SRE_AT_UNI_NON_BOUNDARY: + break; + default: + FAIL; + } + break; + + case SRE_OP_ANY: + case SRE_OP_ANY_ALL: + /* These have no operands */ + break; + + case SRE_OP_IN: + case SRE_OP_IN_IGNORE: + GET_SKIP; + /* Stop 1 before the end; we check the FAILURE below */ + if (!_validate_charset(code, code+skip-2)) + FAIL; + if (code[skip-2] != SRE_OP_FAILURE) + FAIL; + code += skip-1; + break; + + case SRE_OP_INFO: + { + /* A minimal info field is + <1=skip> <2=flags> <3=min> <4=max>; + If SRE_INFO_PREFIX or SRE_INFO_CHARSET is in the flags, + more follows. */ + SRE_CODE flags, min, max, i; + SRE_CODE *newcode; + GET_SKIP; + newcode = code+skip-1; + GET_ARG; flags = arg; + GET_ARG; min = arg; + GET_ARG; max = arg; + /* Check that only valid flags are present */ + if ((flags & ~(SRE_INFO_PREFIX | + SRE_INFO_LITERAL | + SRE_INFO_CHARSET)) != 0) + FAIL; + /* PREFIX and CHARSET are mutually exclusive */ + if ((flags & SRE_INFO_PREFIX) && + (flags & SRE_INFO_CHARSET)) + FAIL; + /* LITERAL implies PREFIX */ + if ((flags & SRE_INFO_LITERAL) && + !(flags & SRE_INFO_PREFIX)) + FAIL; + /* Validate the prefix */ + if (flags & SRE_INFO_PREFIX) { + SRE_CODE prefix_len, prefix_skip; + GET_ARG; prefix_len = arg; + GET_ARG; prefix_skip = arg; + /* Here comes the prefix string */ + if (code+prefix_len < code || code+prefix_len > newcode) + FAIL; + code += prefix_len; + /* And here comes the overlap table */ + if (code+prefix_len < code || code+prefix_len > newcode) + FAIL; + /* Each overlap value should be < prefix_len */ + for (i = 0; i < prefix_len; i++) { + if (code[i] >= prefix_len) + FAIL; + } + code += prefix_len; + } + /* Validate the charset */ + if (flags & SRE_INFO_CHARSET) { + if (!_validate_charset(code, newcode-1)) + FAIL; + if (newcode[-1] != SRE_OP_FAILURE) + FAIL; + code = newcode; + } + else if (code != newcode) { + VTRACE(("code=%p, newcode=%p\n", code, newcode)); + FAIL; + } + } + break; + + case SRE_OP_BRANCH: + { + SRE_CODE *target = NULL; + for (;;) { + GET_SKIP; + if (skip == 0) + break; + /* Stop 2 before the end; we check the JUMP below */ + if (!_validate_inner(code, code+skip-3, groups)) + FAIL; + code += skip-3; + /* Check that it ends with a JUMP, and that each JUMP + has the same target */ + GET_OP; + if (op != SRE_OP_JUMP) + FAIL; + GET_SKIP; + if (target == NULL) + target = code+skip-1; + else if (code+skip-1 != target) + FAIL; + } + } + break; + + case SRE_OP_REPEAT_ONE: + case SRE_OP_MIN_REPEAT_ONE: + { + SRE_CODE min, max; + GET_SKIP; + GET_ARG; min = arg; + GET_ARG; max = arg; + if (min > max) + FAIL; +#ifdef Py_UNICODE_WIDE + if (max > 65535) + FAIL; +#endif + if (!_validate_inner(code, code+skip-4, groups)) + FAIL; + code += skip-4; + GET_OP; + if (op != SRE_OP_SUCCESS) + FAIL; + } + break; + + case SRE_OP_REPEAT: + { + SRE_CODE min, max; + GET_SKIP; + GET_ARG; min = arg; + GET_ARG; max = arg; + if (min > max) + FAIL; +#ifdef Py_UNICODE_WIDE + if (max > 65535) + FAIL; +#endif + if (!_validate_inner(code, code+skip-3, groups)) + FAIL; + code += skip-3; + GET_OP; + if (op != SRE_OP_MAX_UNTIL && op != SRE_OP_MIN_UNTIL) + FAIL; + } + break; + + case SRE_OP_GROUPREF: + case SRE_OP_GROUPREF_IGNORE: + GET_ARG; + if (arg >= groups) + FAIL; + break; + + case SRE_OP_GROUPREF_EXISTS: + /* The regex syntax for this is: '(?(group)then|else)', where + 'group' is either an integer group number or a group name, + 'then' and 'else' are sub-regexes, and 'else' is optional. */ + GET_ARG; + if (arg >= groups) + FAIL; + GET_SKIP; + code--; /* The skip is relative to the first arg! */ + /* There are two possibilities here: if there is both a 'then' + part and an 'else' part, the generated code looks like: + + GROUPREF_EXISTS + + + ...then part... + JUMP + + ( jumps here) + ...else part... + ( jumps here) + + If there is only a 'then' part, it looks like: + + GROUPREF_EXISTS + + + ...then part... + ( jumps here) + + There is no direct way to decide which it is, and we don't want + to allow arbitrary jumps anywhere in the code; so we just look + for a JUMP opcode preceding our skip target. + */ + if (skip >= 3 && code+skip-3 >= code && + code[skip-3] == SRE_OP_JUMP) + { + VTRACE(("both then and else parts present\n")); + if (!_validate_inner(code+1, code+skip-3, groups)) + FAIL; + code += skip-2; /* Position after JUMP, at */ + GET_SKIP; + if (!_validate_inner(code, code+skip-1, groups)) + FAIL; + code += skip-1; + } + else { + VTRACE(("only a then part present\n")); + if (!_validate_inner(code+1, code+skip-1, groups)) + FAIL; + code += skip-1; + } + break; + + case SRE_OP_ASSERT: + case SRE_OP_ASSERT_NOT: + GET_SKIP; + GET_ARG; /* 0 for lookahead, width for lookbehind */ + code--; /* Back up over arg to simplify math below */ + if (arg & 0x80000000) + FAIL; /* Width too large */ + /* Stop 1 before the end; we check the SUCCESS below */ + if (!_validate_inner(code+1, code+skip-2, groups)) + FAIL; + code += skip-2; + GET_OP; + if (op != SRE_OP_SUCCESS) + FAIL; + break; + + default: + FAIL; + + } + } + + VTRACE(("okay\n")); + return 1; +} + +static int +_validate_outer(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +{ + if (groups < 0 || groups > 100 || code >= end || end[-1] != SRE_OP_SUCCESS) + FAIL; + if (groups == 0) /* fix for simplejson */ + groups = 100; /* 100 groups should always be safe */ + return _validate_inner(code, end-1, groups); +} + +static int +_validate(PatternObject *self) +{ + if (!_validate_outer(self->code, self->code+self->codesize, self->groups)) + { + PyErr_SetString(PyExc_RuntimeError, "invalid SRE code"); + return 0; + } + else + VTRACE(("Success!\n")); + return 1; +} + +/* -------------------------------------------------------------------- */ /* match methods */ static void @@ -3185,6 +5831,7 @@ if (status > 0) { /* create match object (with room for extra group marks) */ + /* coverity[ampersand_in_size] */ match = PyObject_NEW_VAR(MatchObject, &Match_Type, 2*(pattern->groups+1)); if (!match) @@ -3256,12 +5903,13 @@ state_reset(state); state->ptr = state->start; + memset(state->mark, 0, state->pattern_code[0] * sizeof(SRE_CHAR*)); if (state->charsize == 1) { - status = sre_match(state, PatternObject_GetCode(self->pattern)); + status = sre_match(state, state->pattern_ptr); } else { #if defined(HAVE_UNICODE) - status = sre_umatch(state, PatternObject_GetCode(self->pattern)); + status = sre_umatch(state, state->pattern_ptr); #endif } if (PyErr_Occurred()) @@ -3291,10 +5939,10 @@ state->ptr = state->start; if (state->charsize == 1) { - status = sre_search(state, PatternObject_GetCode(self->pattern)); + status = sre_search(state, state->pattern_ptr); } else { #if defined(HAVE_UNICODE) - status = sre_usearch(state, PatternObject_GetCode(self->pattern)); + status = sre_usearch(state, state->pattern_ptr); #endif } if (PyErr_Occurred()) @@ -3357,6 +6005,8 @@ PyObject* string; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; + SRE_CODE* pattern_code; + if (!PyArg_ParseTuple(args, "O|nn:scanner", &string, &start, &end)) return NULL; @@ -3365,7 +6015,9 @@ if (!self) return NULL; - string = state_init(&self->state, pattern, string, start, end); + pattern_code = PatternObject_GetCode(pattern); + + string = state_init(&self->state, pattern, string, start, end, pattern_code); if (!string) { PyObject_DEL(self); return NULL; Index: Modules/sre.h =================================================================== --- sre.h Mon Jun 12 04:05:40 2006 +++ sre.h Tue Sep 09 00:00:06 2008 @@ -60,6 +60,30 @@ struct SRE_REPEAT_T *prev; /* points to previous repeat context */ } SRE_REPEAT; +#define SRE_BACKTRACK_CHUNK_SIZE 1024 +#define SRE_SAVED_MARKS_CHUNK_SIZE 1024 + +typedef struct SRE_BACKTRACK_ITEM { + int op; + SRE_CODE* pattern_ptr; + void* text_ptr; + int index; + int repeat_count; + void* repeat_start; +} SRE_BACKTRACK_ITEM; + +typedef struct SRE_BACKTRACK_CHUNK { + struct SRE_BACKTRACK_CHUNK* previous; + SRE_BACKTRACK_ITEM items[SRE_BACKTRACK_CHUNK_SIZE]; + int count; +} SRE_BACKTRACK_CHUNK; + +typedef struct SRE_SAVED_MARKS_CHUNK { + struct SRE_SAVED_MARKS_CHUNK* previous; + void* marks[SRE_SAVED_MARKS_CHUNK_SIZE]; + int count; +} SRE_SAVED_MARKS_CHUNK; + typedef struct { /* string pointers */ void* ptr; /* current position (also end of current slice) */ @@ -69,6 +93,7 @@ /* attributes for the match object */ PyObject* string; Py_ssize_t pos, endpos; + int reversed; /* character size */ int charsize; /* registers */ @@ -76,9 +101,12 @@ Py_ssize_t lastmark; void* mark[SRE_MARK_SIZE]; /* dynamically allocated stuff */ - char* data_stack; - size_t data_stack_size; - size_t data_stack_base; + SRE_BACKTRACK_CHUNK* backtrack_chunk; + SRE_SAVED_MARKS_CHUNK* saved_marks_chunk; + int* repeat_counts; + void** repeat_starts; + SRE_CODE* pattern_code; + SRE_CODE* pattern_ptr; /* current repeat context */ SRE_REPEAT *repeat; /* hooks */ Index: Modules/sre_constants.h =================================================================== --- sre_constants.h Fri Oct 17 23:13:16 2003 +++ sre_constants.h Tue Sep 09 23:35:04 2008 @@ -16,64 +16,131 @@ #define SRE_OP_SUCCESS 1 #define SRE_OP_ANY 2 #define SRE_OP_ANY_ALL 3 -#define SRE_OP_ASSERT 4 -#define SRE_OP_ASSERT_NOT 5 -#define SRE_OP_AT 6 -#define SRE_OP_BRANCH 7 -#define SRE_OP_CALL 8 -#define SRE_OP_CATEGORY 9 -#define SRE_OP_CHARSET 10 -#define SRE_OP_BIGCHARSET 11 -#define SRE_OP_GROUPREF 12 -#define SRE_OP_GROUPREF_EXISTS 13 -#define SRE_OP_GROUPREF_IGNORE 14 -#define SRE_OP_IN 15 -#define SRE_OP_IN_IGNORE 16 -#define SRE_OP_INFO 17 -#define SRE_OP_JUMP 18 -#define SRE_OP_LITERAL 19 -#define SRE_OP_LITERAL_IGNORE 20 -#define SRE_OP_MARK 21 -#define SRE_OP_MAX_UNTIL 22 -#define SRE_OP_MIN_UNTIL 23 -#define SRE_OP_NOT_LITERAL 24 -#define SRE_OP_NOT_LITERAL_IGNORE 25 -#define SRE_OP_NEGATE 26 -#define SRE_OP_RANGE 27 -#define SRE_OP_REPEAT 28 -#define SRE_OP_REPEAT_ONE 29 -#define SRE_OP_SUBPATTERN 30 -#define SRE_OP_MIN_REPEAT_ONE 31 -#define SRE_AT_BEGINNING 0 -#define SRE_AT_BEGINNING_LINE 1 -#define SRE_AT_BEGINNING_STRING 2 -#define SRE_AT_BOUNDARY 3 -#define SRE_AT_NON_BOUNDARY 4 -#define SRE_AT_END 5 -#define SRE_AT_END_LINE 6 -#define SRE_AT_END_STRING 7 -#define SRE_AT_LOC_BOUNDARY 8 -#define SRE_AT_LOC_NON_BOUNDARY 9 -#define SRE_AT_UNI_BOUNDARY 10 -#define SRE_AT_UNI_NON_BOUNDARY 11 -#define SRE_CATEGORY_DIGIT 0 -#define SRE_CATEGORY_NOT_DIGIT 1 -#define SRE_CATEGORY_SPACE 2 -#define SRE_CATEGORY_NOT_SPACE 3 -#define SRE_CATEGORY_WORD 4 -#define SRE_CATEGORY_NOT_WORD 5 -#define SRE_CATEGORY_LINEBREAK 6 -#define SRE_CATEGORY_NOT_LINEBREAK 7 -#define SRE_CATEGORY_LOC_WORD 8 -#define SRE_CATEGORY_LOC_NOT_WORD 9 -#define SRE_CATEGORY_UNI_DIGIT 10 -#define SRE_CATEGORY_UNI_NOT_DIGIT 11 -#define SRE_CATEGORY_UNI_SPACE 12 -#define SRE_CATEGORY_UNI_NOT_SPACE 13 -#define SRE_CATEGORY_UNI_WORD 14 -#define SRE_CATEGORY_UNI_NOT_WORD 15 -#define SRE_CATEGORY_UNI_LINEBREAK 16 -#define SRE_CATEGORY_UNI_NOT_LINEBREAK 17 +#define SRE_OP_ANY_ALL_REV 4 +#define SRE_OP_ANY_REV 5 +#define SRE_OP_ASSERT 6 +#define SRE_OP_ASSERT_NOT 7 +#define SRE_OP_ATOMIC 8 +#define SRE_OP_BIGCHARSET 9 +#define SRE_OP_BIGCHARSET_IGNORE 10 +#define SRE_OP_BIGCHARSET_IGNORE_REV 11 +#define SRE_OP_BIGCHARSET_REV 12 +#define SRE_OP_BOUNDARY 13 +#define SRE_OP_BRANCH 14 +#define SRE_OP_CALL 15 +#define SRE_OP_CHARSET 16 +#define SRE_OP_CHARSET_IGNORE 17 +#define SRE_OP_CHARSET_IGNORE_REV 18 +#define SRE_OP_CHARSET_REV 19 +#define SRE_OP_DIGIT 20 +#define SRE_OP_DIGIT_REV 21 +#define SRE_OP_END_ASSERT 22 +#define SRE_OP_END_ASSERT_NOT 23 +#define SRE_OP_END_ATOMIC 24 +#define SRE_OP_END_OF_LINE 25 +#define SRE_OP_END_OF_STRING 26 +#define SRE_OP_END_OF_STRING_2 27 +#define SRE_OP_END_REPEAT_MAX 28 +#define SRE_OP_END_REPEAT_MAX_REV 29 +#define SRE_OP_END_REPEAT_MIN 30 +#define SRE_OP_END_REPEAT_MIN_REV 31 +#define SRE_OP_END_REPEAT_ONE_MAX 32 +#define SRE_OP_END_REPEAT_ONE_MAX_REV 33 +#define SRE_OP_END_REPEAT_ONE_MIN 34 +#define SRE_OP_END_REPEAT_ONE_MIN_REV 35 +#define SRE_OP_END_REPEAT_ONE_POSS 36 +#define SRE_OP_END_REPEAT_ONE_POSS_REV 37 +#define SRE_OP_END_REPEAT_POSS 38 +#define SRE_OP_END_REPEAT_POSS_REV 39 +#define SRE_OP_GROUPREF 40 +#define SRE_OP_GROUPREF_EXISTS 41 +#define SRE_OP_GROUPREF_IGNORE 42 +#define SRE_OP_GROUPREF_IGNORE_REV 43 +#define SRE_OP_GROUPREF_REV 44 +#define SRE_OP_IN 45 +#define SRE_OP_IN_IGNORE 46 +#define SRE_OP_IN_IGNORE_REV 47 +#define SRE_OP_IN_REV 48 +#define SRE_OP_JUMP 49 +#define SRE_OP_LITERAL 50 +#define SRE_OP_LITERAL_IGNORE 51 +#define SRE_OP_LITERAL_IGNORE_REV 52 +#define SRE_OP_LITERAL_REV 53 +#define SRE_OP_LITERAL_STRING 54 +#define SRE_OP_LITERAL_STRING_IGNORE 55 +#define SRE_OP_LITERAL_STRING_IGNORE_REV 56 +#define SRE_OP_LITERAL_STRING_REV 57 +#define SRE_OP_LOC_BOUNDARY 58 +#define SRE_OP_LOC_NOT_BOUNDARY 59 +#define SRE_OP_LOC_NOT_WORD 60 +#define SRE_OP_LOC_NOT_WORD_REV 61 +#define SRE_OP_LOC_WORD 62 +#define SRE_OP_LOC_WORD_REV 63 +#define SRE_OP_MARK 64 +#define SRE_OP_NOT_BIGCHARSET 65 +#define SRE_OP_NOT_BIGCHARSET_IGNORE 66 +#define SRE_OP_NOT_BIGCHARSET_IGNORE_REV 67 +#define SRE_OP_NOT_BIGCHARSET_REV 68 +#define SRE_OP_NOT_BOUNDARY 69 +#define SRE_OP_NOT_CHARSET 70 +#define SRE_OP_NOT_CHARSET_IGNORE 71 +#define SRE_OP_NOT_CHARSET_IGNORE_REV 72 +#define SRE_OP_NOT_CHARSET_REV 73 +#define SRE_OP_NOT_DIGIT 74 +#define SRE_OP_NOT_DIGIT_REV 75 +#define SRE_OP_NOT_IN 76 +#define SRE_OP_NOT_IN_IGNORE 77 +#define SRE_OP_NOT_IN_IGNORE_REV 78 +#define SRE_OP_NOT_IN_REV 79 +#define SRE_OP_NOT_LITERAL 80 +#define SRE_OP_NOT_LITERAL_IGNORE 81 +#define SRE_OP_NOT_LITERAL_IGNORE_REV 82 +#define SRE_OP_NOT_LITERAL_REV 83 +#define SRE_OP_NOT_RANGE 84 +#define SRE_OP_NOT_RANGE_IGNORE 85 +#define SRE_OP_NOT_RANGE_IGNORE_REV 86 +#define SRE_OP_NOT_RANGE_REV 87 +#define SRE_OP_NOT_WHITESPACE 88 +#define SRE_OP_NOT_WHITESPACE_REV 89 +#define SRE_OP_NOT_WORD 90 +#define SRE_OP_NOT_WORD_REV 91 +#define SRE_OP_RANGE 92 +#define SRE_OP_RANGE_IGNORE 93 +#define SRE_OP_RANGE_IGNORE_REV 94 +#define SRE_OP_RANGE_REV 95 +#define SRE_OP_REPEAT_MAX 96 +#define SRE_OP_REPEAT_MAX_REV 97 +#define SRE_OP_REPEAT_MIN 98 +#define SRE_OP_REPEAT_MIN_REV 99 +#define SRE_OP_REPEAT_ONE_MAX 100 +#define SRE_OP_REPEAT_ONE_MAX_REV 101 +#define SRE_OP_REPEAT_ONE_MIN 102 +#define SRE_OP_REPEAT_ONE_MIN_REV 103 +#define SRE_OP_REPEAT_ONE_POSS 104 +#define SRE_OP_REPEAT_ONE_POSS_REV 105 +#define SRE_OP_REPEAT_POSS 106 +#define SRE_OP_REPEAT_POSS_REV 107 +#define SRE_OP_START_OF_LINE 108 +#define SRE_OP_START_OF_STRING 109 +#define SRE_OP_SUBPATTERN 110 +#define SRE_OP_UNI_BOUNDARY 111 +#define SRE_OP_UNI_DIGIT 112 +#define SRE_OP_UNI_DIGIT_REV 113 +#define SRE_OP_UNI_NOT_BOUNDARY 114 +#define SRE_OP_UNI_NOT_DIGIT 115 +#define SRE_OP_UNI_NOT_DIGIT_REV 116 +#define SRE_OP_UNI_NOT_WHITESPACE 117 +#define SRE_OP_UNI_NOT_WHITESPACE_REV 118 +#define SRE_OP_UNI_NOT_WORD 119 +#define SRE_OP_UNI_NOT_WORD_REV 120 +#define SRE_OP_UNI_WHITESPACE 121 +#define SRE_OP_UNI_WHITESPACE_REV 122 +#define SRE_OP_UNI_WORD 123 +#define SRE_OP_UNI_WORD_REV 124 +#define SRE_OP_WHITESPACE 125 +#define SRE_OP_WHITESPACE_REV 126 +#define SRE_OP_WORD 127 +#define SRE_OP_WORD_REV 128 #define SRE_FLAG_TEMPLATE 1 #define SRE_FLAG_IGNORECASE 2 #define SRE_FLAG_LOCALE 4