Index: Lib/re.py =================================================================== --- re.py Tue Feb 05 18:31:37 2008 +++ re.py Mon Sep 08 16:28:07 2008 @@ -36,8 +36,10 @@ "+" Matches 1 or more (greedy) repetitions of the preceding RE. "?" Matches 0 or 1 (greedy) of the preceding RE. *?,+?,?? Non-greedy versions of the previous three special characters. + *+,++,?+ Possessive versions of the previous three special characters. {m,n} Matches from m to n repetitions of the preceding RE. {m,n}? Non-greedy version of the above. + {m,n}+ Possessive version of the above. "\\" Either escapes special characters or signals a special sequence. [] Indicates a set of characters. A "^" as the first character indicates a complementing set. @@ -45,10 +47,12 @@ (...) Matches the RE inside the parentheses. The contents can be retrieved or matched later in the string. (?iLmsux) Set the I, L, M, S, U, or X flag for the RE (see below). - (?:...) Non-grouping version of regular parentheses. + (?:...) Non-capturing version of regular parentheses. (?P...) The substring matched by the group is accessible by name. (?P=name) Matches the text matched earlier by the group named name. (?#...) A comment; ignored. + (?>...) Atomic group. Like (?:...) but won't retry the RE within the + parentheses. (?=...) Matches if ... matches next, but doesn't consume the string. (?!...) Matches if ... doesn't match next. (?<=...) Matches if preceded by ... (must be fixed length). @@ -111,7 +115,7 @@ "U", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", "UNICODE", "error" ] -__version__ = "2.2.1" +__version__ = "2.2.2" # flags I = IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE # ignore case Index: Lib/sre_compile.py =================================================================== --- sre_compile.py Thu Jan 03 20:08:15 2008 +++ sre_compile.py Tue Sep 09 20:38:04 2008 @@ -17,266 +17,414 @@ assert _sre.MAGIC == MAGIC, "SRE module mismatch" if _sre.CODESIZE == 2: - MAXCODE = 65535 + MAXCODE = 0xFFFF + _subchunk_size = 16 else: MAXCODE = 0xFFFFFFFFL + _subchunk_size = 32 +_subchunk_mask = (1 << _subchunk_size) - 1 -def _identityfunction(x): - return x +ASSERT_CODES = { + ASSERT: END_ASSERT, + ASSERT_NOT: END_ASSERT_NOT, +} + +CATEGORY_CODES = [ + (ANY, ANY, ANY), + (BOUNDARY, LOC_BOUNDARY, UNI_BOUNDARY), + (DIGIT, DIGIT, UNI_DIGIT), + (NOT_BOUNDARY, LOC_NOT_BOUNDARY, UNI_NOT_BOUNDARY), + (NOT_DIGIT, NOT_DIGIT, UNI_NOT_DIGIT), + (NOT_WHITESPACE, NOT_WHITESPACE, UNI_NOT_WHITESPACE), + (NOT_WORD, LOC_NOT_WORD, UNI_NOT_WORD), + (WHITESPACE, WHITESPACE, UNI_WHITESPACE), + (WORD, LOC_WORD, UNI_WORD), +] + +IGNORE_CODES = [ + (BIGCHARSET, BIGCHARSET_IGNORE), + (CHARSET, CHARSET_IGNORE), + (GROUPREF, GROUPREF_IGNORE), + (IN, IN_IGNORE), + (LITERAL, LITERAL_IGNORE), + (LITERAL_STRING, LITERAL_STRING_IGNORE), + (NOT_BIGCHARSET, NOT_BIGCHARSET_IGNORE), + (NOT_CHARSET, NOT_CHARSET_IGNORE), + (NOT_IN, NOT_IN_IGNORE), + (NOT_LITERAL, NOT_LITERAL_IGNORE), + (NOT_RANGE, NOT_RANGE_IGNORE), + (RANGE, RANGE_IGNORE), +] + +LINE_CODES = [ + (END_OF_LINE, END_OF_STRING_2), + (END_OF_STRING, END_OF_STRING), + (START_OF_LINE, START_OF_STRING), + (START_OF_STRING, START_OF_STRING), +] + +REPEAT_CODES = { + REPEAT_MAX: END_REPEAT_MAX, + REPEAT_MIN: END_REPEAT_MIN, + REPEAT_POSS: END_REPEAT_POSS, +} + +SINGLE_CHARACTER_CODES = set([ + ANY, + BIGCHARSET, + CHARSET, + DIGIT, + IN, + LITERAL, + NOT_BIGCHARSET, + NOT_CHARSET, + NOT_DIGIT, + NOT_IN, + NOT_LITERAL, + NOT_RANGE, + NOT_WHITESPACE, + NOT_WORD, + RANGE, + WHITESPACE, + WORD, +]) + +REVERSE_CODES = [ + (ANY, ANY_REV), + (ANY_ALL, ANY_ALL_REV), + (ASSERT, ASSERT), + (ASSERT_NOT, ASSERT_NOT), + (ATOMIC, ATOMIC), + (BIGCHARSET, BIGCHARSET_REV), + (BIGCHARSET_IGNORE, BIGCHARSET_IGNORE_REV), + (BOUNDARY, BOUNDARY), + (BRANCH, BRANCH), + (CALL, CALL), + (CHARSET, CHARSET_REV), + (CHARSET_IGNORE, CHARSET_IGNORE_REV), + (DIGIT, DIGIT_REV), + (END_ASSERT, END_ASSERT), + (END_ASSERT_NOT, END_ASSERT_NOT), + (END_ATOMIC, END_ATOMIC), + (END_OF_LINE, END_OF_LINE), + (END_OF_STRING, END_OF_STRING), + (END_OF_STRING_2, END_OF_STRING_2), + (END_REPEAT_MAX, END_REPEAT_MAX_REV), + (END_REPEAT_MIN, END_REPEAT_MIN_REV), + (END_REPEAT_ONE_MAX, END_REPEAT_ONE_MAX_REV), + (END_REPEAT_ONE_MIN, END_REPEAT_ONE_MIN_REV), + (END_REPEAT_ONE_POSS, END_REPEAT_ONE_POSS_REV), + (END_REPEAT_POSS, END_REPEAT_POSS_REV), + (FAILURE, FAILURE), + (GROUPREF, GROUPREF_REV), + (GROUPREF_EXISTS, GROUPREF_EXISTS), + (GROUPREF_IGNORE, GROUPREF_IGNORE_REV), + (IN, IN_REV), + (IN_IGNORE, IN_IGNORE_REV), + (JUMP, JUMP), + (LITERAL, LITERAL_REV), + (LITERAL_IGNORE, LITERAL_IGNORE_REV), + (LITERAL_STRING, LITERAL_STRING_REV), + (LITERAL_STRING_IGNORE, LITERAL_STRING_IGNORE_REV), + (LOC_BOUNDARY, LOC_BOUNDARY), + (LOC_NOT_BOUNDARY, LOC_NOT_BOUNDARY), + (LOC_NOT_WORD, LOC_NOT_WORD_REV), + (LOC_WORD, LOC_WORD_REV), + (MARK, MARK), + (NOT_BIGCHARSET, NOT_BIGCHARSET_REV), + (NOT_BIGCHARSET_IGNORE, NOT_BIGCHARSET_IGNORE_REV), + (NOT_BOUNDARY, NOT_BOUNDARY), + (NOT_CHARSET, NOT_CHARSET_REV), + (NOT_CHARSET_IGNORE, NOT_CHARSET_IGNORE_REV), + (NOT_DIGIT, NOT_DIGIT_REV), + (NOT_IN, NOT_IN_REV), + (NOT_IN_IGNORE, NOT_IN_IGNORE_REV), + (NOT_LITERAL, NOT_LITERAL_REV), + (NOT_LITERAL_IGNORE, NOT_LITERAL_IGNORE_REV), + (NOT_RANGE, NOT_RANGE_REV), + (NOT_RANGE_IGNORE, NOT_RANGE_IGNORE_REV), + (NOT_WHITESPACE, NOT_WHITESPACE_REV), + (NOT_WORD, NOT_WORD_REV), + (RANGE, RANGE_REV), + (RANGE_IGNORE, RANGE_IGNORE_REV), + (REPEAT_MAX, REPEAT_MAX_REV), + (REPEAT_MIN, REPEAT_MIN_REV), + (REPEAT_ONE_MAX, REPEAT_ONE_MAX_REV), + (REPEAT_ONE_MIN, REPEAT_ONE_MIN_REV), + (REPEAT_ONE_POSS, REPEAT_ONE_POSS_REV), + (REPEAT_POSS, REPEAT_POSS_REV), + (START_OF_LINE, START_OF_LINE), + (START_OF_STRING, START_OF_STRING), + (SUBPATTERN, SUBPATTERN), + (SUCCESS, SUCCESS), + (UNI_BOUNDARY, UNI_BOUNDARY), + (UNI_DIGIT, UNI_DIGIT_REV), + (UNI_NOT_BOUNDARY, UNI_NOT_BOUNDARY), + (UNI_NOT_DIGIT, UNI_NOT_DIGIT_REV), + (UNI_NOT_WHITESPACE, UNI_NOT_WHITESPACE_REV), + (UNI_NOT_WORD, UNI_NOT_WORD_REV), + (UNI_WHITESPACE, UNI_WHITESPACE_REV), + (UNI_WORD, UNI_WORD_REV), + (WHITESPACE, WHITESPACE_REV), + (WORD, WORD_REV), +] + +REPEAT_ONE_CODES = { + REPEAT_MAX: REPEAT_ONE_MAX, + REPEAT_MIN: REPEAT_ONE_MIN, + REPEAT_POSS: REPEAT_ONE_POSS, +} + +BIGCHARSET_SET = set([BIGCHARSET, NOT_BIGCHARSET]) +CATEGORY_SET = set([op for op, loc_op, uni_op in CATEGORY_CODES]) +CHARSET_SET = set([CHARSET, NOT_CHARSET]) +IN_SET = set([IN, NOT_IN]) +LINE_SET = set([END_OF_LINE, END_OF_STRING, START_OF_LINE, START_OF_STRING]) +LITERAL_SET = set([LITERAL, NOT_LITERAL]) +RANGE_SET = set([NOT_RANGE, RANGE]) +REPEAT_SET = set([REPEAT_MAX, REPEAT_MIN, REPEAT_POSS]) -def set(seq): - s = {} - for elem in seq: - s[elem] = 1 - return s - -_LITERAL_CODES = set([LITERAL, NOT_LITERAL]) -_REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT]) -_SUCCESS_CODES = set([SUCCESS, FAILURE]) -_ASSERT_CODES = set([ASSERT, ASSERT_NOT]) - -def _compile(code, pattern, flags): +def _compile(code, pattern, flags, info, dir=1): # internal: compile a (sub)pattern emit = code.append - _len = len - LITERAL_CODES = _LITERAL_CODES - REPEATING_CODES = _REPEATING_CODES - SUCCESS_CODES = _SUCCESS_CODES - ASSERT_CODES = _ASSERT_CODES + literal_string = [] + fix_op = {} + if flags & SRE_FLAG_LOCALE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = loc_op + elif flags & SRE_FLAG_UNICODE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = uni_op + else: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = op + if flags & SRE_FLAG_DOTALL: + fix_op[ANY] = ANY_ALL + else: + fix_op[ANY] = ANY + if flags & SRE_FLAG_IGNORECASE: + def fix_case(ch): + return _sre.getlower(ch, flags) + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op_ignore + else: + def fix_case(ch): + return ch + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op + if flags & SRE_FLAG_MULTILINE: + for line_op, op in LINE_CODES: + fix_op[line_op] = line_op + else: + for line_op, op in LINE_CODES: + fix_op[line_op] = op + if dir < 0: + fix_direction = dict((op, op_rev) for op, op_rev in REVERSE_CODES) + else: + fix_direction = dict((op, op) for op, op_rev in REVERSE_CODES) + if dir < 0: + # Within lookbehind, so reverse the order of the matching. + pattern = reversed(pattern) for op, av in pattern: - if op in LITERAL_CODES: - if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - emit(_sre.getlower(av, flags)) - else: + if op in IN_SET: + op, av = _optimize_in(op, av, flags) + if op is LITERAL: + literal_string.append(av) + else: + if literal_string: + if dir < 0: + literal_string = literal_string[::-1] + emit_literal_string(emit, literal_string, fix_case, fix_op, fix_direction) + literal_string = [] + if op in ASSERT_CODES: emit(OPCODES[op]) - emit(av) - elif op is IN: - if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - def fixup(literal, flags=flags): - return _sre.getlower(literal, flags) - else: + skip = len(code); emit(0) + _compile(code, av[1], flags, info, av[0]) + emit(OPCODES[ASSERT_CODES[op]]) + code[skip] = len(code) - skip + elif op in BIGCHARSET_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + _compile_bigcharset(av, flags, code) + elif op is BRANCH: emit(OPCODES[op]) - fixup = _identityfunction - skip = _len(code); emit(0) - _compile_charset(av, flags, code, fixup) - code[skip] = _len(code) - skip - elif op is ANY: - if flags & SRE_FLAG_DOTALL: - emit(OPCODES[ANY_ALL]) - else: - emit(OPCODES[ANY]) - elif op in REPEATING_CODES: - if flags & SRE_FLAG_TEMPLATE: - raise error, "internal: unsupported template operator" - emit(OPCODES[REPEAT]) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) + tail = [] + tailappend = tail.append + for av in av[1]: + skip = len(code); emit(0) + _compile(code, av, flags, info, dir) + emit(OPCODES[JUMP]) + tailappend(len(code)); emit(0) + code[skip] = len(code) - skip + emit(0) # end of branchs + for tail in tail: + code[tail] = len(code) - tail + elif op is CALL: + emit(OPCODES[op]) + skip = len(code); emit(0) + _compile(code, av, flags, info, dir) emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - elif _simple(av) and op is not REPEAT: - if op is MAX_REPEAT: - emit(OPCODES[REPEAT_ONE]) + code[skip] = len(code) - skip + elif op in CHARSET_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + _compile_charset(av, flags, code) + elif op in CATEGORY_SET or op in LINE_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + elif op is GROUPREF: + emit(OPCODES[fix_direction[fix_op[op]]]) + emit(av - 1) + elif op is GROUPREF_EXISTS: + emit(OPCODES[op]) + emit(av[0] - 1) + skipyes = len(code); emit(0) + _compile(code, av[1], flags, info, dir) + if av[2]: + emit(OPCODES[JUMP]) + skipno = len(code); emit(0) + code[skipyes] = len(code) - skipyes + 1 + _compile(code, av[2], flags, info, dir) + code[skipno] = len(code) - skipno else: - emit(OPCODES[MIN_REPEAT_ONE]) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) - emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - else: - emit(OPCODES[REPEAT]) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) - code[skip] = _len(code) - skip - if op is MAX_REPEAT: - emit(OPCODES[MAX_UNTIL]) + code[skipyes] = len(code) - skipyes + 1 + elif op in IN_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + _compile_in(av, flags, code, fix_direction) + elif op in LITERAL_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + emit(fix_case(av)) + elif op in RANGE_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + emit(fix_case(av[0])) + emit(fix_case(av[1])) + elif op in REPEAT_SET: + if flags & SRE_FLAG_TEMPLATE: + raise error, "internal: unsupported template operator" else: - emit(OPCODES[MIN_UNTIL]) - elif op is SUBPATTERN: - if av[0]: - emit(OPCODES[MARK]) - emit((av[0]-1)*2) - # _compile_info(code, av[1], flags) - _compile(code, av[1], flags) - if av[0]: - emit(OPCODES[MARK]) - emit((av[0]-1)*2+1) - elif op in SUCCESS_CODES: - emit(OPCODES[op]) - elif op in ASSERT_CODES: - emit(OPCODES[op]) - skip = _len(code); emit(0) - if av[0] >= 0: - emit(0) # look ahead - else: - lo, hi = av[1].getwidth() - if lo != hi: - raise error, "look-behind requires fixed-width pattern" - emit(lo) # look behind - _compile(code, av[1], flags) - emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - elif op is CALL: - emit(OPCODES[op]) - skip = _len(code); emit(0) - _compile(code, av, flags) - emit(OPCODES[SUCCESS]) - code[skip] = _len(code) - skip - elif op is AT: - emit(OPCODES[op]) - if flags & SRE_FLAG_MULTILINE: - av = AT_MULTILINE.get(av, av) - if flags & SRE_FLAG_LOCALE: - av = AT_LOCALE.get(av, av) - elif flags & SRE_FLAG_UNICODE: - av = AT_UNICODE.get(av, av) - emit(ATCODES[av]) - elif op is BRANCH: - emit(OPCODES[op]) - tail = [] - tailappend = tail.append - for av in av[1]: - skip = _len(code); emit(0) - # _compile_info(code, av, flags) - _compile(code, av, flags) - emit(OPCODES[JUMP]) - tailappend(_len(code)); emit(0) - code[skip] = _len(code) - skip - emit(0) # end of branch - for tail in tail: - code[tail] = _len(code) - tail - elif op is CATEGORY: - emit(OPCODES[op]) - if flags & SRE_FLAG_LOCALE: - av = CH_LOCALE[av] - elif flags & SRE_FLAG_UNICODE: - av = CH_UNICODE[av] - emit(CHCODES[av]) - elif op is GROUPREF: - if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - else: - emit(OPCODES[op]) - emit(av-1) - elif op is GROUPREF_EXISTS: - emit(OPCODES[op]) - emit(av[0]-1) - skipyes = _len(code); emit(0) - _compile(code, av[1], flags) - if av[2]: - emit(OPCODES[JUMP]) - skipno = _len(code); emit(0) - code[skipyes] = _len(code) - skipyes + 1 - _compile(code, av[2], flags) - code[skipno] = _len(code) - skipno + single = get_single_character(av[2]) + if single: + emit(OPCODES[fix_direction[REPEAT_ONE_CODES[op]]]) + skip = len(code); emit(0) + emit(info.repeat_count) + info.repeat_count += 1 + emit(av[0]) + emit(av[1]) + _compile(code, single, flags, info, dir) + code[skip] = len(code) - skip + else: + emit(OPCODES[fix_direction[op]]) + skip = len(code); emit(0) + emit(info.repeat_count) + info.repeat_count += 1 + emit(av[0]) + emit(av[1]) + _compile(code, av[2], flags, info, dir) + emit(OPCODES[fix_direction[REPEAT_CODES[op]]]) + offset = len(code) - skip + code[skip] = offset + emit(offset) + elif op is SUBPATTERN: + if av[0]: + group_index = info.group_count + info.group_count += 1 + emit(OPCODES[MARK]) + emit(group_index * 2) + _compile(code, av[1], flags, info, dir) + if av[0]: + emit(OPCODES[MARK]) + emit(group_index * 2 + 1) + elif op is ATOMIC: + emit(OPCODES[ATOMIC]) + skip = len(code); emit(0) + _compile(code, av[1], flags, info, dir) + emit(OPCODES[END_ATOMIC]) + code[skip] = len(code) - skip else: - code[skipyes] = _len(code) - skipyes + 1 - else: - raise ValueError, ("unsupported operand type", op) + raise ValueError, ("unsupported operand type", op) + if literal_string: + if dir < 0: + literal_string = literal_string[::-1] + emit_literal_string(emit, literal_string, fix_case, fix_op, fix_direction) + +def emit_literal_string(emit, literal_string, fix_case, fix_op, fix_direction): + if len(literal_string) > 1: + emit(OPCODES[fix_direction[fix_op[LITERAL_STRING]]]) + emit(len(literal_string)) + for ch in literal_string: + emit(fix_case(ch)) + else: + emit(OPCODES[fix_direction[fix_op[LITERAL]]]) + emit(fix_case(literal_string[0])) + +def get_single_character(pattern): + if len(pattern) == 1 and pattern[0][0] in SINGLE_CHARACTER_CODES: + return pattern + return None -def _compile_charset(charset, flags, code, fixup=None): +def _compile_in(charset, flags, code, fix_direction): # compile charset subprogram emit = code.append - if fixup is None: - fixup = _identityfunction - for op, av in _optimize_charset(charset, fixup): - emit(OPCODES[op]) - if op is NEGATE: - pass + fix_op = {} + if flags & SRE_FLAG_LOCALE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = loc_op + elif flags & SRE_FLAG_UNICODE: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = uni_op + else: + for op, loc_op, uni_op in CATEGORY_CODES: + fix_op[op] = op + if flags & SRE_FLAG_IGNORECASE: + def fix_case(ch): + return _sre.getlower(ch, flags) + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op_ignore + else: + def fix_case(ch): + return ch + for op, op_ignore in IGNORE_CODES: + fix_op[op] = op + skip = len(code); emit(0) + for op, av in charset: + if op is BIGCHARSET: + emit(OPCODES[fix_direction[op]]) + _compile_bigcharset(av, flags, code) + elif op in CATEGORY_SET: + emit(OPCODES[fix_direction[fix_op[op]]]) + elif op is CHARSET: + emit(OPCODES[fix_direction[op]]) + _compile_charset(av, flags, code) elif op is LITERAL: - emit(fixup(av)) + emit(OPCODES[fix_direction[op]]) + emit(fix_case(av)) elif op is RANGE: - emit(fixup(av[0])) - emit(fixup(av[1])) - elif op is CHARSET: - code.extend(av) - elif op is BIGCHARSET: - code.extend(av) - elif op is CATEGORY: - if flags & SRE_FLAG_LOCALE: - emit(CHCODES[CH_LOCALE[av]]) - elif flags & SRE_FLAG_UNICODE: - emit(CHCODES[CH_UNICODE[av]]) - else: - emit(CHCODES[av]) + emit(OPCODES[fix_direction[op]]) + emit(fix_case(av[0])) + emit(fix_case(av[1])) else: - raise error, "internal: unsupported set operator" - emit(OPCODES[FAILURE]) + raise error, "internal: unsupported charset operator" + code[skip] = len(code) - skip -def _optimize_charset(charset, fixup): - # internal: optimize character set - out = [] - outappend = out.append - charmap = [0]*256 - try: - for op, av in charset: - if op is NEGATE: - outappend((op, av)) - elif op is LITERAL: - charmap[fixup(av)] = 1 - elif op is RANGE: - for i in range(fixup(av[0]), fixup(av[1])+1): - charmap[i] = 1 - elif op is CATEGORY: - # XXX: could append to charmap tail - return charset # cannot compress - except IndexError: - # character set contains unicode characters - return _optimize_unicode(charset, fixup) - # compress character map - i = p = n = 0 - runs = [] - runsappend = runs.append - for c in charmap: - if c: - if n == 0: - p = i - n = n + 1 - elif n: - runsappend((p, n)) - n = 0 - i = i + 1 - if n: - runsappend((p, n)) - if len(runs) <= 2: - # use literal/range - for p, n in runs: - if n == 1: - outappend((LITERAL, p)) - else: - outappend((RANGE, (p, p+n-1))) - if len(out) < len(charset): - return out - else: - # use bitmap - data = _mk_bitmap(charmap) - outappend((CHARSET, data)) - return out - return charset - -def _mk_bitmap(bits): - data = [] - dataappend = data.append - if _sre.CODESIZE == 2: - start = (1, 0) - else: - start = (1L, 0L) - m, v = start - for c in bits: - if c: - v = v + m - m = m + m - if m > MAXCODE: - dataappend(v) - m, v = start - return data +def _compile_charset(av, flags, code): + emit = code.append + for i in range(256 / _subchunk_size): + emit(av & _subchunk_mask) + av >>= _subchunk_size + +def _compile_bigcharset(av, flags, code): + emit = code.append + skip = len(code); emit(0) + bits, shift = 0, 0 + for index in av[0]: + bits |= index << shift; shift += 8 + if shift == _subchunk_size: + emit(bits) + bits, shift = 0, 0 + for chunk in av[1]: + for i in range(256 / _subchunk_size): + emit(chunk & _subchunk_mask) + chunk >>= _subchunk_size + code[skip] = len(code) - skip + +# The characters may be mapped to a bitmap. # To represent a big charset, first a bitmap of all characters in the # set is constructed. Then, this bitmap is sliced into chunks of 256 @@ -304,171 +452,94 @@ # in particular, that negated charsets cannot be represented as # bigcharsets. -def _optimize_unicode(charset, fixup): - try: - import array - except ImportError: - return charset - charmap = [0]*65536 - negate = 0 - try: - for op, av in charset: - if op is NEGATE: - negate = 1 - elif op is LITERAL: - charmap[fixup(av)] = 1 - elif op is RANGE: - for i in xrange(fixup(av[0]), fixup(av[1])+1): - charmap[i] = 1 - elif op is CATEGORY: - # XXX: could expand category - return charset # cannot compress - except IndexError: - # non-BMP characters - return charset - if negate: - if sys.maxunicode != 65535: - # XXX: negation does not work with big charsets - return charset - for i in xrange(65536): - charmap[i] = not charmap[i] - comps = {} - mapping = [0]*256 - block = 0 - data = [] - for i in xrange(256): - chunk = tuple(charmap[i*256:(i+1)*256]) - new = comps.setdefault(chunk, block) - mapping[i] = new - if new == block: - block = block + 1 - data = data + _mk_bitmap(chunk) - header = [block] - if _sre.CODESIZE == 2: - code = 'H' - else: - code = 'I' - # Convert block indices to byte array of 256 bytes - mapping = array.array('b', mapping).tostring() - # Convert byte array to word array - mapping = array.array(code, mapping) - assert mapping.itemsize == _sre.CODESIZE - header = header + mapping.tolist() - data[0:0] = header - return [(BIGCHARSET, data)] - -def _simple(av): - # check if av is a "simple" operator - lo, hi = av[2].getwidth() - if lo == 0 and hi == MAXREPEAT: - raise error, "nothing to repeat" - return lo == hi == 1 and av[2][0][0] != SUBPATTERN - -def _compile_info(code, pattern, flags): - # internal: compile an info block. in the current version, - # this contains min/max pattern width, and an optional literal - # prefix or a character map - lo, hi = pattern.getwidth() - if lo == 0: - return # not worth it - # look for a literal prefix - prefix = [] - prefixappend = prefix.append - prefix_skip = 0 - charset = [] # not used - charsetappend = charset.append - if not (flags & SRE_FLAG_IGNORECASE): - # look for literal prefix - for op, av in pattern.data: - if op is LITERAL: - if len(prefix) == prefix_skip: - prefix_skip = prefix_skip + 1 - prefixappend(av) - elif op is SUBPATTERN and len(av[1]) == 1: - op, av = av[1][0] - if op is LITERAL: - prefixappend(av) - else: - break - else: - break - # if no prefix, look for charset prefix - if not prefix and pattern.data: - op, av = pattern.data[0] - if op is SUBPATTERN and av[1]: - op, av = av[1][0] - if op is LITERAL: - charsetappend((op, av)) - elif op is BRANCH: - c = [] - cappend = c.append - for p in av[1]: - if not p: - break - op, av = p[0] - if op is LITERAL: - cappend((op, av)) - else: - break - else: - charset = c - elif op is BRANCH: - c = [] - cappend = c.append - for p in av[1]: - if not p: - break - op, av = p[0] - if op is LITERAL: - cappend((op, av)) - else: - break - else: - charset = c - elif op is IN: - charset = av -## if prefix: -## print "*** PREFIX", prefix, prefix_skip -## if charset: -## print "*** CHARSET", charset - # add an info block - emit = code.append - emit(OPCODES[INFO]) - skip = len(code); emit(0) - # literal flag - mask = 0 - if prefix: - mask = SRE_INFO_PREFIX - if len(prefix) == prefix_skip == len(pattern.data): - mask = mask + SRE_INFO_LITERAL - elif charset: - mask = mask + SRE_INFO_CHARSET - emit(mask) - # pattern length - if lo < MAXCODE: - emit(lo) +def _ones(n): + return (1 << n) - 1 + +NOT_CODES = [ + (BIGCHARSET, NOT_BIGCHARSET), + (CHARSET, NOT_CHARSET), + (DIGIT, NOT_DIGIT), + (LITERAL, NOT_LITERAL), + (RANGE, NOT_RANGE), + (WHITESPACE, NOT_WHITESPACE), + (WORD, NOT_WORD), +] + +def _optimize_in(op, charset, flags): + # internal: optimize character set + if flags & SRE_FLAG_IGNORECASE: + def fix_case(ch): + return _sre.getlower(ch, flags) else: - emit(MAXCODE) - prefix = prefix[:MAXCODE] - if hi < MAXCODE: - emit(hi) + def fix_case(ch): + return ch + fix_op = {} + if op is IN: + for o, not_o in NOT_CODES: + fix_op[o] = o + fix_op[not_o] = not_o else: - emit(0) - # add literal prefix - if prefix: - emit(len(prefix)) # length - emit(prefix_skip) # skip - code.extend(prefix) - # generate overlap table - table = [-1] + ([0]*len(prefix)) - for i in xrange(len(prefix)): - table[i+1] = table[i]+1 - while table[i+1] > 0 and prefix[i] != prefix[table[i+1]-1]: - table[i+1] = table[table[i+1]-1]+1 - code.extend(table[1:]) # don't store first entry - elif charset: - _compile_charset(charset, flags, code) - code[skip] = len(code) - skip + for o, not_o in NOT_CODES: + fix_op[o] = not_o + fix_op[not_o] = o + # consolidate the ranges + ranges = [] + categories = [] + for o, a in charset: + if o is LITERAL: + c = fix_case(a) + ranges.append((c, c)) + elif o is RANGE: + r = iter(xrange(a[0], a[1] + 1)) + start = fix_case(r.next()) + end = start + for c in r: + c = fix_case(c) + if c - end != 1: + ranges.append((start, end)) + start = c + end = c + ranges.append((start, end)) + else: + categories.append((o, a)) + ranges.sort() + for i in range(len(ranges) - 2, -1, -1): + if ranges[i + 1][0] - ranges[i][1] <= 1: + ranges[i : i + 2] = [(ranges[i][0], max(ranges[i][1], ranges[i + 1][1]))] + # try to optimise + if len(ranges) == 1: + for r in ranges: + if r[0] == r[1]: + categories.append((LITERAL, r[0])) + else: + categories.append((RANGE, r)) + elif len(ranges) > 1: + if ranges[-1][1] <= 255: + # small (bytestring) charset + bitmap = 0 + for r in ranges: + bitmap |= _ones(r[1] + 1) ^ _ones(r[0]) + categories.append((CHARSET, bitmap)) + else: + # big (unicode) charset + bitmap = 0 + for r in ranges: + bitmap |= _ones(r[1] + 1) ^ _ones(r[0]) + chunk_mask = _ones(256) + index_table = [] + reverse_index = {} + for high_byte in range(256): + chunk = bitmap & chunk_mask + index = reverse_index.setdefault(chunk, len(reverse_index)) + index_table.append(index) + bitmap >>= 256 + chunks = [(index, chunk) for chunk, index in reverse_index.items()] + chunks.sort() + categories.append((BIGCHARSET, (index_table, [chunk for index, chunk in chunks]))) + if len(categories) == 1: + o, a = categories[0] + return (fix_op[o], a) + return (op, categories) try: unicode @@ -484,17 +555,23 @@ return 0 def _code(p, flags): - flags = p.pattern.flags | flags code = [] # compile info block - _compile_info(code, p, flags) + #_compile_info(code, p, flags) # compile the pattern - _compile(code, p.data, flags) + class Record(object): + pass + info = Record() + info.group_count = 0 + info.repeat_count = 0 + _compile(code, p.data, flags, info) code.append(OPCODES[SUCCESS]) + + code = [info.group_count * 2, info.repeat_count, len(code)] + code return code Index: Lib/sre_constants.py =================================================================== --- sre_constants.py Wed Aug 25 03:22:30 2004 +++ sre_constants.py Mon Sep 08 16:30:22 2008 @@ -17,7 +17,7 @@ # max code word in this release -MAXREPEAT = 65535 +MAX_REPEATS = 65535 # SRE standard exception (access as sre.error) # should this really be here? @@ -25,123 +25,270 @@ class error(Exception): pass -# operators - +# codes FAILURE = "failure" SUCCESS = "success" ANY = "any" ANY_ALL = "any_all" +ANY_ALL_REV = "any_all_rev" +ANY_REV = "any_rev" ASSERT = "assert" ASSERT_NOT = "assert_not" -AT = "at" +ATOMIC = "atomic" BIGCHARSET = "bigcharset" +BIGCHARSET_IGNORE = "bigcharset_ignore" +BIGCHARSET_IGNORE_REV = "bigcharset_ignore_rev" +BIGCHARSET_REV = "bigcharset_rev" +BOUNDARY = "boundary" BRANCH = "branch" CALL = "call" -CATEGORY = "category" CHARSET = "charset" +CHARSET_IGNORE = "charset_ignore" +CHARSET_IGNORE_REV = "charset_ignore_rev" +CHARSET_REV = "charset_rev" +DIGIT = "digit" +DIGIT_REV = "digit_rev" +END_ASSERT = "end_assert" +END_ASSERT_NOT = "end_assert_not" +END_ATOMIC = "end_atomic" +END_OF_LINE = "end_of_line" +END_OF_STRING = "end_of_string" +END_OF_STRING_2 = "end_of_string_2" +END_REPEAT_MAX = "end_repeat_max" +END_REPEAT_MAX_REV = "end_repeat_max_rev" +END_REPEAT_MIN = "end_repeat_min" +END_REPEAT_MIN_REV = "end_repeat_min_rev" +END_REPEAT_ONE_MAX = "end_repeat_one_max" +END_REPEAT_ONE_MAX_REV = "end_repeat_one_max_rev" +END_REPEAT_ONE_MIN = "end_repeat_one_min" +END_REPEAT_ONE_MIN_REV = "end_repeat_one_min_rev" +END_REPEAT_ONE_POSS = "end_repeat_one_poss" +END_REPEAT_ONE_POSS_REV = "end_repeat_one_poss_rev" +END_REPEAT_POSS = "end_repeat_poss" +END_REPEAT_POSS_REV = "end_repeat_poss_rev" GROUPREF = "groupref" -GROUPREF_IGNORE = "groupref_ignore" GROUPREF_EXISTS = "groupref_exists" +GROUPREF_IGNORE = "groupref_ignore" +GROUPREF_IGNORE_REV = "groupref_ignore_rev" +GROUPREF_REV = "groupref_rev" IN = "in" IN_IGNORE = "in_ignore" -INFO = "info" +IN_IGNORE_REV = "in_ignore_rev" +IN_REV = "in_rev" JUMP = "jump" LITERAL = "literal" LITERAL_IGNORE = "literal_ignore" +LITERAL_IGNORE_REV = "literal_ignore_rev" +LITERAL_REV = "literal_rev" +LITERAL_STRING = "literal_string" +LITERAL_STRING_IGNORE = "literal_string_ignore" +LITERAL_STRING_IGNORE_REV = "literal_string_ignore_rev" +LITERAL_STRING_REV = "literal_string_rev" +LOC_BOUNDARY = "loc_boundary" +LOC_NOT_BOUNDARY = "loc_not_boundary" +LOC_NOT_WORD = "loc_not_word" +LOC_NOT_WORD_REV = "loc_not_word_rev" +LOC_WORD = "loc_word" +LOC_WORD_REV = "loc_word_rev" MARK = "mark" -MAX_REPEAT = "max_repeat" -MAX_UNTIL = "max_until" -MIN_REPEAT = "min_repeat" -MIN_UNTIL = "min_until" -NEGATE = "negate" +NOT_BIGCHARSET = "not_bigcharset" +NOT_BIGCHARSET_IGNORE = "not_bigcharset_ignore" +NOT_BIGCHARSET_IGNORE_REV = "not_bigcharset_ignore_rev" +NOT_BIGCHARSET_REV = "not_bigcharset_rev" +NOT_BOUNDARY = "not_boundary" +NOT_CHARSET = "not_charset" +NOT_CHARSET_IGNORE = "not_charset_ignore" +NOT_CHARSET_IGNORE_REV = "not_charset_ignore_rev" +NOT_CHARSET_REV = "not_charset_rev" +NOT_DIGIT = "not_digit" +NOT_DIGIT_REV = "not_digit_rev" +NOT_IN = "not_in" +NOT_IN_IGNORE = "not_in_ignore" +NOT_IN_IGNORE_REV = "not_in_ignore_rev" +NOT_IN_REV = "not_in_rev" NOT_LITERAL = "not_literal" NOT_LITERAL_IGNORE = "not_literal_ignore" +NOT_LITERAL_IGNORE_REV = "not_literal_ignore_rev" +NOT_LITERAL_REV = "not_literal_rev" +NOT_RANGE = "not_range" +NOT_RANGE_IGNORE = "not_range_ignore" +NOT_RANGE_IGNORE_REV = "not_range_ignore_rev" +NOT_RANGE_REV = "not_range_rev" +NOT_WHITESPACE = "not_whitespace" +NOT_WHITESPACE_REV = "not_whitespace_rev" +NOT_WORD = "not_word" +NOT_WORD_REV = "not_word_rev" RANGE = "range" -REPEAT = "repeat" -REPEAT_ONE = "repeat_one" +RANGE_IGNORE = "range_ignore" +RANGE_IGNORE_REV = "range_ignore_rev" +RANGE_REV = "range_rev" +REPEAT_MAX = "repeat_max" +REPEAT_MAX_REV = "repeat_max_rev" +REPEAT_MIN = "repeat_min" +REPEAT_MIN_REV = "repeat_min_rev" +REPEAT_ONE_MAX = "repeat_one_max" +REPEAT_ONE_MAX_REV = "repeat_one_max_rev" +REPEAT_ONE_MIN = "repeat_one_min" +REPEAT_ONE_MIN_REV = "repeat_one_min_rev" +REPEAT_ONE_POSS = "repeat_one_poss" +REPEAT_ONE_POSS_REV = "repeat_one_poss_rev" +REPEAT_POSS = "repeat_poss" +REPEAT_POSS_REV = "repeat_poss_rev" +START_OF_LINE = "start_of_line" +START_OF_STRING = "start_of_string" SUBPATTERN = "subpattern" -MIN_REPEAT_ONE = "min_repeat_one" - -# positions -AT_BEGINNING = "at_beginning" -AT_BEGINNING_LINE = "at_beginning_line" -AT_BEGINNING_STRING = "at_beginning_string" -AT_BOUNDARY = "at_boundary" -AT_NON_BOUNDARY = "at_non_boundary" -AT_END = "at_end" -AT_END_LINE = "at_end_line" -AT_END_STRING = "at_end_string" -AT_LOC_BOUNDARY = "at_loc_boundary" -AT_LOC_NON_BOUNDARY = "at_loc_non_boundary" -AT_UNI_BOUNDARY = "at_uni_boundary" -AT_UNI_NON_BOUNDARY = "at_uni_non_boundary" - -# categories -CATEGORY_DIGIT = "category_digit" -CATEGORY_NOT_DIGIT = "category_not_digit" -CATEGORY_SPACE = "category_space" -CATEGORY_NOT_SPACE = "category_not_space" -CATEGORY_WORD = "category_word" -CATEGORY_NOT_WORD = "category_not_word" -CATEGORY_LINEBREAK = "category_linebreak" -CATEGORY_NOT_LINEBREAK = "category_not_linebreak" -CATEGORY_LOC_WORD = "category_loc_word" -CATEGORY_LOC_NOT_WORD = "category_loc_not_word" -CATEGORY_UNI_DIGIT = "category_uni_digit" -CATEGORY_UNI_NOT_DIGIT = "category_uni_not_digit" -CATEGORY_UNI_SPACE = "category_uni_space" -CATEGORY_UNI_NOT_SPACE = "category_uni_not_space" -CATEGORY_UNI_WORD = "category_uni_word" -CATEGORY_UNI_NOT_WORD = "category_uni_not_word" -CATEGORY_UNI_LINEBREAK = "category_uni_linebreak" -CATEGORY_UNI_NOT_LINEBREAK = "category_uni_not_linebreak" +UNI_BOUNDARY = "uni_boundary" +UNI_DIGIT = "uni_digit" +UNI_DIGIT_REV = "uni_digit_rev" +UNI_NOT_BOUNDARY = "uni_not_boundary" +UNI_NOT_DIGIT = "uni_not_digit" +UNI_NOT_DIGIT_REV = "uni_not_digit_rev" +UNI_NOT_WHITESPACE = "uni_not_whitespace" +UNI_NOT_WHITESPACE_REV = "uni_not_whitespace_rev" +UNI_NOT_WORD = "uni_not_word" +UNI_NOT_WORD_REV = "uni_not_word_rev" +UNI_WHITESPACE = "uni_whitespace" +UNI_WHITESPACE_REV = "uni_whitespace_rev" +UNI_WORD = "uni_word" +UNI_WORD_REV = "uni_word_rev" +WHITESPACE = "whitespace" +WHITESPACE_REV = "whitespace_rev" +WORD = "word" +WORD_REV = "word_rev" OPCODES = [ - # failure=0 success=1 (just because it looks better that way :-) - FAILURE, SUCCESS, + FAILURE, + SUCCESS, - ANY, ANY_ALL, - ASSERT, ASSERT_NOT, - AT, + ANY, + ANY_ALL, + ANY_ALL_REV, + ANY_REV, + ASSERT, + ASSERT_NOT, + ATOMIC, + BIGCHARSET, + BIGCHARSET_IGNORE, + BIGCHARSET_IGNORE_REV, + BIGCHARSET_REV, + BOUNDARY, BRANCH, CALL, - CATEGORY, - CHARSET, BIGCHARSET, - GROUPREF, GROUPREF_EXISTS, GROUPREF_IGNORE, - IN, IN_IGNORE, - INFO, + CHARSET, + CHARSET_IGNORE, + CHARSET_IGNORE_REV, + CHARSET_REV, + DIGIT, + DIGIT_REV, + END_ASSERT, + END_ASSERT_NOT, + END_ATOMIC, + END_OF_LINE, + END_OF_STRING, + END_OF_STRING_2, + END_REPEAT_MAX, + END_REPEAT_MAX_REV, + END_REPEAT_MIN, + END_REPEAT_MIN_REV, + END_REPEAT_ONE_MAX, + END_REPEAT_ONE_MAX_REV, + END_REPEAT_ONE_MIN, + END_REPEAT_ONE_MIN_REV, + END_REPEAT_ONE_POSS, + END_REPEAT_ONE_POSS_REV, + END_REPEAT_POSS, + END_REPEAT_POSS_REV, + GROUPREF, + GROUPREF_EXISTS, + GROUPREF_IGNORE, + GROUPREF_IGNORE_REV, + GROUPREF_REV, + IN, + IN_IGNORE, + IN_IGNORE_REV, + IN_REV, JUMP, - LITERAL, LITERAL_IGNORE, + LITERAL, + LITERAL_IGNORE, + LITERAL_IGNORE_REV, + LITERAL_REV, + LITERAL_STRING, + LITERAL_STRING_IGNORE, + LITERAL_STRING_IGNORE_REV, + LITERAL_STRING_REV, + LOC_BOUNDARY, + LOC_NOT_BOUNDARY, + LOC_NOT_WORD, + LOC_NOT_WORD_REV, + LOC_WORD, + LOC_WORD_REV, MARK, - MAX_UNTIL, - MIN_UNTIL, - NOT_LITERAL, NOT_LITERAL_IGNORE, - NEGATE, + NOT_BIGCHARSET, + NOT_BIGCHARSET_IGNORE, + NOT_BIGCHARSET_IGNORE_REV, + NOT_BIGCHARSET_REV, + NOT_BOUNDARY, + NOT_CHARSET, + NOT_CHARSET_IGNORE, + NOT_CHARSET_IGNORE_REV, + NOT_CHARSET_REV, + NOT_DIGIT, + NOT_DIGIT_REV, + NOT_IN, + NOT_IN_IGNORE, + NOT_IN_IGNORE_REV, + NOT_IN_REV, + NOT_LITERAL, + NOT_LITERAL_IGNORE, + NOT_LITERAL_IGNORE_REV, + NOT_LITERAL_REV, + NOT_RANGE, + NOT_RANGE_IGNORE, + NOT_RANGE_IGNORE_REV, + NOT_RANGE_REV, + NOT_WHITESPACE, + NOT_WHITESPACE_REV, + NOT_WORD, + NOT_WORD_REV, RANGE, - REPEAT, - REPEAT_ONE, + RANGE_IGNORE, + RANGE_IGNORE_REV, + RANGE_REV, + REPEAT_MAX, + REPEAT_MAX_REV, + REPEAT_MIN, + REPEAT_MIN_REV, + REPEAT_ONE_MAX, + REPEAT_ONE_MAX_REV, + REPEAT_ONE_MIN, + REPEAT_ONE_MIN_REV, + REPEAT_ONE_POSS, + REPEAT_ONE_POSS_REV, + REPEAT_POSS, + REPEAT_POSS_REV, + START_OF_LINE, + START_OF_STRING, SUBPATTERN, - MIN_REPEAT_ONE - -] - -ATCODES = [ - AT_BEGINNING, AT_BEGINNING_LINE, AT_BEGINNING_STRING, AT_BOUNDARY, - AT_NON_BOUNDARY, AT_END, AT_END_LINE, AT_END_STRING, - AT_LOC_BOUNDARY, AT_LOC_NON_BOUNDARY, AT_UNI_BOUNDARY, - AT_UNI_NON_BOUNDARY -] - -CHCODES = [ - CATEGORY_DIGIT, CATEGORY_NOT_DIGIT, CATEGORY_SPACE, - CATEGORY_NOT_SPACE, CATEGORY_WORD, CATEGORY_NOT_WORD, - CATEGORY_LINEBREAK, CATEGORY_NOT_LINEBREAK, CATEGORY_LOC_WORD, - CATEGORY_LOC_NOT_WORD, CATEGORY_UNI_DIGIT, CATEGORY_UNI_NOT_DIGIT, - CATEGORY_UNI_SPACE, CATEGORY_UNI_NOT_SPACE, CATEGORY_UNI_WORD, - CATEGORY_UNI_NOT_WORD, CATEGORY_UNI_LINEBREAK, - CATEGORY_UNI_NOT_LINEBREAK + UNI_BOUNDARY, + UNI_DIGIT, + UNI_DIGIT_REV, + UNI_NOT_BOUNDARY, + UNI_NOT_DIGIT, + UNI_NOT_DIGIT_REV, + UNI_NOT_WHITESPACE, + UNI_NOT_WHITESPACE_REV, + UNI_NOT_WORD, + UNI_NOT_WORD_REV, + UNI_WHITESPACE, + UNI_WHITESPACE_REV, + UNI_WORD, + UNI_WORD_REV, + WHITESPACE, + WHITESPACE_REV, + WORD, + WORD_REV, ] def makedict(list): @@ -153,53 +300,6 @@ return d OPCODES = makedict(OPCODES) -ATCODES = makedict(ATCODES) -CHCODES = makedict(CHCODES) - -# replacement operations for "ignore case" mode -OP_IGNORE = { - GROUPREF: GROUPREF_IGNORE, - IN: IN_IGNORE, - LITERAL: LITERAL_IGNORE, - NOT_LITERAL: NOT_LITERAL_IGNORE -} - -AT_MULTILINE = { - AT_BEGINNING: AT_BEGINNING_LINE, - AT_END: AT_END_LINE -} - -AT_LOCALE = { - AT_BOUNDARY: AT_LOC_BOUNDARY, - AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY -} - -AT_UNICODE = { - AT_BOUNDARY: AT_UNI_BOUNDARY, - AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY -} - -CH_LOCALE = { - CATEGORY_DIGIT: CATEGORY_DIGIT, - CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT, - CATEGORY_SPACE: CATEGORY_SPACE, - CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE, - CATEGORY_WORD: CATEGORY_LOC_WORD, - CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD, - CATEGORY_LINEBREAK: CATEGORY_LINEBREAK, - CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK -} - -CH_UNICODE = { - CATEGORY_DIGIT: CATEGORY_UNI_DIGIT, - CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT, - CATEGORY_SPACE: CATEGORY_UNI_SPACE, - CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE, - CATEGORY_WORD: CATEGORY_UNI_WORD, - CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD, - CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK, - CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK -} # flags SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking) @@ -242,8 +342,6 @@ f.write("#define SRE_MAGIC %d\n" % MAGIC) dump(f, OPCODES, "SRE_OP") - dump(f, ATCODES, "SRE") - dump(f, CHCODES, "SRE") f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE) f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE) Index: Lib/sre_parse.py =================================================================== --- sre_parse.py Wed Sep 14 09:54:39 2005 +++ sre_parse.py Mon Sep 08 16:31:06 2008 @@ -16,12 +16,6 @@ from sre_constants import * -def set(seq): - s = {} - for elem in seq: - s[elem] = 1 - return s - SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -30,7 +24,7 @@ OCTDIGITS = set("01234567") HEXDIGITS = set("0123456789abcdefABCDEF") -WHITESPACE = set(" \t\n\r\v\f") +WHITESPACE_CHARS = set(" \t\n\r\v\f") ESCAPES = { r"\a": (LITERAL, ord("\a")), @@ -43,17 +37,20 @@ r"\\": (LITERAL, ord("\\")) } -CATEGORIES = { - r"\A": (AT, AT_BEGINNING_STRING), # start of string - r"\b": (AT, AT_BOUNDARY), - r"\B": (AT, AT_NON_BOUNDARY), - r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), - r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), - r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), - r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), - r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), - r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), - r"\Z": (AT, AT_END_STRING), # end of string +POSITIONS = { + r"\A": (START_OF_STRING, None), + r"\b": (BOUNDARY, None), + r"\B": (NOT_BOUNDARY, None), + r"\Z": (END_OF_STRING, None) +} + +CHARSETS = { + r"\d": (DIGIT, None), + r"\D": (NOT_DIGIT, None), + r"\s": (WHITESPACE, None), + r"\S": (NOT_WHITESPACE, None), + r"\w": (WORD, None), + r"\W": (NOT_WORD, None) } FLAGS = { @@ -91,6 +88,12 @@ def checkgroup(self, gid): return gid < self.groups and gid not in self.open +UNIT_CODES = set([ANY, DIGIT, IN, LITERAL, NOT_DIGIT, + NOT_IN, NOT_LITERAL, NOT_WHITESPACE, NOT_WORD, RANGE, + WHITESPACE, WORD]) + +REPEAT_CODES = set([REPEAT_MIN, REPEAT_MAX, REPEAT_POSS]) + class SubPattern: # a subpattern, in intermediate form def __init__(self, pattern, data=None): @@ -104,12 +107,17 @@ seqtypes = type(()), type([]) for op, av in self.data: print level*" " + op,; nl = 0 - if op == "in": + if op == IN: + # member sublanguage + print; nl = 1 + for op, a in av: + print (level+1)*" " + op, a + elif op == NOT_IN: # member sublanguage print; nl = 1 for op, a in av: print (level+1)*" " + op, a - elif op == "branch": + elif op == BRANCH: print; nl = 1 i = 0 for a in av[1]: @@ -148,8 +156,6 @@ if self.width: return self.width lo = hi = 0L - UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY) - REPEATCODES = (MIN_REPEAT, MAX_REPEAT) for op, av in self.data: if op is BRANCH: i = sys.maxint @@ -164,19 +170,17 @@ i, j = av.getwidth() lo = lo + i hi = hi + j - elif op is SUBPATTERN: + elif op in [SUBPATTERN, ATOMIC]: i, j = av[1].getwidth() lo = lo + i hi = hi + j - elif op in REPEATCODES: + elif op in REPEAT_CODES: i, j = av[2].getwidth() lo = lo + long(i) * av[0] hi = hi + long(j) * av[1] - elif op in UNITCODES: + elif op in UNIT_CODES: lo = lo + 1 hi = hi + 1 - elif op == SUCCESS: - break self.width = int(min(lo, sys.maxint)), int(min(hi, sys.maxint)) return self.width @@ -230,10 +234,10 @@ def _class_escape(source, escape): # handle escape code inside character class - code = ESCAPES.get(escape) + code = CHARSETS.get(escape) if code: return code - code = CATEGORIES.get(escape) + code = ESCAPES.get(escape) if code: return code try: @@ -262,7 +266,10 @@ def _escape(source, escape, state): # handle escape code in expression - code = CATEGORIES.get(escape) + code = POSITIONS.get(escape) + if code: + return code + code = CHARSETS.get(escape) if code: return code code = ESCAPES.get(escape) @@ -325,40 +332,6 @@ return items[0] subpattern = SubPattern(state) - subpatternappend = subpattern.append - - # check if all items share a common prefix - while 1: - prefix = None - for item in items: - if not item: - break - if prefix is None: - prefix = item[0] - elif item[0] != prefix: - break - else: - # all subitems start with a common "prefix". - # move it out of the branch - for item in items: - del item[0] - subpatternappend(prefix) - continue # check next one - break - - # check if the branch can be replaced by a character set - for item in items: - if len(item) != 1 or item[0][0] != LITERAL: - break - else: - # we can store this as a character set instead of a - # branch (the compiler may optimize this even more) - set = [] - setappend = set.append - for item in items: - setappend(item[0]) - subpatternappend((IN, set)) - return subpattern subpattern.append((BRANCH, (None, items))) return subpattern @@ -377,10 +350,15 @@ subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern -_PATTERNENDERS = set("|)") -_ASSERTCHARS = set("=!<") -_LOOKBEHINDASSERTCHARS = set("=!") -_REPEATCODES = set([MIN_REPEAT, MAX_REPEAT]) +PATTERN_ENDERS = set("|)") +ASSERT_CHARS = set("=!<") +LOOKBEHIND_ASSERT_CHARS = set("=!") +POSITION_CODES = set([BOUNDARY, END_OF_LINE, END_OF_STRING, NOT_BOUNDARY, START_OF_LINE, START_OF_STRING]) +REPEAT_CODES = set([REPEAT_MIN, REPEAT_MAX, REPEAT_POSS]) +QUERY_GROUP = 0 +CAPTURE_GROUP = 1 +NONCAPTURE_GROUP = 2 +ATOMIC_GROUP = 3 def _parse(source, state): # parse a simple pattern @@ -390,15 +368,9 @@ subpatternappend = subpattern.append sourceget = source.get sourcematch = source.match - _len = len - PATTERNENDERS = _PATTERNENDERS - ASSERTCHARS = _ASSERTCHARS - LOOKBEHINDASSERTCHARS = _LOOKBEHINDASSERTCHARS - REPEATCODES = _REPEATCODES while 1: - - if source.next in PATTERNENDERS: + if source.next in PATTERN_ENDERS: break # end of subpattern this = sourceget() if this is None: @@ -406,7 +378,7 @@ if state.flags & SRE_FLAG_VERBOSE: # skip whitespace and comments - if this in WHITESPACE: + if this in WHITESPACE_CHARS: continue if this == "#": while 1: @@ -417,20 +389,17 @@ if this and this[0] not in SPECIAL_CHARS: subpatternappend((LITERAL, ord(this))) - elif this == "[": # character set - set = [] - setappend = set.append + charset = [] + setappend = charset.append ## if sourcematch(":"): ## pass # handle character classes - if sourcematch("^"): - setappend((NEGATE, None)) + negate = sourcematch("^") # check remaining characters - start = set[:] while 1: this = sourceget() - if this == "]" and set != start: + if this == "]" and charset: break elif this and this[0] == "\\": code1 = _class_escape(source, this) @@ -442,8 +411,6 @@ # potential range this = sourceget() if this == "]": - if code1[0] is IN: - code1 = code1[1][0] setappend(code1) setappend((LITERAL, ord("-"))) break @@ -462,34 +429,25 @@ else: raise error, "unexpected end of regular expression" else: - if code1[0] is IN: - code1 = code1[1][0] setappend(code1) - - # XXX: should move set optimization to compiler! - if _len(set)==1 and set[0][0] is LITERAL: - subpatternappend(set[0]) # optimization - elif _len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL: - subpatternappend((NOT_LITERAL, set[1][1])) # optimization + if negate: + subpatternappend((NOT_IN, charset)) else: - # XXX: should add charmap optimization here - subpatternappend((IN, set)) - + subpatternappend((IN, charset)) elif this and this[0] in REPEAT_CHARS: # repeat previous item if this == "?": min, max = 0, 1 elif this == "*": - min, max = 0, MAXREPEAT - + min, max = 0, MAX_REPEATS elif this == "+": - min, max = 1, MAXREPEAT + min, max = 1, MAX_REPEATS elif this == "{": if source.next == "}": subpatternappend((LITERAL, ord(this))) continue here = source.tell() - min, max = 0, MAXREPEAT + min, max = 0, MAX_REPEATS lo = hi = "" while source.next in DIGITS: lo = lo + source.get() @@ -511,28 +469,25 @@ else: raise error, "not supported" # figure out which item to repeat - if subpattern: - item = subpattern[-1:] - else: - item = None - if not item or (_len(item) == 1 and item[0][0] == AT): + item = subpattern[-1:] + if not item or len(item) == 1 and item[0][0] in POSITION_CODES: raise error, "nothing to repeat" - if item[0][0] in REPEATCODES: + if item[0][0] in REPEAT_CODES: raise error, "multiple repeat" if sourcematch("?"): - subpattern[-1] = (MIN_REPEAT, (min, max, item)) + subpattern[-1] = (REPEAT_MIN, (min, max, item)) + elif sourcematch("+"): + subpattern[-1] = (REPEAT_POSS, (min, max, item)) else: - subpattern[-1] = (MAX_REPEAT, (min, max, item)) - + subpattern[-1] = (REPEAT_MAX, (min, max, item)) elif this == ".": subpatternappend((ANY, None)) - elif this == "(": - group = 1 + group = CAPTURE_GROUP name = None condgroup = None if sourcematch("?"): - group = 0 + group = QUERY_GROUP # options if sourcematch("P"): # python extensions @@ -546,7 +501,7 @@ if char == ">": break name = name + char - group = 1 + group = CAPTURE_GROUP if not isname(name): raise error, "bad character in group name" elif sourcematch("="): @@ -573,7 +528,10 @@ raise error, "unknown specifier: ?P%s" % char elif sourcematch(":"): # non-capturing group - group = 2 + group = NONCAPTURE_GROUP + elif sourcematch(">"): + # atomic group + group = ATOMIC_GROUP elif sourcematch("#"): # comment while 1: @@ -583,12 +541,12 @@ if not sourcematch(")"): raise error, "unbalanced parenthesis" continue - elif source.next in ASSERTCHARS: + elif source.next in ASSERT_CHARS: # lookahead assertions char = sourceget() dir = 1 if char == "<": - if source.next not in LOOKBEHINDASSERTCHARS: + if source.next not in LOOKBEHIND_ASSERT_CHARS: raise error, "syntax error" dir = -1 # lookbehind char = sourceget() @@ -610,7 +568,7 @@ if char == ")": break condname = condname + char - group = 2 + group = NONCAPTURE_GROUP if isname(condname): condgroup = state.groupdict.get(condname) if condgroup is None: @@ -627,8 +585,9 @@ while source.next in FLAGS: state.flags = state.flags | FLAGS[sourceget()] if group: + atomic = group == ATOMIC_GROUP # parse group contents - if group == 2: + if group in [NONCAPTURE_GROUP, ATOMIC_GROUP]: # anonymous group group = None else: @@ -641,7 +600,10 @@ raise error, "unbalanced parenthesis" if group is not None: state.closegroup(group) - subpatternappend((SUBPATTERN, (group, p))) + if atomic: + subpatternappend((ATOMIC, (group, p))) + else: + subpatternappend((SUBPATTERN, (group, p))) else: while 1: char = sourceget() @@ -650,17 +612,13 @@ if char == ")": break raise error, "unknown extension" - elif this == "^": - subpatternappend((AT, AT_BEGINNING)) - + subpatternappend((START_OF_LINE, None)) elif this == "$": - subpattern.append((AT, AT_END)) - + subpattern.append((END_OF_LINE, None)) elif this and this[0] == "\\": code = _escape(source, this, state) subpatternappend(code) - else: raise error, "parser error" Index: Modules/_sre.c =================================================================== --- _sre.c Tue Sep 09 23:59:58 2008 +++ _sre.c Wed Sep 10 00:18:12 2008 @@ -20,7 +20,7 @@ * 2001-10-24 fl added finditer primitive (for 2.2 only) * 2001-12-07 fl fixed memory leak in sub/subn (Guido van Rossum) * 2002-11-09 fl fixed empty sub/subn return type - * 2003-04-18 mvl fully support 4-byte codes + * 2003-04-18 mvl fully support 4-byte codes * 2003-10-17 gn implemented non recursive scheme * 2008-09-09 mrab major reworking; added atomic groups, possessive quantifiers, * let lookbehinds be variable length; faster @@ -3830,7 +3830,7 @@ LOCAL(PyObject*) state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, - Py_ssize_t start, Py_ssize_t end) + Py_ssize_t start, Py_ssize_t end, SRE_CODE* pattern_code) { /* prepare state object */ @@ -5431,7 +5431,7 @@ memset(state->mark, 0, state->pattern_code[0] * sizeof(SRE_CHAR*)); if (state->charsize == 1) { - status = sre_match(state, PatternObject_GetCode(self->pattern)); + status = sre_match(state, state->pattern_ptr); } else { #if defined(HAVE_UNICODE) status = sre_umatch(state, state->pattern_ptr); Index: Modules/sre.h =================================================================== --- sre.h Mon Jun 12 04:05:40 2006 +++ sre.h Tue Sep 09 00:00:06 2008 @@ -60,6 +60,30 @@ struct SRE_REPEAT_T *prev; /* points to previous repeat context */ } SRE_REPEAT; +#define SRE_BACKTRACK_CHUNK_SIZE 1024 +#define SRE_SAVED_MARKS_CHUNK_SIZE 1024 + +typedef struct SRE_BACKTRACK_ITEM { + int op; + SRE_CODE* pattern_ptr; + void* text_ptr; + int index; + int repeat_count; + void* repeat_start; +} SRE_BACKTRACK_ITEM; + +typedef struct SRE_BACKTRACK_CHUNK { + struct SRE_BACKTRACK_CHUNK* previous; + SRE_BACKTRACK_ITEM items[SRE_BACKTRACK_CHUNK_SIZE]; + int count; +} SRE_BACKTRACK_CHUNK; + +typedef struct SRE_SAVED_MARKS_CHUNK { + struct SRE_SAVED_MARKS_CHUNK* previous; + void* marks[SRE_SAVED_MARKS_CHUNK_SIZE]; + int count; +} SRE_SAVED_MARKS_CHUNK; + typedef struct { /* string pointers */ void* ptr; /* current position (also end of current slice) */ @@ -69,6 +93,7 @@ /* attributes for the match object */ PyObject* string; Py_ssize_t pos, endpos; + int reversed; /* character size */ int charsize; /* registers */ @@ -76,9 +101,12 @@ Py_ssize_t lastmark; void* mark[SRE_MARK_SIZE]; /* dynamically allocated stuff */ - char* data_stack; - size_t data_stack_size; - size_t data_stack_base; + SRE_BACKTRACK_CHUNK* backtrack_chunk; + SRE_SAVED_MARKS_CHUNK* saved_marks_chunk; + int* repeat_counts; + void** repeat_starts; + SRE_CODE* pattern_code; + SRE_CODE* pattern_ptr; /* current repeat context */ SRE_REPEAT *repeat; /* hooks */ Index: Modules/sre_constants.h =================================================================== --- sre_constants.h Fri Oct 17 23:13:16 2003 +++ sre_constants.h Tue Sep 09 23:35:04 2008 @@ -16,64 +16,131 @@ #define SRE_OP_SUCCESS 1 #define SRE_OP_ANY 2 #define SRE_OP_ANY_ALL 3 -#define SRE_OP_ASSERT 4 -#define SRE_OP_ASSERT_NOT 5 -#define SRE_OP_AT 6 -#define SRE_OP_BRANCH 7 -#define SRE_OP_CALL 8 -#define SRE_OP_CATEGORY 9 -#define SRE_OP_CHARSET 10 -#define SRE_OP_BIGCHARSET 11 -#define SRE_OP_GROUPREF 12 -#define SRE_OP_GROUPREF_EXISTS 13 -#define SRE_OP_GROUPREF_IGNORE 14 -#define SRE_OP_IN 15 -#define SRE_OP_IN_IGNORE 16 -#define SRE_OP_INFO 17 -#define SRE_OP_JUMP 18 -#define SRE_OP_LITERAL 19 -#define SRE_OP_LITERAL_IGNORE 20 -#define SRE_OP_MARK 21 -#define SRE_OP_MAX_UNTIL 22 -#define SRE_OP_MIN_UNTIL 23 -#define SRE_OP_NOT_LITERAL 24 -#define SRE_OP_NOT_LITERAL_IGNORE 25 -#define SRE_OP_NEGATE 26 -#define SRE_OP_RANGE 27 -#define SRE_OP_REPEAT 28 -#define SRE_OP_REPEAT_ONE 29 -#define SRE_OP_SUBPATTERN 30 -#define SRE_OP_MIN_REPEAT_ONE 31 -#define SRE_AT_BEGINNING 0 -#define SRE_AT_BEGINNING_LINE 1 -#define SRE_AT_BEGINNING_STRING 2 -#define SRE_AT_BOUNDARY 3 -#define SRE_AT_NON_BOUNDARY 4 -#define SRE_AT_END 5 -#define SRE_AT_END_LINE 6 -#define SRE_AT_END_STRING 7 -#define SRE_AT_LOC_BOUNDARY 8 -#define SRE_AT_LOC_NON_BOUNDARY 9 -#define SRE_AT_UNI_BOUNDARY 10 -#define SRE_AT_UNI_NON_BOUNDARY 11 -#define SRE_CATEGORY_DIGIT 0 -#define SRE_CATEGORY_NOT_DIGIT 1 -#define SRE_CATEGORY_SPACE 2 -#define SRE_CATEGORY_NOT_SPACE 3 -#define SRE_CATEGORY_WORD 4 -#define SRE_CATEGORY_NOT_WORD 5 -#define SRE_CATEGORY_LINEBREAK 6 -#define SRE_CATEGORY_NOT_LINEBREAK 7 -#define SRE_CATEGORY_LOC_WORD 8 -#define SRE_CATEGORY_LOC_NOT_WORD 9 -#define SRE_CATEGORY_UNI_DIGIT 10 -#define SRE_CATEGORY_UNI_NOT_DIGIT 11 -#define SRE_CATEGORY_UNI_SPACE 12 -#define SRE_CATEGORY_UNI_NOT_SPACE 13 -#define SRE_CATEGORY_UNI_WORD 14 -#define SRE_CATEGORY_UNI_NOT_WORD 15 -#define SRE_CATEGORY_UNI_LINEBREAK 16 -#define SRE_CATEGORY_UNI_NOT_LINEBREAK 17 +#define SRE_OP_ANY_ALL_REV 4 +#define SRE_OP_ANY_REV 5 +#define SRE_OP_ASSERT 6 +#define SRE_OP_ASSERT_NOT 7 +#define SRE_OP_ATOMIC 8 +#define SRE_OP_BIGCHARSET 9 +#define SRE_OP_BIGCHARSET_IGNORE 10 +#define SRE_OP_BIGCHARSET_IGNORE_REV 11 +#define SRE_OP_BIGCHARSET_REV 12 +#define SRE_OP_BOUNDARY 13 +#define SRE_OP_BRANCH 14 +#define SRE_OP_CALL 15 +#define SRE_OP_CHARSET 16 +#define SRE_OP_CHARSET_IGNORE 17 +#define SRE_OP_CHARSET_IGNORE_REV 18 +#define SRE_OP_CHARSET_REV 19 +#define SRE_OP_DIGIT 20 +#define SRE_OP_DIGIT_REV 21 +#define SRE_OP_END_ASSERT 22 +#define SRE_OP_END_ASSERT_NOT 23 +#define SRE_OP_END_ATOMIC 24 +#define SRE_OP_END_OF_LINE 25 +#define SRE_OP_END_OF_STRING 26 +#define SRE_OP_END_OF_STRING_2 27 +#define SRE_OP_END_REPEAT_MAX 28 +#define SRE_OP_END_REPEAT_MAX_REV 29 +#define SRE_OP_END_REPEAT_MIN 30 +#define SRE_OP_END_REPEAT_MIN_REV 31 +#define SRE_OP_END_REPEAT_ONE_MAX 32 +#define SRE_OP_END_REPEAT_ONE_MAX_REV 33 +#define SRE_OP_END_REPEAT_ONE_MIN 34 +#define SRE_OP_END_REPEAT_ONE_MIN_REV 35 +#define SRE_OP_END_REPEAT_ONE_POSS 36 +#define SRE_OP_END_REPEAT_ONE_POSS_REV 37 +#define SRE_OP_END_REPEAT_POSS 38 +#define SRE_OP_END_REPEAT_POSS_REV 39 +#define SRE_OP_GROUPREF 40 +#define SRE_OP_GROUPREF_EXISTS 41 +#define SRE_OP_GROUPREF_IGNORE 42 +#define SRE_OP_GROUPREF_IGNORE_REV 43 +#define SRE_OP_GROUPREF_REV 44 +#define SRE_OP_IN 45 +#define SRE_OP_IN_IGNORE 46 +#define SRE_OP_IN_IGNORE_REV 47 +#define SRE_OP_IN_REV 48 +#define SRE_OP_JUMP 49 +#define SRE_OP_LITERAL 50 +#define SRE_OP_LITERAL_IGNORE 51 +#define SRE_OP_LITERAL_IGNORE_REV 52 +#define SRE_OP_LITERAL_REV 53 +#define SRE_OP_LITERAL_STRING 54 +#define SRE_OP_LITERAL_STRING_IGNORE 55 +#define SRE_OP_LITERAL_STRING_IGNORE_REV 56 +#define SRE_OP_LITERAL_STRING_REV 57 +#define SRE_OP_LOC_BOUNDARY 58 +#define SRE_OP_LOC_NOT_BOUNDARY 59 +#define SRE_OP_LOC_NOT_WORD 60 +#define SRE_OP_LOC_NOT_WORD_REV 61 +#define SRE_OP_LOC_WORD 62 +#define SRE_OP_LOC_WORD_REV 63 +#define SRE_OP_MARK 64 +#define SRE_OP_NOT_BIGCHARSET 65 +#define SRE_OP_NOT_BIGCHARSET_IGNORE 66 +#define SRE_OP_NOT_BIGCHARSET_IGNORE_REV 67 +#define SRE_OP_NOT_BIGCHARSET_REV 68 +#define SRE_OP_NOT_BOUNDARY 69 +#define SRE_OP_NOT_CHARSET 70 +#define SRE_OP_NOT_CHARSET_IGNORE 71 +#define SRE_OP_NOT_CHARSET_IGNORE_REV 72 +#define SRE_OP_NOT_CHARSET_REV 73 +#define SRE_OP_NOT_DIGIT 74 +#define SRE_OP_NOT_DIGIT_REV 75 +#define SRE_OP_NOT_IN 76 +#define SRE_OP_NOT_IN_IGNORE 77 +#define SRE_OP_NOT_IN_IGNORE_REV 78 +#define SRE_OP_NOT_IN_REV 79 +#define SRE_OP_NOT_LITERAL 80 +#define SRE_OP_NOT_LITERAL_IGNORE 81 +#define SRE_OP_NOT_LITERAL_IGNORE_REV 82 +#define SRE_OP_NOT_LITERAL_REV 83 +#define SRE_OP_NOT_RANGE 84 +#define SRE_OP_NOT_RANGE_IGNORE 85 +#define SRE_OP_NOT_RANGE_IGNORE_REV 86 +#define SRE_OP_NOT_RANGE_REV 87 +#define SRE_OP_NOT_WHITESPACE 88 +#define SRE_OP_NOT_WHITESPACE_REV 89 +#define SRE_OP_NOT_WORD 90 +#define SRE_OP_NOT_WORD_REV 91 +#define SRE_OP_RANGE 92 +#define SRE_OP_RANGE_IGNORE 93 +#define SRE_OP_RANGE_IGNORE_REV 94 +#define SRE_OP_RANGE_REV 95 +#define SRE_OP_REPEAT_MAX 96 +#define SRE_OP_REPEAT_MAX_REV 97 +#define SRE_OP_REPEAT_MIN 98 +#define SRE_OP_REPEAT_MIN_REV 99 +#define SRE_OP_REPEAT_ONE_MAX 100 +#define SRE_OP_REPEAT_ONE_MAX_REV 101 +#define SRE_OP_REPEAT_ONE_MIN 102 +#define SRE_OP_REPEAT_ONE_MIN_REV 103 +#define SRE_OP_REPEAT_ONE_POSS 104 +#define SRE_OP_REPEAT_ONE_POSS_REV 105 +#define SRE_OP_REPEAT_POSS 106 +#define SRE_OP_REPEAT_POSS_REV 107 +#define SRE_OP_START_OF_LINE 108 +#define SRE_OP_START_OF_STRING 109 +#define SRE_OP_SUBPATTERN 110 +#define SRE_OP_UNI_BOUNDARY 111 +#define SRE_OP_UNI_DIGIT 112 +#define SRE_OP_UNI_DIGIT_REV 113 +#define SRE_OP_UNI_NOT_BOUNDARY 114 +#define SRE_OP_UNI_NOT_DIGIT 115 +#define SRE_OP_UNI_NOT_DIGIT_REV 116 +#define SRE_OP_UNI_NOT_WHITESPACE 117 +#define SRE_OP_UNI_NOT_WHITESPACE_REV 118 +#define SRE_OP_UNI_NOT_WORD 119 +#define SRE_OP_UNI_NOT_WORD_REV 120 +#define SRE_OP_UNI_WHITESPACE 121 +#define SRE_OP_UNI_WHITESPACE_REV 122 +#define SRE_OP_UNI_WORD 123 +#define SRE_OP_UNI_WORD_REV 124 +#define SRE_OP_WHITESPACE 125 +#define SRE_OP_WHITESPACE_REV 126 +#define SRE_OP_WORD 127 +#define SRE_OP_WORD_REV 128 #define SRE_FLAG_TEMPLATE 1 #define SRE_FLAG_IGNORECASE 2 #define SRE_FLAG_LOCALE 4