-- use charset bitmaps where appropriate. this gives a 5-10%

speedup for some tests, including the python tokenizer.

-- added support for an optional charset anchor to the engine
   (currently unused by the code generator).

-- removed workaround for array module bug.
This commit is contained in:
Fredrik Lundh 2000-07-02 12:00:07 +00:00
parent c13222cdff
commit 3562f11764
5 changed files with 182 additions and 63 deletions

View File

@ -16,12 +16,71 @@ import _sre
from sre_constants import * from sre_constants import *
# find an array type code that matches the engine's code size # find an array type code that matches the engine's code size
for WORDSIZE in "BHil": for WORDSIZE in "Hil":
if len(array.array(WORDSIZE, [0]).tostring()) == _sre.getcodesize(): if len(array.array(WORDSIZE, [0]).tostring()) == _sre.getcodesize():
break break
else: else:
raise RuntimeError, "cannot find a useable array type" raise RuntimeError, "cannot find a useable array type"
MAXCODE = 65535
def _charset(charset, fixup):
# internal: optimize character set
out = []
charmap = [0]*256
try:
for op, av in charset:
if op is NEGATE:
out.append((op, av))
elif op is LITERAL:
charmap[fixup(av)] = 1
elif op is RANGE:
for i in range(fixup(av[0]), fixup(av[1])+1):
charmap[i] = 1
elif op is CATEGORY:
# FIXME: could append to charmap tail
return charset # cannot compress
except IndexError:
# unicode
return charset
# compress character map
i = p = n = 0
runs = []
for c in charmap:
if c:
if n == 0:
p = i
n = n + 1
elif n:
runs.append((p, n))
n = 0
i = i + 1
if n:
runs.append((p, n))
if len(runs) <= 2:
# use literal/range
for p, n in runs:
if n == 1:
out.append((LITERAL, p))
else:
out.append((RANGE, (p, p+n-1)))
if len(out) < len(charset):
return out
else:
# use bitmap
data = []
m = 1; v = 0
for c in charmap:
if c:
v = v + m
m = m << 1
if m > MAXCODE:
data.append(v)
m = 1; v = 0
out.append((CHARSET, data))
return out
return charset
def _compile(code, pattern, flags): def _compile(code, pattern, flags):
# internal: compile a (sub)pattern # internal: compile a (sub)pattern
emit = code.append emit = code.append
@ -41,7 +100,7 @@ def _compile(code, pattern, flags):
emit(OPCODES[op]) emit(OPCODES[op])
fixup = lambda x: x fixup = lambda x: x
skip = len(code); emit(0) skip = len(code); emit(0)
for op, av in av: for op, av in _charset(av, fixup):
emit(OPCODES[op]) emit(OPCODES[op])
if op is NEGATE: if op is NEGATE:
pass pass
@ -50,6 +109,8 @@ def _compile(code, pattern, flags):
elif op is RANGE: elif op is RANGE:
emit(fixup(av[0])) emit(fixup(av[0]))
emit(fixup(av[1])) emit(fixup(av[1]))
elif op is CHARSET:
code.extend(av)
elif op is CATEGORY: elif op is CATEGORY:
if flags & SRE_FLAG_LOCALE: if flags & SRE_FLAG_LOCALE:
emit(CHCODES[CH_LOCALE[av]]) emit(CHCODES[CH_LOCALE[av]])
@ -155,13 +216,14 @@ def _compile(code, pattern, flags):
def _compile_info(code, pattern, flags): def _compile_info(code, pattern, flags):
# internal: compile an info block. in the current version, # internal: compile an info block. in the current version,
# this contains min/max pattern width and a literal prefix, # this contains min/max pattern width, and an optional literal
# if any # prefix or a character map
lo, hi = pattern.getwidth() lo, hi = pattern.getwidth()
if lo == 0: if lo == 0:
return # not worth it return # not worth it
# look for a literal prefix # look for a literal prefix
prefix = [] prefix = []
charset = [] # not used
if not (flags & SRE_FLAG_IGNORECASE): if not (flags & SRE_FLAG_IGNORECASE):
for op, av in pattern.data: for op, av in pattern.data:
if op is LITERAL: if op is LITERAL:
@ -174,16 +236,25 @@ def _compile_info(code, pattern, flags):
skip = len(code); emit(0) skip = len(code); emit(0)
# literal flag # literal flag
mask = 0 mask = 0
if prefix:
mask = SRE_INFO_PREFIX
if len(prefix) == len(pattern.data): if len(prefix) == len(pattern.data):
mask = 1 mask = mask + SRE_INFO_LITERAL
elif charset:
mask = mask + SRE_INFO_CHARSET
emit(mask) emit(mask)
# pattern length # pattern length
if lo < MAXCODE:
emit(lo) emit(lo)
if hi < 32768: else:
emit(MAXCODE)
prefix = prefix[:MAXCODE]
if hi < MAXCODE:
emit(hi) emit(hi)
else: else:
emit(0) emit(0)
# add literal prefix # add literal prefix
if prefix:
emit(len(prefix)) emit(len(prefix))
if prefix: if prefix:
code.extend(prefix) code.extend(prefix)
@ -194,6 +265,11 @@ def _compile_info(code, pattern, flags):
while table[i+1] > 0 and prefix[i] != prefix[table[i+1]-1]: while table[i+1] > 0 and prefix[i] != prefix[table[i+1]-1]:
table[i+1] = table[table[i+1]-1]+1 table[i+1] = table[table[i+1]-1]+1
code.extend(table[1:]) # don't store first entry code.extend(table[1:]) # don't store first entry
elif charset:
for char in charset:
emit(OPCODES[LITERAL])
emit(char)
emit(OPCODES[FAILURE])
code[skip] = len(code) - skip code[skip] = len(code) - skip
def compile(p, flags=0): def compile(p, flags=0):

View File

@ -28,6 +28,7 @@ AT = "at"
BRANCH = "branch" BRANCH = "branch"
CALL = "call" CALL = "call"
CATEGORY = "category" CATEGORY = "category"
CHARSET = "charset"
GROUP = "group" GROUP = "group"
GROUP_IGNORE = "group_ignore" GROUP_IGNORE = "group_ignore"
IN = "in" IN = "in"
@ -87,6 +88,7 @@ OPCODES = [
BRANCH, BRANCH,
CALL, CALL,
CATEGORY, CATEGORY,
CHARSET,
GROUP, GROUP_IGNORE, GROUP, GROUP_IGNORE,
IN, IN_IGNORE, IN, IN_IGNORE,
INFO, INFO,
@ -166,13 +168,18 @@ CH_UNICODE = {
} }
# flags # flags
SRE_FLAG_TEMPLATE = 1 SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking)
SRE_FLAG_IGNORECASE = 2 SRE_FLAG_IGNORECASE = 2 # case insensitive
SRE_FLAG_LOCALE = 4 SRE_FLAG_LOCALE = 4 # honour system locale
SRE_FLAG_MULTILINE = 8 SRE_FLAG_MULTILINE = 8 # treat target as multiline string
SRE_FLAG_DOTALL = 16 SRE_FLAG_DOTALL = 16 # treat target as a single string
SRE_FLAG_UNICODE = 32 SRE_FLAG_UNICODE = 32 # use unicode locale
SRE_FLAG_VERBOSE = 64 SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments
# flags for INFO primitive
SRE_INFO_PREFIX = 1 # has prefix
SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix)
SRE_INFO_CHARSET = 4 # pattern starts with character from given set
if __name__ == "__main__": if __name__ == "__main__":
import string import string
@ -201,6 +208,7 @@ if __name__ == "__main__":
dump(f, OPCODES, "SRE_OP") dump(f, OPCODES, "SRE_OP")
dump(f, ATCODES, "SRE") dump(f, ATCODES, "SRE")
dump(f, CHCODES, "SRE") dump(f, CHCODES, "SRE")
f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE) f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE)
f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE) f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE)
f.write("#define SRE_FLAG_LOCALE %d\n" % SRE_FLAG_LOCALE) f.write("#define SRE_FLAG_LOCALE %d\n" % SRE_FLAG_LOCALE)
@ -208,5 +216,10 @@ if __name__ == "__main__":
f.write("#define SRE_FLAG_DOTALL %d\n" % SRE_FLAG_DOTALL) f.write("#define SRE_FLAG_DOTALL %d\n" % SRE_FLAG_DOTALL)
f.write("#define SRE_FLAG_UNICODE %d\n" % SRE_FLAG_UNICODE) f.write("#define SRE_FLAG_UNICODE %d\n" % SRE_FLAG_UNICODE)
f.write("#define SRE_FLAG_VERBOSE %d\n" % SRE_FLAG_VERBOSE) f.write("#define SRE_FLAG_VERBOSE %d\n" % SRE_FLAG_VERBOSE)
f.write("#define SRE_INFO_PREFIX %d\n" % SRE_INFO_PREFIX)
f.write("#define SRE_INFO_LITERAL %d\n" % SRE_INFO_LITERAL)
f.write("#define SRE_INFO_CHARSET %d\n" % SRE_INFO_CHARSET)
f.close() f.close()
print "done" print "done"

View File

@ -16,11 +16,10 @@ import _sre
from sre_constants import * from sre_constants import *
# FIXME: should be 65535, but the arraymodule is still broken MAXREPEAT = 65535
MAXREPEAT = 32767
# FIXME: might change in 2.0 final. but for now, this seems # FIXME: the following might change in 2.0 final. but for now, this
# to be the best way to be compatible with 1.5.2 # seems to be the best way to be compatible with 1.5.2
CHARMASK = 0xff CHARMASK = 0xff
SPECIAL_CHARS = ".\\[{()*+?^$|" SPECIAL_CHARS = ".\\[{()*+?^$|"

View File

@ -378,6 +378,13 @@ SRE_MEMBER(SRE_CODE* set, SRE_CODE ch)
set += 2; set += 2;
break; break;
case SRE_OP_CHARSET:
/* args: <bitmap> (16 bits per code word) */
if (ch < 256 && (set[ch >> 4] & (1 << (ch & 15))))
return ok;
set += 16;
break;
case SRE_OP_CATEGORY: case SRE_OP_CATEGORY:
/* args: <category> */ /* args: <category> */
if (sre_category(set[0], (int) ch)) if (sre_category(set[0], (int) ch))
@ -952,35 +959,38 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern)
SRE_CHAR* ptr = state->start; SRE_CHAR* ptr = state->start;
SRE_CHAR* end = state->end; SRE_CHAR* end = state->end;
int status = 0; int status = 0;
int prefix_len = 0; int prefix_len;
SRE_CODE* prefix; SRE_CODE* prefix = NULL;
SRE_CODE* overlap; SRE_CODE* charset = NULL;
int literal = 0; SRE_CODE* overlap = NULL;
int flags = 0;
if (pattern[0] == SRE_OP_INFO) { if (pattern[0] == SRE_OP_INFO) {
/* optimization info block */ /* optimization info block */
/* args: <1=skip> <2=flags> <3=min> <4=max> <5=prefix> <6=data...> */ /* args: <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> */
flags = pattern[2];
if (pattern[3] > 0) { if (pattern[3] > 0) {
/* adjust end point (but make sure we leave at least one /* adjust end point (but make sure we leave at least one
character in there) */ character in there, so literal search will work) */
end -= pattern[3]-1; end -= pattern[3]-1;
if (end <= ptr) if (end <= ptr)
end = ptr+1; end = ptr+1;
} }
literal = pattern[2]; if (flags & SRE_INFO_PREFIX) {
prefix = pattern + 6;
prefix_len = pattern[5]; prefix_len = pattern[5];
prefix = pattern + 6;
overlap = prefix + prefix_len - 1; overlap = prefix + prefix_len - 1;
} else if (flags & SRE_INFO_CHARSET)
charset = pattern + 5;
pattern += 1 + pattern[1]; pattern += 1 + pattern[1];
} }
#if defined(USE_FAST_SEARCH) #if defined(USE_FAST_SEARCH)
if (prefix_len > 1) { if (prefix && overlap && prefix_len > 1) {
/* pattern starts with a known prefix. use the overlap /* pattern starts with a known prefix. use the overlap
table to skip forward as fast as we possibly can */ table to skip forward as fast as we possibly can */
int i = 0; int i = 0;
@ -998,8 +1008,8 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern)
TRACE(("%8d: === SEARCH === hit\n", PTR(ptr))); TRACE(("%8d: === SEARCH === hit\n", PTR(ptr)));
state->start = ptr - prefix_len + 1; state->start = ptr - prefix_len + 1;
state->ptr = ptr + 1; state->ptr = ptr + 1;
if (literal) if (flags & SRE_INFO_LITERAL)
return 1; /* all of it */ return 1; /* we got all of it */
status = SRE_MATCH(state, pattern + 2*prefix_len); status = SRE_MATCH(state, pattern + 2*prefix_len);
if (status != 0) if (status != 0)
return status; return status;
@ -1017,8 +1027,8 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern)
#endif #endif
if (pattern[0] == SRE_OP_LITERAL) { if (pattern[0] == SRE_OP_LITERAL) {
/* pattern starts with a literal character. this is used for /* pattern starts with a literal character. this is used
short prefixes, and if fast search is disabled*/ for short prefixes, and if fast search is disabled */
SRE_CODE chr = pattern[1]; SRE_CODE chr = pattern[1];
for (;;) { for (;;) {
while (ptr < end && (SRE_CODE) ptr[0] != chr) while (ptr < end && (SRE_CODE) ptr[0] != chr)
@ -1032,6 +1042,22 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern)
if (status != 0) if (status != 0)
break; break;
} }
#if 0
} else if (charset) {
/* pattern starts with a character from a known set */
for (;;) {
while (ptr < end && !SRE_MEMBER(charset, ptr[0]))
ptr++;
if (ptr == end)
return 0;
TRACE(("%8d: === SEARCH === charset\n", PTR(ptr)));
state->start = ptr;
state->ptr = ptr;
status = SRE_MATCH(state, pattern);
if (status != 0)
break;
}
#endif
} else } else
/* general case */ /* general case */
while (ptr <= end) { while (ptr <= end) {
@ -1045,6 +1071,7 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern)
return status; return status;
} }
#if !defined(SRE_RECURSIVE) #if !defined(SRE_RECURSIVE)
/* -------------------------------------------------------------------- */ /* -------------------------------------------------------------------- */

View File

@ -20,23 +20,24 @@
#define SRE_OP_BRANCH 6 #define SRE_OP_BRANCH 6
#define SRE_OP_CALL 7 #define SRE_OP_CALL 7
#define SRE_OP_CATEGORY 8 #define SRE_OP_CATEGORY 8
#define SRE_OP_GROUP 9 #define SRE_OP_CHARSET 9
#define SRE_OP_GROUP_IGNORE 10 #define SRE_OP_GROUP 10
#define SRE_OP_IN 11 #define SRE_OP_GROUP_IGNORE 11
#define SRE_OP_IN_IGNORE 12 #define SRE_OP_IN 12
#define SRE_OP_INFO 13 #define SRE_OP_IN_IGNORE 13
#define SRE_OP_JUMP 14 #define SRE_OP_INFO 14
#define SRE_OP_LITERAL 15 #define SRE_OP_JUMP 15
#define SRE_OP_LITERAL_IGNORE 16 #define SRE_OP_LITERAL 16
#define SRE_OP_MARK 17 #define SRE_OP_LITERAL_IGNORE 17
#define SRE_OP_MAX_REPEAT 18 #define SRE_OP_MARK 18
#define SRE_OP_MAX_REPEAT_ONE 19 #define SRE_OP_MAX_REPEAT 19
#define SRE_OP_MIN_REPEAT 20 #define SRE_OP_MAX_REPEAT_ONE 20
#define SRE_OP_NOT_LITERAL 21 #define SRE_OP_MIN_REPEAT 21
#define SRE_OP_NOT_LITERAL_IGNORE 22 #define SRE_OP_NOT_LITERAL 22
#define SRE_OP_NEGATE 23 #define SRE_OP_NOT_LITERAL_IGNORE 23
#define SRE_OP_RANGE 24 #define SRE_OP_NEGATE 24
#define SRE_OP_REPEAT 25 #define SRE_OP_RANGE 25
#define SRE_OP_REPEAT 26
#define SRE_AT_BEGINNING 0 #define SRE_AT_BEGINNING 0
#define SRE_AT_BEGINNING_LINE 1 #define SRE_AT_BEGINNING_LINE 1
#define SRE_AT_BOUNDARY 2 #define SRE_AT_BOUNDARY 2
@ -68,3 +69,6 @@
#define SRE_FLAG_DOTALL 16 #define SRE_FLAG_DOTALL 16
#define SRE_FLAG_UNICODE 32 #define SRE_FLAG_UNICODE 32
#define SRE_FLAG_VERBOSE 64 #define SRE_FLAG_VERBOSE 64
#define SRE_INFO_PREFIX 1
#define SRE_INFO_LITERAL 2
#define SRE_INFO_CHARSET 4