traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Simple Optimizations: 

* Factor constant expressions out of loops.
* Presize a list being grown to a known length.
This commit is contained in:
Raymond Hettinger 2004-03-26 23:24:00 +00:00
parent 29e383754e
commit 968c56a626
1 changed files with 92 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
Lib/sre_parse.py
View File

@ -103,6 +103,7 @@ class SubPattern:
self.width = None self.width = None
def dump(self, level=0): def dump(self, level=0):
nl = 1 nl = 1
seqtypes = type(()), type([])
for op, av in self.data: for op, av in self.data:
print level*" " + op,; nl = 0 print level*" " + op,; nl = 0
if op == "in": if op == "in":
@ -118,7 +119,7 @@ class SubPattern:
print level*" " + "or" print level*" " + "or"
a.dump(level+1); nl = 1 a.dump(level+1); nl = 1
i = i + 1 i = i + 1
elif type(av) in (type(()), type([])): elif type(av) in seqtypes:
for a in av: for a in av:
if isinstance(a, SubPattern): if isinstance(a, SubPattern):
if not nl: print if not nl: print
@ -149,6 +150,8 @@ class SubPattern:
if self.width: if self.width:
return self.width return self.width
lo = hi = 0L lo = hi = 0L
UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY)
REPEATCODES = (MIN_REPEAT, MAX_REPEAT)
for op, av in self.data: for op, av in self.data:
if op is BRANCH: if op is BRANCH:
i = sys.maxint i = sys.maxint
@ -167,11 +170,11 @@ class SubPattern:
i, j = av[1].getwidth() i, j = av[1].getwidth()
lo = lo + i lo = lo + i
hi = hi + j hi = hi + j
elif op in (MIN_REPEAT, MAX_REPEAT): elif op in REPEATCODES:
i, j = av[2].getwidth() i, j = av[2].getwidth()
lo = lo + long(i) * av[0] lo = lo + long(i) * av[0]
hi = hi + long(j) * av[1] hi = hi + long(j) * av[1]
elif op in (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY): elif op in UNITCODES:
lo = lo + 1 lo = lo + 1
hi = hi + 1 hi = hi + 1
elif op == SUCCESS: elif op == SUCCESS:
@ -313,13 +316,15 @@ def _parse_sub(source, state, nested=1):
# parse an alternation: a|b|c # parse an alternation: a|b|c
items = [] items = []
itemsappend = items.append
sourcematch = source.match
while 1: while 1:
items.append(_parse(source, state)) itemsappend(_parse(source, state))
if source.match("|"): if sourcematch("|"):
continue continue
if not nested: if not nested:
break break
if not source.next or source.match(")", 0): if not source.next or sourcematch(")", 0):
break break
else: else:
raise error, "pattern not properly closed" raise error, "pattern not properly closed"
@ -328,6 +333,7 @@ def _parse_sub(source, state, nested=1):
return items[0] return items[0]
subpattern = SubPattern(state) subpattern = SubPattern(state)
subpatternappend = subpattern.append
# check if all items share a common prefix # check if all items share a common prefix
while 1: while 1:
@ -344,7 +350,7 @@ def _parse_sub(source, state, nested=1):
# move it out of the branch # move it out of the branch
for item in items: for item in items:
del item[0] del item[0]
subpattern.append(prefix) subpatternappend(prefix)
continue # check next one continue # check next one
break break
@ -356,9 +362,10 @@ def _parse_sub(source, state, nested=1):
# we can store this as a character set instead of a # we can store this as a character set instead of a
# branch (the compiler may optimize this even more) # branch (the compiler may optimize this even more)
set = [] set = []
setappend = set.append
for item in items: for item in items:
set.append(item[0]) setappend(item[0])
subpattern.append((IN, set)) subpatternappend((IN, set))
return subpattern return subpattern
subpattern.append((BRANCH, (None, items))) subpattern.append((BRANCH, (None, items)))
@ -380,14 +387,23 @@ def _parse_sub_cond(source, state, condgroup):
def _parse(source, state): def _parse(source, state):
# parse a simple pattern # parse a simple pattern
subpattern = SubPattern(state) subpattern = SubPattern(state)
# precompute constants into local variables
subpatternappend = subpattern.append
sourceget = source.get
sourcematch = source.match
_len = len
PATTERNENDERS = ("|", ")")
ASSERTCHARS = ("=", "!", "<")
LOOKBEHINDASSERTCHARS = ("=", "!")
REPEATCODES = (MIN_REPEAT, MAX_REPEAT)
while 1: while 1:
if source.next in ("|", ")"): if source.next in PATTERNENDERS:
break # end of subpattern break # end of subpattern
this = source.get() this = sourceget()
if this is None: if this is None:
break # end of pattern break # end of pattern
@ -397,25 +413,26 @@ def _parse(source, state):
continue continue
if this == "#": if this == "#":
while 1: while 1:
this = source.get() this = sourceget()
if this in (None, "\n"): if this in (None, "\n"):
break break
continue continue
if this and this[0] not in SPECIAL_CHARS: if this and this[0] not in SPECIAL_CHARS:
subpattern.append((LITERAL, ord(this))) subpatternappend((LITERAL, ord(this)))
elif this == "[": elif this == "[":
# character set # character set
set = [] set = []
## if source.match(":"): setappend = set.append
## if sourcematch(":"):
## pass # handle character classes ## pass # handle character classes
if source.match("^"): if sourcematch("^"):
set.append((NEGATE, None)) setappend((NEGATE, None))
# check remaining characters # check remaining characters
start = set[:] start = set[:]
while 1: while 1:
this = source.get() this = sourceget()
if this == "]" and set != start: if this == "]" and set != start:
break break
elif this and this[0] == "\\": elif this and this[0] == "\\":
@ -424,14 +441,14 @@ def _parse(source, state):
code1 = LITERAL, ord(this) code1 = LITERAL, ord(this)
else: else:
raise error, "unexpected end of regular expression" raise error, "unexpected end of regular expression"
if source.match("-"): if sourcematch("-"):
# potential range # potential range
this = source.get() this = sourceget()
if this == "]": if this == "]":
if code1[0] is IN: if code1[0] is IN:
code1 = code1[1][0] code1 = code1[1][0]
set.append(code1) setappend(code1)
set.append((LITERAL, ord("-"))) setappend((LITERAL, ord("-")))
break break
elif this: elif this:
if this[0] == "\\": if this[0] == "\\":
@ -444,22 +461,22 @@ def _parse(source, state):
hi = code2[1] hi = code2[1]
if hi < lo: if hi < lo:
raise error, "bad character range" raise error, "bad character range"
set.append((RANGE, (lo, hi))) setappend((RANGE, (lo, hi)))
else: else:
raise error, "unexpected end of regular expression" raise error, "unexpected end of regular expression"
else: else:
if code1[0] is IN: if code1[0] is IN:
code1 = code1[1][0] code1 = code1[1][0]
set.append(code1) setappend(code1)
# XXX: <fl> should move set optimization to compiler! # XXX: <fl> should move set optimization to compiler!
if len(set)==1 and set[0][0] is LITERAL: if _len(set)==1 and set[0][0] is LITERAL:
subpattern.append(set[0]) # optimization subpatternappend(set[0]) # optimization
elif len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL: elif _len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL:
subpattern.append((NOT_LITERAL, set[1][1])) # optimization subpatternappend((NOT_LITERAL, set[1][1])) # optimization
else: else:
# XXX: <fl> should add charmap optimization here # XXX: <fl> should add charmap optimization here
subpattern.append((IN, set)) subpatternappend((IN, set))
elif this and this[0] in REPEAT_CHARS: elif this and this[0] in REPEAT_CHARS:
# repeat previous item # repeat previous item
@ -476,13 +493,13 @@ def _parse(source, state):
lo = hi = "" lo = hi = ""
while source.next in DIGITS: while source.next in DIGITS:
lo = lo + source.get() lo = lo + source.get()
if source.match(","): if sourcematch(","):
while source.next in DIGITS: while source.next in DIGITS:
hi = hi + source.get() hi = hi + sourceget()
else: else:
hi = lo hi = lo
if not source.match("}"): if not sourcematch("}"):
subpattern.append((LITERAL, ord(this))) subpatternappend((LITERAL, ord(this)))
source.seek(here) source.seek(here)
continue continue
if lo: if lo:
@ -498,32 +515,32 @@ def _parse(source, state):
item = subpattern[-1:] item = subpattern[-1:]
else: else:
item = None item = None
if not item or (len(item) == 1 and item[0][0] == AT): if not item or (_len(item) == 1 and item[0][0] == AT):
raise error, "nothing to repeat" raise error, "nothing to repeat"
if item[0][0] in (MIN_REPEAT, MAX_REPEAT): if item[0][0] in REPEATCODES:
raise error, "multiple repeat" raise error, "multiple repeat"
if source.match("?"): if sourcematch("?"):
subpattern[-1] = (MIN_REPEAT, (min, max, item)) subpattern[-1] = (MIN_REPEAT, (min, max, item))
else: else:
subpattern[-1] = (MAX_REPEAT, (min, max, item)) subpattern[-1] = (MAX_REPEAT, (min, max, item))
elif this == ".": elif this == ".":
subpattern.append((ANY, None)) subpatternappend((ANY, None))
elif this == "(": elif this == "(":
group = 1 group = 1
name = None name = None
condgroup = None condgroup = None
if source.match("?"): if sourcematch("?"):
group = 0 group = 0
# options # options
if source.match("P"): if sourcematch("P"):
# python extensions # python extensions
if source.match("<"): if sourcematch("<"):
# named group: skip forward to end of name # named group: skip forward to end of name
name = "" name = ""
while 1: while 1:
char = source.get() char = sourceget()
if char is None: if char is None:
raise error, "unterminated name" raise error, "unterminated name"
if char == ">": if char == ">":
@ -532,11 +549,11 @@ def _parse(source, state):
group = 1 group = 1
if not isname(name): if not isname(name):
raise error, "bad character in group name" raise error, "bad character in group name"
elif source.match("="): elif sourcematch("="):
# named backreference # named backreference
name = "" name = ""
while 1: while 1:
char = source.get() char = sourceget()
if char is None: if char is None:
raise error, "unterminated name" raise error, "unterminated name"
if char == ")": if char == ")":
@ -547,47 +564,47 @@ def _parse(source, state):
gid = state.groupdict.get(name) gid = state.groupdict.get(name)
if gid is None: if gid is None:
raise error, "unknown group name" raise error, "unknown group name"
subpattern.append((GROUPREF, gid)) subpatternappend((GROUPREF, gid))
continue continue
else: else:
char = source.get() char = sourceget()
if char is None: if char is None:
raise error, "unexpected end of pattern" raise error, "unexpected end of pattern"
raise error, "unknown specifier: ?P%s" % char raise error, "unknown specifier: ?P%s" % char
elif source.match(":"): elif sourcematch(":"):
# non-capturing group # non-capturing group
group = 2 group = 2
elif source.match("#"): elif sourcematch("#"):
# comment # comment
while 1: while 1:
if source.next is None or source.next == ")": if source.next is None or source.next == ")":
break break
source.get() sourceget()
if not source.match(")"): if not sourcematch(")"):
raise error, "unbalanced parenthesis" raise error, "unbalanced parenthesis"
continue continue
elif source.next in ("=", "!", "<"): elif source.next in ASSERTCHARS:
# lookahead assertions # lookahead assertions
char = source.get() char = sourceget()
dir = 1 dir = 1
if char == "<": if char == "<":
if source.next not in ("=", "!"): if source.next not in LOOKBEHINDASSERTCHARS:
raise error, "syntax error" raise error, "syntax error"
dir = -1 # lookbehind dir = -1 # lookbehind
char = source.get() char = sourceget()
p = _parse_sub(source, state) p = _parse_sub(source, state)
if not source.match(")"): if not sourcematch(")"):
raise error, "unbalanced parenthesis" raise error, "unbalanced parenthesis"
if char == "=": if char == "=":
subpattern.append((ASSERT, (dir, p))) subpatternappend((ASSERT, (dir, p)))
else: else:
subpattern.append((ASSERT_NOT, (dir, p))) subpatternappend((ASSERT_NOT, (dir, p)))
continue continue
elif source.match("("): elif sourcematch("("):
# conditional backreference group # conditional backreference group
condname = "" condname = ""
while 1: while 1:
char = source.get() char = sourceget()
if char is None: if char is None:
raise error, "unterminated name" raise error, "unterminated name"
if char == ")": if char == ")":
@ -608,7 +625,7 @@ def _parse(source, state):
if not source.next in FLAGS: if not source.next in FLAGS:
raise error, "unexpected end of pattern" raise error, "unexpected end of pattern"
while source.next in FLAGS: while source.next in FLAGS:
state.flags = state.flags | FLAGS[source.get()] state.flags = state.flags | FLAGS[sourceget()]
if group: if group:
# parse group contents # parse group contents
if group == 2: if group == 2:
@ -620,14 +637,14 @@ def _parse(source, state):
p = _parse_sub_cond(source, state, condgroup) p = _parse_sub_cond(source, state, condgroup)
else: else:
p = _parse_sub(source, state) p = _parse_sub(source, state)
if not source.match(")"): if not sourcematch(")"):
raise error, "unbalanced parenthesis" raise error, "unbalanced parenthesis"
if group is not None: if group is not None:
state.closegroup(group) state.closegroup(group)
subpattern.append((SUBPATTERN, (group, p))) subpatternappend((SUBPATTERN, (group, p)))
else: else:
while 1: while 1:
char = source.get() char = sourceget()
if char is None: if char is None:
raise error, "unexpected end of pattern" raise error, "unexpected end of pattern"
if char == ")": if char == ")":
@ -635,14 +652,14 @@ def _parse(source, state):
raise error, "unknown extension" raise error, "unknown extension"
elif this == "^": elif this == "^":
subpattern.append((AT, AT_BEGINNING)) subpatternappend((AT, AT_BEGINNING))
elif this == "$": elif this == "$":
subpattern.append((AT, AT_END)) subpattern.append((AT, AT_END))
elif this and this[0] == "\\": elif this and this[0] == "\\":
code = _escape(source, this, state) code = _escape(source, this, state)
subpattern.append(code) subpatternappend(code)
else: else:
raise error, "parser error" raise error, "parser error"
@ -681,20 +698,21 @@ def parse_template(source, pattern):
# parse 're' replacement string into list of literals and # parse 're' replacement string into list of literals and
# group references # group references
s = Tokenizer(source) s = Tokenizer(source)
sget = s.get
p = [] p = []
a = p.append a = p.append
def literal(literal, p=p): def literal(literal, p=p, pappend=a):
if p and p[-1][0] is LITERAL: if p and p[-1][0] is LITERAL:
p[-1] = LITERAL, p[-1][1] + literal p[-1] = LITERAL, p[-1][1] + literal
else: else:
p.append((LITERAL, literal)) pappend((LITERAL, literal))
sep = source[:0] sep = source[:0]
if type(sep) is type(""): if type(sep) is type(""):
makechar = chr makechar = chr
else: else:
makechar = unichr makechar = unichr
while 1: while 1:
this = s.get() this = sget()
if this is None: if this is None:
break # end of replacement string break # end of replacement string
if this and this[0] == "\\": if this and this[0] == "\\":
@ -703,7 +721,7 @@ def parse_template(source, pattern):
name = "" name = ""
if s.match("<"): if s.match("<"):
while 1: while 1:
char = s.get() char = sget()
if char is None: if char is None:
raise error, "unterminated group name" raise error, "unterminated group name"
if char == ">": if char == ">":
@ -731,7 +749,7 @@ def parse_template(source, pattern):
code = MARK, group code = MARK, group
break break
elif s.next in OCTDIGITS: elif s.next in OCTDIGITS:
this = this + s.get() this = this + sget()
else: else:
break break
if not code: if not code:
@ -752,13 +770,14 @@ def parse_template(source, pattern):
# convert template to groups and literals lists # convert template to groups and literals lists
i = 0 i = 0
groups = [] groups = []
literals = [] groupsappend = groups.append
literals = [None] * len(p)
for c, s in p: for c, s in p:
if c is MARK: if c is MARK:
groups.append((i, s)) groupsappend((i, s))
literals.append(None) # literal[i] is already None
else: else:
literals.append(s) literals[i] = s
i = i + 1 i = i + 1
return groups, literals return groups, literals