New re version from AMK
This commit is contained in:
Guido van Rossum
parent
f3d729c8f9
commit
dfa6790bd6
126
Lib/re.py
126
Lib/re.py
|
|
@ -1,7 +1,5 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- mode: python -*-
|
||||
# $Id$
|
||||
|
||||
|
||||
import sys
|
||||
import string
|
||||
|
|
@ -17,6 +15,7 @@ from pcre import *
|
|||
# compilation flags
|
||||
|
||||
I = IGNORECASE
|
||||
L = LOCALE
|
||||
M = MULTILINE
|
||||
S = DOTALL
|
||||
X = VERBOSE
|
||||
|
|
@ -61,8 +60,25 @@ def split(pattern, string, maxsplit=0):
|
|||
pattern = _cachecompile(pattern)
|
||||
return pattern.split(string, maxsplit)
|
||||
|
||||
def escape(pattern):
|
||||
"Escape all non-alphanumeric characters in pattern."
|
||||
result = []
|
||||
alphanum=string.letters+'_'+string.digits
|
||||
for char in pattern:
|
||||
if char not in alphanum:
|
||||
result.append('\\')
|
||||
result.append(char)
|
||||
return string.join(result, '')
|
||||
|
||||
def compile(pattern, flags=0):
|
||||
"Compile a regular expression pattern, returning a RegexObject."
|
||||
groupindex={}
|
||||
code=pcre_compile(pattern, flags, groupindex)
|
||||
return RegexObject(pattern, flags, code, groupindex)
|
||||
|
||||
|
||||
#
|
||||
#
|
||||
# Class definitions
|
||||
#
|
||||
|
||||
class RegexObject:
|
||||
|
|
@ -71,31 +87,54 @@ class RegexObject:
|
|||
self.flags = flags
|
||||
self.pattern = pattern
|
||||
self.groupindex = groupindex
|
||||
def search(self, string, pos=0):
|
||||
regs = self.code.match(string, pos, 0)
|
||||
|
||||
def search(self, string, pos=0, endpos=None):
|
||||
"""Scan through string looking for a match to the pattern, returning
|
||||
a MatchObject instance, or None if no match was found."""
|
||||
|
||||
if endpos is None or endpos>len(string):
|
||||
endpos=len(string)
|
||||
if endpos<pos: endpos=pos
|
||||
regs = self.code.match(string, pos, endpos, 0)
|
||||
if regs is None:
|
||||
return None
|
||||
self.num_regs=len(regs)
|
||||
self._num_regs=len(regs)
|
||||
|
||||
return MatchObject(self,
|
||||
string,
|
||||
pos,
|
||||
pos, endpos,
|
||||
regs)
|
||||
|
||||
def match(self, string, pos=0):
|
||||
regs = self.code.match(string, pos, ANCHORED)
|
||||
def match(self, string, pos=0, endpos=None):
|
||||
"""Try to apply the pattern at the start of the string, returning
|
||||
a MatchObject instance, or None if no match was found."""
|
||||
|
||||
if endpos is None or endpos>len(string):
|
||||
endpos=len(string)
|
||||
if endpos<pos: endpos=pos
|
||||
regs = self.code.match(string, pos, endpos, ANCHORED)
|
||||
if regs is None:
|
||||
return None
|
||||
self.num_regs=len(regs)
|
||||
self._num_regs=len(regs)
|
||||
return MatchObject(self,
|
||||
string,
|
||||
pos,
|
||||
pos, endpos,
|
||||
regs)
|
||||
|
||||
def sub(self, repl, string, count=0):
|
||||
"""Return the string obtained by replacing the leftmost
|
||||
non-overlapping occurrences of the pattern in string by the
|
||||
replacement repl"""
|
||||
|
||||
return self.subn(repl, string, count)[0]
|
||||
|
||||
def subn(self, repl, source, count=0):
|
||||
"""Return a 2-tuple containing (new_string, number).
|
||||
new_string is the string obtained by replacing the leftmost
|
||||
non-overlapping occurrences of the pattern in string by the
|
||||
replacement repl. number is the number of substitutions that
|
||||
were made."""
|
||||
|
||||
if count < 0:
|
||||
raise error, "negative substitution count"
|
||||
if count == 0:
|
||||
|
|
@ -134,6 +173,9 @@ class RegexObject:
|
|||
return (string.join(results, ''), n)
|
||||
|
||||
def split(self, source, maxsplit=0):
|
||||
"""Split \var{string} by the occurrences of the pattern,
|
||||
returning a list containing the resulting substrings."""
|
||||
|
||||
if maxsplit < 0:
|
||||
raise error, "negative split count"
|
||||
if maxsplit == 0:
|
||||
|
|
@ -156,21 +198,39 @@ class RegexObject:
|
|||
pos = pos+1
|
||||
continue
|
||||
results.append(source[lastmatch:i])
|
||||
g = m.group()
|
||||
g = m.groups()
|
||||
if g:
|
||||
if type(g)==type( "" ): g = [g]
|
||||
results[len(results):] = list(g)
|
||||
pos = lastmatch = j
|
||||
results.append(source[lastmatch:])
|
||||
return results
|
||||
|
||||
# The following 3 functions were contributed by Mike Fletcher, and
|
||||
# allow pickling and unpickling of RegexObject instances.
|
||||
def __getinitargs__(self):
|
||||
return (None,None,None,None) # any 4 elements, to work around
|
||||
# problems with the
|
||||
# pickle/cPickle modules not yet
|
||||
# ignoring the __init__ function
|
||||
def __getstate__(self):
|
||||
return self.pattern, self.flags, self.groupindex
|
||||
def __setstate__(self, statetuple):
|
||||
self.pattern = statetuple[0]
|
||||
self.flags = statetuple[1]
|
||||
self.groupindex = statetuple[2]
|
||||
self.code = apply(pcre_compile, statetuple)
|
||||
|
||||
class MatchObject:
|
||||
def __init__(self, re, string, pos, regs):
|
||||
def __init__(self, re, string, pos, endpos, regs):
|
||||
self.re = re
|
||||
self.string = string
|
||||
self.pos = pos
|
||||
self.endpos = endpos
|
||||
self.regs = regs
|
||||
|
||||
def start(self, g):
|
||||
def start(self, g = 0):
|
||||
"Return the start of the substring matched by group g"
|
||||
if type(g) == type(''):
|
||||
try:
|
||||
g = self.re.groupindex[g]
|
||||
|
|
@ -178,7 +238,8 @@ class MatchObject:
|
|||
raise IndexError, ('group "' + g + '" is undefined')
|
||||
return self.regs[g][0]
|
||||
|
||||
def end(self, g):
|
||||
def end(self, g = 0):
|
||||
"Return the end of the substring matched by group g"
|
||||
if type(g) == type(''):
|
||||
try:
|
||||
g = self.re.groupindex[g]
|
||||
|
|
@ -186,7 +247,9 @@ class MatchObject:
|
|||
raise IndexError, ('group "' + g + '" is undefined')
|
||||
return self.regs[g][1]
|
||||
|
||||
def span(self, g):
|
||||
def span(self, g = 0):
|
||||
"""Return a tuple containing the start,end of the substring
|
||||
matched by group g"""
|
||||
if type(g) == type(''):
|
||||
try:
|
||||
g = self.re.groupindex[g]
|
||||
|
|
@ -194,12 +257,18 @@ class MatchObject:
|
|||
raise IndexError, ('group "' + g + '" is undefined')
|
||||
return self.regs[g]
|
||||
|
||||
def groups(self):
|
||||
"Return a tuple containing all subgroups of the match object"
|
||||
|
||||
# If _num_regs==1, we don't want to call self.group with an
|
||||
# empty tuple.
|
||||
if self.re._num_regs == 1: return ()
|
||||
return apply(self.group, tuple(range(1, self.re._num_regs) ) )
|
||||
|
||||
def group(self, *groups):
|
||||
"Return one or more groups of the match."
|
||||
if len(groups) == 0:
|
||||
groups = range(1, self.re.num_regs)
|
||||
use_all = 1
|
||||
else:
|
||||
use_all = 0
|
||||
groups = (0,)
|
||||
result = []
|
||||
for g in groups:
|
||||
if type(g) == type(''):
|
||||
|
|
@ -212,25 +281,10 @@ class MatchObject:
|
|||
result.append(None)
|
||||
else:
|
||||
result.append(self.string[self.regs[g][0]:self.regs[g][1]])
|
||||
if use_all or len(result) > 1:
|
||||
if len(result) > 1:
|
||||
return tuple(result)
|
||||
elif len(result) == 1:
|
||||
return result[0]
|
||||
else:
|
||||
return ()
|
||||
|
||||
def escape(pattern):
|
||||
result = []
|
||||
alphanum=string.letters+'_'+string.digits
|
||||
for char in pattern:
|
||||
if char not in alphanum:
|
||||
result.append('\\')
|
||||
result.append(char)
|
||||
return string.join(result, '')
|
||||
|
||||
def compile(pattern, flags=0):
|
||||
groupindex={}
|
||||
code=pcre_compile(pattern, flags, groupindex)
|
||||
return RegexObject(pattern, flags, code, groupindex)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# -*- mode: python -*-
|
||||
# $Id$
|
||||
|
||||
# Re test suite and benchmark suite v1.5a2
|
||||
# Re test suite and benchmark suite v1.5b2
|
||||
|
||||
# The 3 possible outcomes for each pattern
|
||||
[SUCCEED, FAIL, SYNTAX_ERROR] = range(3)
|
||||
|
|
@ -48,6 +48,61 @@ benchmarks = [
|
|||
# If the regex isn't expected to work, the latter two elements can be omitted.
|
||||
|
||||
tests = [
|
||||
# Test ?P< and ?P= extensions
|
||||
('(?P<foo_123', '', SYNTAX_ERROR), # Unterminated group identifier
|
||||
('(?P<1>a)', '', SYNTAX_ERROR), # Begins with a digit
|
||||
('(?P<!>a)', '', SYNTAX_ERROR), # Begins with an illegal char
|
||||
('(?P<foo!>a)', '', SYNTAX_ERROR), # Begins with an illegal char
|
||||
|
||||
# Same tests, for the ?P= form
|
||||
('(?P<foo_123>a)(?P=foo_123', 'aa', SYNTAX_ERROR),
|
||||
('(?P<foo_123>a)(?P=1)', 'aa', SYNTAX_ERROR),
|
||||
('(?P<foo_123>a)(?P=!)', 'aa', SYNTAX_ERROR),
|
||||
('(?P<foo_123>a)(?P=foo_124', 'aa', SYNTAX_ERROR), # Backref to undefined group
|
||||
|
||||
('(?P<foo_123>a)', 'a', SUCCEED, 'g1', 'a'),
|
||||
('(?P<foo_123>a)(?P=foo_123)', 'aa', SUCCEED, 'g1', 'a'),
|
||||
|
||||
# Test octal escapes
|
||||
('\\1', 'a', SYNTAX_ERROR),
|
||||
('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'),
|
||||
('\\141', 'a', SUCCEED, 'found', 'a'),
|
||||
('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'),
|
||||
|
||||
# Test that a literal \0 is handled everywhere
|
||||
('\0', '\0', SUCCEED, 'found', '\0'),
|
||||
(r'\0', '\0', SUCCEED, 'found', '\0'),
|
||||
('[\0a]', '\0', SUCCEED, 'found', '\0'),
|
||||
('[a\0]', '\0', SUCCEED, 'found', '\0'),
|
||||
('[^a\0]', '\0', FAIL),
|
||||
(r'[\0a]', '\0', SUCCEED, 'found', '\0'),
|
||||
(r'[a\0]', '\0', SUCCEED, 'found', '\0'),
|
||||
(r'[^a\0]', '\0', FAIL),
|
||||
|
||||
# Test various letter escapes
|
||||
(r'\a[\b]\f\n\r\t\v', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'),
|
||||
(r'[\a][\b][\f][\n][\r][\t][\v]', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'),
|
||||
(r'\u', '', SYNTAX_ERROR), # A Perl escape
|
||||
(r'\c\e\g\h\i\j\k\m\o\p\q\y\z', 'ceghijkmopqyz', SUCCEED, 'found', 'ceghijkmopqyz'),
|
||||
(r'\xff', '\377', SUCCEED, 'found', chr(255)),
|
||||
(r'\x00ffffffffffffff', '\377', SUCCEED, 'found', chr(255)),
|
||||
(r'\x00f', '\017', SUCCEED, 'found', chr(15)),
|
||||
(r'\x00fe', '\376', SUCCEED, 'found', chr(254)),
|
||||
|
||||
(r"^\w+=(\\[\000-\277]|[^\n\\])*", "SRC=eval.c g.c blah blah blah \\\\\n\tapes.c",
|
||||
SUCCEED, 'found', "SRC=eval.c g.c blah blah blah \\\\"),
|
||||
|
||||
# Test that . only matches \n in DOTALL mode
|
||||
('a.b', 'acb', SUCCEED, 'found', 'acb'),
|
||||
('a.b', 'a\nb', FAIL),
|
||||
('a.*b', 'acc\nccb', FAIL),
|
||||
('a.{4,5}b', 'acc\nccb', FAIL),
|
||||
('a.b', 'a\rb', SUCCEED, 'found', 'a\rb'),
|
||||
('a.b(?s)', 'a\nb', SUCCEED, 'found', 'a\nb'),
|
||||
('a.*(?s)b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'),
|
||||
('(?s)a.{4,5}b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'),
|
||||
('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'),
|
||||
|
||||
('abc', 'abc', SUCCEED, 'found', 'abc'),
|
||||
('abc', 'xbc', FAIL),
|
||||
('abc', 'axc', FAIL),
|
||||
|
|
@ -338,8 +393,9 @@ tests = [
|
|||
('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'),
|
||||
('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'),
|
||||
('[k]', 'ab', FAIL),
|
||||
##('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'),
|
||||
##('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'),
|
||||
# XXX
|
||||
# ('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'),
|
||||
# ('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'),
|
||||
('a[-]?c', 'ac', SUCCEED, 'found', 'ac'),
|
||||
('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
|
||||
('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
|
||||
|
|
@ -470,15 +526,14 @@ tests = [
|
|||
('(?i)(.*)c(.*)', 'ABCDE', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCDE-AB-DE'),
|
||||
('(?i)\\((.*), (.*)\\)', '(A, B)', SUCCEED, 'g2+"-"+g1', 'B-A'),
|
||||
('(?i)[k]', 'AB', FAIL),
|
||||
##('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'),
|
||||
##('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'),
|
||||
# ('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'),
|
||||
# ('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'),
|
||||
('(?i)a[-]?c', 'AC', SUCCEED, 'found', 'AC'),
|
||||
('(?i)(abc)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'),
|
||||
('(?i)([a-c]*)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'),
|
||||
# these zero-width assertions are not supported
|
||||
#('a(?!b).', 'abad', SUCCEED, 'found', 'ad'),
|
||||
#('a(?=d).', 'abad', SUCCEED, 'found', 'ad'),
|
||||
#('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'),
|
||||
('a(?!b).', 'abad', SUCCEED, 'found', 'ad'),
|
||||
('a(?=d).', 'abad', SUCCEED, 'found', 'ad'),
|
||||
('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'),
|
||||
('a(?:b|c|d)(.)', 'ace', SUCCEED, 'g1', 'e'),
|
||||
('a(?:b|c|d)*(.)', 'ace', SUCCEED, 'g1', 'e'),
|
||||
('a(?:b|c|d)+?(.)', 'ace', SUCCEED, 'g1', 'e'),
|
||||
|
|
@ -535,5 +590,5 @@ xyzabc
|
|||
(r'\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'),
|
||||
('\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'),
|
||||
(r'\t\n\v\r\f\a', '\t\n\v\r\f\a', SUCCEED, 'found', chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)),
|
||||
(r'[\t][\n][\v][\r][\f][\a][\A][\b][\B][\Z][\g]', '\t\n\v\r\f\aA\bBZg', SUCCEED, 'found', '\t\n\v\r\f\aA\bBZg'),
|
||||
(r'[\t][\n][\v][\r][\f][\b]', '\t\n\v\r\f\b', SUCCEED, 'found', '\t\n\v\r\f\b'),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
|
||||
from test_support import verbose, TestFailed
|
||||
import re
|
||||
import reop
|
||||
import sys, os, string, traceback
|
||||
|
||||
# Misc tests from Tim Peters' re.doc
|
||||
|
|
@ -29,6 +28,7 @@ try:
|
|||
assert re.sub('(.)', re.escape(s), 'x') == s
|
||||
assert re.sub('(.)', lambda m: s, 'x') == s
|
||||
|
||||
assert re.sub('(?P<a>x)', '\g<a>\g<a>', 'xx') == 'xxxx'
|
||||
assert re.sub('(?P<unk>x)', '\g<unk>\g<unk>', 'xx') == 'xxxx'
|
||||
|
||||
assert re.sub('a', r'\t\n\v\r\f\a\b\B\Z\a\A\w\W\s\S\d\D', 'a') == '\t\n\v\r\f\a\bBZ\aAwWsSdD'
|
||||
|
|
@ -98,13 +98,9 @@ try:
|
|||
assert re.subn("b+", "x", "bbbb BBBB") == ('x BBBB', 1)
|
||||
assert re.subn("b+", "x", "xyz") == ('xyz', 0)
|
||||
assert re.subn("b*", "x", "xyz") == ('xxxyxzx', 4)
|
||||
|
||||
except AssertionError:
|
||||
raise TestFailed, "re.subn"
|
||||
|
||||
if verbose:
|
||||
print 'Running tests on re.split'
|
||||
|
||||
try:
|
||||
assert re.split(":", ":a:b::c") == ['', 'a', 'b', '', 'c']
|
||||
assert re.split(":*", ":a:b::c") == ['', 'a', 'b', 'c']
|
||||
|
|
@ -119,12 +115,37 @@ try:
|
|||
except AssertionError:
|
||||
raise TestFailed, "re.split"
|
||||
|
||||
if verbose:
|
||||
print 'Pickling a RegexObject instance'
|
||||
import pickle
|
||||
pat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)')
|
||||
s = pickle.dumps(pat)
|
||||
pat = pickle.loads(s)
|
||||
|
||||
if verbose:
|
||||
print 'Running tests on re.split'
|
||||
|
||||
try:
|
||||
assert re.I == re.IGNORECASE
|
||||
assert re.L == re.LOCALE
|
||||
assert re.M == re.MULTILINE
|
||||
assert re.S == re.DOTALL
|
||||
assert re.X == re.VERBOSE
|
||||
except AssertionError:
|
||||
raise TestFailed, 're module constants'
|
||||
|
||||
for flags in [re.I, re.M, re.X, re.S]:
|
||||
try:
|
||||
r = re.compile('^pattern$', flags)
|
||||
except:
|
||||
print 'Exception raised on flag', flags
|
||||
|
||||
from re_tests import *
|
||||
if verbose:
|
||||
print 'Running re_tests test suite'
|
||||
else:
|
||||
# To save time, only run the first and last 10 tests
|
||||
tests = tests[:10] + tests[-10:]
|
||||
pass #tests = tests[:10] + tests[-10:]
|
||||
|
||||
for t in tests:
|
||||
sys.stdout.flush()
|
||||
|
|
@ -150,7 +171,7 @@ for t in tests:
|
|||
else:
|
||||
try:
|
||||
result=obj.search(s)
|
||||
except (re.error, reop.error), msg:
|
||||
except (re.error), msg:
|
||||
print '=== Unexpected exception', t, repr(msg)
|
||||
if outcome==SYNTAX_ERROR:
|
||||
# This should have been a syntax error; forget it.
|
||||
|
|
@ -190,9 +211,22 @@ for t in tests:
|
|||
else:
|
||||
print '=== Failed incorrectly', t
|
||||
|
||||
# Try the match with the search area limited to the extent
|
||||
# of the match and see if it still succeeds. \B will
|
||||
# break (because it won't match at the end or start of a
|
||||
# string), so we'll ignore patterns that feature it.
|
||||
|
||||
if pattern[:2]!='\\B' and pattern[-2:]!='\\B':
|
||||
obj=re.compile(pattern)
|
||||
result=obj.search(s, pos=result.start(0), endpos=result.end(0)+1)
|
||||
if result==None:
|
||||
print '=== Failed on range-limited match', t
|
||||
|
||||
# Try the match with IGNORECASE enabled, and check that it
|
||||
# still succeeds.
|
||||
obj=re.compile(pattern, re.IGNORECASE)
|
||||
result=obj.search(s)
|
||||
if result==None:
|
||||
print '=== Fails on case-insensitive match', t
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue