traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue #7462: Implement the stringlib fast search algorithm for the `rfind`, 

`rindex`, `rsplit` and `rpartition` methods.  Patch by Florent Xicluna.
This commit is contained in:
Antoine Pitrou 2010-01-02 21:12:58 +00:00
parent d3e323215c
commit 5b7139aab4
11 changed files with 148 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
Lib/test/string_tests.py
View File

@ -230,6 +230,31 @@ class CommonTest(unittest.TestCase):
self.checkraises(TypeError, 'hello', 'rfind')
self.checkraises(TypeError, 'hello', 'rfind', 42)
# For a variety of combinations,
# verify that str.rfind() matches __contains__
# and that the found substring is really at that location
charset = ['', 'a', 'b', 'c']
digits = 5
base = len(charset)
teststrings = set()
for i in xrange(base ** digits):
entry = []
for j in xrange(digits):
i, m = divmod(i, base)
entry.append(charset[m])
teststrings.add(''.join(entry))
teststrings = list(teststrings)
for i in teststrings:
i = self.fixtype(i)
for j in teststrings:
loc = i.rfind(j)
r1 = (loc != -1)
r2 = j in i
if r1 != r2:
self.assertEqual(r1, r2)
if loc != -1:
self.assertEqual(i[loc:loc+len(j)], j)
def test_index(self):
self.checkequal(0, 'abcdefghiabc', 'index', '')
self.checkequal(3, 'abcdefghiabc', 'index', 'def')
@ -686,8 +711,10 @@ class CommonTest(unittest.TestCase):
EQ("bobobXbobob", "bobobobXbobobob", "replace", "bobob", "bob")
EQ("BOBOBOB", "BOBOBOB", "replace", "bob", "bobby")
ba = buffer('a')
bb = buffer('b')
# Silence Py3k warning
with test_support.check_warnings():
ba = buffer('a')
bb = buffer('b')
EQ("bbc", "abc", "replace", ba, bb)
EQ("aac", "abc", "replace", bb, ba)

5
Lib/test/test_unicode.py
View File

@ -499,9 +499,12 @@ class UnicodeTest(
)
if not sys.platform.startswith('java'):
# Silence Py3k warning
with test_support.check_warnings():
buf = buffer('character buffers are decoded to unicode')
self.assertEqual(
unicode(
buffer('character buffers are decoded to unicode'),
buf,
'utf-8',
'strict'
),

3
Misc/NEWS
View File

@ -12,6 +12,9 @@ What's New in Python 2.7 alpha 2?
Core and Builtins
-----------------
- Issue #7462: Implement the stringlib fast search algorithm for the `rfind`,
`rindex`, `rsplit` and `rpartition` methods. Patch by Florent Xicluna.
- Issue #5080: A number of functions and methods previously produced a
DeprecationWarning when passed a float argument where an integer was
expected. These functions and methods now raise TypeError instead.

37
Objects/bytearrayobject.c
View File

@ -1111,7 +1111,6 @@ bytearray_dealloc(PyByteArrayObject *self)
/* Methods */
#define STRINGLIB_CHAR char
#define STRINGLIB_CMP memcmp
#define STRINGLIB_LEN PyByteArray_GET_SIZE
#define STRINGLIB_STR PyByteArray_AS_STRING
#define STRINGLIB_NEW PyByteArray_FromStringAndSize
@ -2282,14 +2281,11 @@ If maxsplit is given, at most maxsplit splits are done.");
static PyObject *
bytearray_split(PyByteArrayObject *self, PyObject *args)
{
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j;
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j, pos;
Py_ssize_t maxsplit = -1, count = 0;
const char *s = PyByteArray_AS_STRING(self), *sub;
PyObject *list, *str, *subobj = Py_None;
Py_buffer vsub;
#ifdef USE_FAST
Py_ssize_t pos;
#endif
if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit))
return NULL;
@ -2321,7 +2317,6 @@ bytearray_split(PyByteArrayObject *self, PyObject *args)
return NULL;
}
#ifdef USE_FAST
i = j = 0;
while (maxsplit-- > 0) {
pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH);
@ -2331,18 +2326,6 @@ bytearray_split(PyByteArrayObject *self, PyObject *args)
SPLIT_ADD(s, i, j);
i = j + n;
}
#else
i = j = 0;
while ((j+n <= len) && (maxsplit-- > 0)) {
for (; j+n <= len; j++) {
if (Py_STRING_MATCH(s, j, sub, n)) {
SPLIT_ADD(s, i, j);
i = j = j + n;
break;
}
}
}
#endif
SPLIT_ADD(s, i, len);
FIX_PREALLOC_SIZE(list);
PyBuffer_Release(&vsub);
@ -2520,7 +2503,7 @@ If maxsplit is given, at most maxsplit splits are done.");
static PyObject *
bytearray_rsplit(PyByteArrayObject *self, PyObject *args)
{
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j;
Py_ssize_t len = PyByteArray_GET_SIZE(self), n, j, pos;
Py_ssize_t maxsplit = -1, count = 0;
const char *s = PyByteArray_AS_STRING(self), *sub;
PyObject *list, *str, *subobj = Py_None;
@ -2557,17 +2540,13 @@ bytearray_rsplit(PyByteArrayObject *self, PyObject *args)
}
j = len;
i = j - n;
while ( (i >= 0) && (maxsplit-- > 0) ) {
for (; i>=0; i--) {
if (Py_STRING_MATCH(s, i, sub, n)) {
SPLIT_ADD(s, i + n, j);
j = i;
i -= n;
break;
}
}
while (maxsplit-- > 0) {
pos = fastsearch(s, j, sub, n, FAST_RSEARCH);
if (pos < 0)
break;
SPLIT_ADD(s, pos + n, j);
j = pos;
}
SPLIT_ADD(s, 0, j);
FIX_PREALLOC_SIZE(list);

4
Objects/stringlib/README.txt
View File

@ -15,10 +15,6 @@ STRINGLIB_EMPTY
a PyObject representing the empty string
int STRINGLIB_CMP(STRINGLIB_CHAR*, STRINGLIB_CHAR*, Py_ssize_t)
compares two strings. returns 0 if they match, and non-zero if not.
Py_ssize_t STRINGLIB_LEN(PyObject*)
returns the length of the given string object (which must be of the

112
Objects/stringlib/fastsearch.h
View File

@ -5,7 +5,7 @@
/* fast search/count implementation, based on a mix between boyer-
moore and horspool, with a few more bells and whistles on the top.
for some more background, see: http://effbot.org/stringlib.htm */
for some more background, see: http://effbot.org/zone/stringlib.htm */
/* note: fastsearch may access s[n], which isn't a problem when using
Python's ordinary string types, but may cause problems if you're
@ -16,6 +16,7 @@
#define FAST_COUNT 0
#define FAST_SEARCH 1
#define FAST_RSEARCH 2
Py_LOCAL_INLINE(Py_ssize_t)
fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
@ -41,51 +42,92 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
if (s[i] == p[0])
count++;
return count;
} else {
} else if (mode == FAST_SEARCH) {
for (i = 0; i < n; i++)
if (s[i] == p[0])
return i;
} else { /* FAST_RSEARCH */
for (i = n - 1; i > -1; i--)
if (s[i] == p[0])
return i;
}
return -1;
}
mlast = m - 1;
/* create compressed boyer-moore delta 1 table */
skip = mlast - 1;
/* process pattern[:-1] */
for (mask = i = 0; i < mlast; i++) {
mask |= (1 << (p[i] & 0x1F));
if (p[i] == p[mlast])
skip = mlast - i - 1;
}
/* process pattern[-1] outside the loop */
mask |= (1 << (p[mlast] & 0x1F));
for (i = 0; i <= w; i++) {
/* note: using mlast in the skip path slows things down on x86 */
if (s[i+m-1] == p[m-1]) {
/* candidate match */
for (j = 0; j < mlast; j++)
if (s[i+j] != p[j])
break;
if (j == mlast) {
/* got a match! */
if (mode != FAST_COUNT)
return i;
count++;
i = i + mlast;
continue;
if (mode != FAST_RSEARCH) {
/* create compressed boyer-moore delta 1 table */
/* process pattern[:-1] */
for (mask = i = 0; i < mlast; i++) {
mask |= (1 << (p[i] & 0x1F));
if (p[i] == p[mlast])
skip = mlast - i - 1;
}
/* process pattern[-1] outside the loop */
mask |= (1 << (p[mlast] & 0x1F));
for (i = 0; i <= w; i++) {
/* note: using mlast in the skip path slows things down on x86 */
if (s[i+m-1] == p[m-1]) {
/* candidate match */
for (j = 0; j < mlast; j++)
if (s[i+j] != p[j])
break;
if (j == mlast) {
/* got a match! */
if (mode != FAST_COUNT)
return i;
count++;
i = i + mlast;
continue;
}
/* miss: check if next character is part of pattern */
if (!(mask & (1 << (s[i+m] & 0x1F))))
i = i + m;
else
i = i + skip;
} else {
/* skip: check if next character is part of pattern */
if (!(mask & (1 << (s[i+m] & 0x1F))))
i = i + m;
}
}
} else { /* FAST_RSEARCH */
/* create compressed boyer-moore delta 1 table */
/* process pattern[0] outside the loop */
mask = (1 << (p[0] & 0x1F));
/* process pattern[:0:-1] */
for (i = mlast; i > 0; i--) {
mask |= (1 << (p[i] & 0x1F));
if (p[i] == p[0])
skip = i - 1;
}
for (i = w; i >= 0; i--) {
if (s[i] == p[0]) {
/* candidate match */
for (j = mlast; j > 0; j--)
if (s[i+j] != p[j])
break;
if (j == 0)
/* got a match! */
return i;
/* miss: check if previous character is part of pattern */
if (!(mask & (1 << (s[i-1] & 0x1F))))
i = i - m;
else
i = i - skip;
} else {
/* skip: check if previous character is part of pattern */
if (!(mask & (1 << (s[i-1] & 0x1F))))
i = i - m;
}
/* miss: check if next character is part of pattern */
if (!(mask & (1 << (s[i+m] & 0x1F))))
i = i + m;
else
i = i + skip;
} else {
/* skip: check if next character is part of pattern */
if (!(mask & (1 << (s[i+m] & 0x1F))))
i = i + m;
}
}

32
Objects/stringlib/find.h
View File

@ -32,20 +32,19 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
Py_ssize_t offset)
{
/* XXX - create reversefastsearch helper! */
if (sub_len == 0) {
if (str_len < 0)
return -1;
return str_len + offset;
} else {
Py_ssize_t j, pos = -1;
for (j = str_len - sub_len; j >= 0; --j)
if (STRINGLIB_CMP(str+j, sub, sub_len) == 0) {
pos = j + offset;
break;
}
return pos;
}
Py_ssize_t pos;
if (str_len < 0)
return -1;
if (sub_len == 0)
return str_len + offset;
pos = fastsearch(str, str_len, sub, sub_len, FAST_RSEARCH);
if (pos >= 0)
pos += offset;
return pos;
}
Py_LOCAL_INLINE(Py_ssize_t)
@ -64,10 +63,7 @@ stringlib_find_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
if (end < 0)
end = 0;
return stringlib_find(
str + start, end - start,
sub, sub_len, start
);
return stringlib_find(str + start, end - start, sub, sub_len, start);
}
Py_LOCAL_INLINE(Py_ssize_t)

12
Objects/stringlib/partition.h
View File

@ -58,7 +58,7 @@ stringlib_rpartition(
)
{
PyObject* out;
Py_ssize_t pos, j;
Py_ssize_t pos;
if (sep_len == 0) {
PyErr_SetString(PyExc_ValueError, "empty separator");
@ -69,20 +69,14 @@ stringlib_rpartition(
if (!out)
return NULL;
/* XXX - create reversefastsearch helper! */
pos = -1;
for (j = str_len - sep_len; j >= 0; --j)
if (STRINGLIB_CMP(str+j, sep, sep_len) == 0) {
pos = j;
break;
}
pos = fastsearch(str, str_len, sep, sep_len, FAST_RSEARCH);
if (pos < 0) {
Py_INCREF(STRINGLIB_EMPTY);
PyTuple_SET_ITEM(out, 0, (PyObject*) STRINGLIB_EMPTY);
Py_INCREF(STRINGLIB_EMPTY);
PyTuple_SET_ITEM(out, 1, (PyObject*) STRINGLIB_EMPTY);
Py_INCREF(str_obj);
Py_INCREF(str_obj);
PyTuple_SET_ITEM(out, 2, (PyObject*) str_obj);
return out;
}

1
Objects/stringlib/stringdefs.h
View File

@ -21,7 +21,6 @@
#define STRINGLIB_NEW PyString_FromStringAndSize
#define STRINGLIB_RESIZE _PyString_Resize
#define STRINGLIB_CHECK PyString_Check
#define STRINGLIB_CMP memcmp
#define STRINGLIB_TOSTR PyObject_Str
#define STRINGLIB_GROUPING _PyString_InsertThousandsGrouping
#define STRINGLIB_GROUPING_LOCALE _PyString_InsertThousandsGroupingLocale

19
Objects/stringlib/unicodedefs.h
View File

@ -31,23 +31,4 @@
#define STRINGLIB_WANT_CONTAINS_OBJ 1
/* STRINGLIB_CMP was defined as:
Py_LOCAL_INLINE(int)
STRINGLIB_CMP(const Py_UNICODE* str, const Py_UNICODE* other, Py_ssize_t len)
{
if (str[0] != other[0])
return 1;
return memcmp((void*) str, (void*) other, len * sizeof(Py_UNICODE));
}
but unfortunately that gives a error if the function isn't used in a file that
includes this file. So, reluctantly convert it to a macro instead. */
#define STRINGLIB_CMP(str, other, len) \
(((str)[0] != (other)[0]) ? \
1 : \
memcmp((void*) (str), (void*) (other), (len) * sizeof(Py_UNICODE)))
#endif /* !STRINGLIB_UNICODEDEFS_H */

41
Objects/stringobject.c
View File

@ -1576,13 +1576,10 @@ from the result.");
static PyObject *
string_split(PyStringObject *self, PyObject *args)
{
Py_ssize_t len = PyString_GET_SIZE(self), n, i, j;
Py_ssize_t len = PyString_GET_SIZE(self), n, i, j, pos;
Py_ssize_t maxsplit = -1, count=0;
const char *s = PyString_AS_STRING(self), *sub;
PyObject *list, *str, *subobj = Py_None;
#ifdef USE_FAST
Py_ssize_t pos;
#endif
if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit))
return NULL;
@ -1612,28 +1609,15 @@ string_split(PyStringObject *self, PyObject *args)
if (list == NULL)
return NULL;
#ifdef USE_FAST
i = j = 0;
while (maxsplit-- > 0) {
pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH);
if (pos < 0)
break;
j = i+pos;
j = i + pos;
SPLIT_ADD(s, i, j);
i = j + n;
}
#else
i = j = 0;
while ((j+n <= len) && (maxsplit-- > 0)) {
for (; j+n <= len; j++) {
if (Py_STRING_MATCH(s, j, sub, n)) {
SPLIT_ADD(s, i, j);
i = j = j + n;
break;
}
}
}
#endif
SPLIT_ADD(s, i, len);
FIX_PREALLOC_SIZE(list);
return list;
@ -1801,9 +1785,9 @@ is a separator.");
static PyObject *
string_rsplit(PyStringObject *self, PyObject *args)
{
Py_ssize_t len = PyString_GET_SIZE(self), n, i, j;
Py_ssize_t len = PyString_GET_SIZE(self), n, j, pos;
Py_ssize_t maxsplit = -1, count=0;
const char *s, *sub;
const char *s = PyString_AS_STRING(self), *sub;
PyObject *list, *str, *subobj = Py_None;
if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit))
@ -1835,18 +1819,13 @@ string_rsplit(PyStringObject *self, PyObject *args)
return NULL;
j = len;
i = j - n;
s = PyString_AS_STRING(self);
while ( (i >= 0) && (maxsplit-- > 0) ) {
for (; i>=0; i--) {
if (Py_STRING_MATCH(s, i, sub, n)) {
SPLIT_ADD(s, i + n, j);
j = i;
i -= n;
break;
}
}
while (maxsplit-- > 0) {
pos = fastsearch(s, j, sub, n, FAST_RSEARCH);
if (pos < 0)
break;
SPLIT_ADD(s, pos + n, j);
j = pos;
}
SPLIT_ADD(s, 0, j);
FIX_PREALLOC_SIZE(list);