Issue #7462: Implement the stringlib fast search algorithm for the rfind,

`rindex`, `rsplit` and `rpartition` methods.  Patch by Florent Xicluna.

Lib/test/string_tests.py

@@ -230,6 +230,31 @@ class CommonTest(unittest.TestCase):
         self.checkraises(TypeError, 'hello', 'rfind')
         self.checkraises(TypeError, 'hello', 'rfind', 42)
 
+        # For a variety of combinations,
+        #    verify that str.rfind() matches __contains__
+        #    and that the found substring is really at that location
+        charset = ['', 'a', 'b', 'c']
+        digits = 5
+        base = len(charset)
+        teststrings = set()
+        for i in xrange(base ** digits):
+            entry = []
+            for j in xrange(digits):
+                i, m = divmod(i, base)
+                entry.append(charset[m])
+            teststrings.add(''.join(entry))
+        teststrings = list(teststrings)
+        for i in teststrings:
+            i = self.fixtype(i)
+            for j in teststrings:
+                loc = i.rfind(j)
+                r1 = (loc != -1)
+                r2 = j in i
+                if r1 != r2:
+                    self.assertEqual(r1, r2)
+                if loc != -1:
+                    self.assertEqual(i[loc:loc+len(j)], j)
+
     def test_index(self):
         self.checkequal(0, 'abcdefghiabc', 'index', '')
         self.checkequal(3, 'abcdefghiabc', 'index', 'def')
@@ -686,8 +711,10 @@ class CommonTest(unittest.TestCase):
         EQ("bobobXbobob", "bobobobXbobobob", "replace", "bobob", "bob")
         EQ("BOBOBOB", "BOBOBOB", "replace", "bob", "bobby")
 
-        ba = buffer('a')
+        # Silence Py3k warning
-        bb = buffer('b')
+        with test_support.check_warnings():
+            ba = buffer('a')
+            bb = buffer('b')
         EQ("bbc", "abc", "replace", ba, bb)
         EQ("aac", "abc", "replace", bb, ba)
 

Lib/test/test_unicode.py

@@ -499,9 +499,12 @@ class UnicodeTest(
         )
 
         if not sys.platform.startswith('java'):
+            # Silence Py3k warning
+            with test_support.check_warnings():
+                buf = buffer('character buffers are decoded to unicode')
             self.assertEqual(
                 unicode(
-                    buffer('character buffers are decoded to unicode'),
+                    buf,
                     'utf-8',
                     'strict'
                 ),

Misc/NEWS

@@ -12,6 +12,9 @@ What's New in Python 2.7 alpha 2?
 Core and Builtins
 -----------------
 
+- Issue #7462: Implement the stringlib fast search algorithm for the `rfind`,
+  `rindex`, `rsplit` and `rpartition` methods.  Patch by Florent Xicluna.
+
 - Issue #5080: A number of functions and methods previously produced a
   DeprecationWarning when passed a float argument where an integer was
   expected.  These functions and methods now raise TypeError instead.

Objects/bytearrayobject.c

@@ -1111,7 +1111,6 @@ bytearray_dealloc(PyByteArrayObject *self)
 /* Methods */
 
 #define STRINGLIB_CHAR char
-#define STRINGLIB_CMP memcmp
 #define STRINGLIB_LEN PyByteArray_GET_SIZE
 #define STRINGLIB_STR PyByteArray_AS_STRING
 #define STRINGLIB_NEW PyByteArray_FromStringAndSize
@@ -2282,14 +2281,11 @@ If maxsplit is given, at most maxsplit splits are done.");
 static PyObject *
 bytearray_split(PyByteArrayObject *self, PyObject *args)
 {
-    Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j;
+    Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j, pos;
     Py_ssize_t maxsplit = -1, count = 0;
     const char *s = PyByteArray_AS_STRING(self), *sub;
     PyObject *list, *str, *subobj = Py_None;
     Py_buffer vsub;
-#ifdef USE_FAST
-    Py_ssize_t pos;
-#endif
 
     if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit))
         return NULL;
@@ -2321,7 +2317,6 @@ bytearray_split(PyByteArrayObject *self, PyObject *args)
         return NULL;
     }
 
-#ifdef USE_FAST
     i = j = 0;
     while (maxsplit-- > 0) {
         pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH);
@@ -2331,18 +2326,6 @@ bytearray_split(PyByteArrayObject *self, PyObject *args)
         SPLIT_ADD(s, i, j);
         i = j + n;
     }
-#else
-    i = j = 0;
-    while ((j+n <= len) && (maxsplit-- > 0)) {
-        for (; j+n <= len; j++) {
-            if (Py_STRING_MATCH(s, j, sub, n)) {
-                SPLIT_ADD(s, i, j);
-                i = j = j + n;
-                break;
-            }
-        }
-    }
-#endif
     SPLIT_ADD(s, i, len);
     FIX_PREALLOC_SIZE(list);
     PyBuffer_Release(&vsub);
@@ -2520,7 +2503,7 @@ If maxsplit is given, at most maxsplit splits are done.");
 static PyObject *
 bytearray_rsplit(PyByteArrayObject *self, PyObject *args)
 {
-    Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j;
+    Py_ssize_t len = PyByteArray_GET_SIZE(self), n, j, pos;
     Py_ssize_t maxsplit = -1, count = 0;
     const char *s = PyByteArray_AS_STRING(self), *sub;
     PyObject *list, *str, *subobj = Py_None;
@@ -2557,17 +2540,13 @@ bytearray_rsplit(PyByteArrayObject *self, PyObject *args)
     }
 
     j = len;
-    i = j - n;
 
-    while ( (i >= 0) && (maxsplit-- > 0) ) {
+    while (maxsplit-- > 0) {
-        for (; i>=0; i--) {
+       pos = fastsearch(s, j, sub, n, FAST_RSEARCH);
-            if (Py_STRING_MATCH(s, i, sub, n)) {
+       if (pos < 0)
-                SPLIT_ADD(s, i + n, j);
+           break;
-                j = i;
+       SPLIT_ADD(s, pos + n, j);
-                i -= n;
+       j = pos;
-                break;
-            }
-        }
     }
     SPLIT_ADD(s, 0, j);
     FIX_PREALLOC_SIZE(list);

Objects/stringlib/README.txt

@@ -15,10 +15,6 @@ STRINGLIB_EMPTY
 
     a PyObject representing the empty string
 
-int STRINGLIB_CMP(STRINGLIB_CHAR*, STRINGLIB_CHAR*, Py_ssize_t)
-
-    compares two strings. returns 0 if they match, and non-zero if not.
-
 Py_ssize_t STRINGLIB_LEN(PyObject*)
 
     returns the length of the given string object (which must be of the

Objects/stringlib/fastsearch.h

@@ -5,7 +5,7 @@
 
 /* fast search/count implementation, based on a mix between boyer-
    moore and horspool, with a few more bells and whistles on the top.
-   for some more background, see: http://effbot.org/stringlib.htm */
+   for some more background, see: http://effbot.org/zone/stringlib.htm */
 
 /* note: fastsearch may access s[n], which isn't a problem when using
    Python's ordinary string types, but may cause problems if you're
@@ -16,6 +16,7 @@
 
 #define FAST_COUNT 0
 #define FAST_SEARCH 1
+#define FAST_RSEARCH 2
 
 Py_LOCAL_INLINE(Py_ssize_t)
 fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
@@ -41,51 +42,92 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
                 if (s[i] == p[0])
                     count++;
             return count;
-        } else {
+        } else if (mode == FAST_SEARCH) {
             for (i = 0; i < n; i++)
                 if (s[i] == p[0])
                     return i;
+        } else {    /* FAST_RSEARCH */
+            for (i = n - 1; i > -1; i--)
+                if (s[i] == p[0])
+                    return i;
         }
         return -1;
     }
 
     mlast = m - 1;
-
-    /* create compressed boyer-moore delta 1 table */
     skip = mlast - 1;
-    /* process pattern[:-1] */
-    for (mask = i = 0; i < mlast; i++) {
-        mask |= (1 << (p[i] & 0x1F));
-        if (p[i] == p[mlast])
-            skip = mlast - i - 1;
-    }
-    /* process pattern[-1] outside the loop */
-    mask |= (1 << (p[mlast] & 0x1F));
 
-    for (i = 0; i <= w; i++) {
+    if (mode != FAST_RSEARCH) {
-        /* note: using mlast in the skip path slows things down on x86 */
+
-        if (s[i+m-1] == p[m-1]) {
+        /* create compressed boyer-moore delta 1 table */
-            /* candidate match */
+
-            for (j = 0; j < mlast; j++)
+        /* process pattern[:-1] */
-                if (s[i+j] != p[j])
+        for (mask = i = 0; i < mlast; i++) {
-                    break;
+            mask |= (1 << (p[i] & 0x1F));
-            if (j == mlast) {
+            if (p[i] == p[mlast])
-                /* got a match! */
+                skip = mlast - i - 1;
-                if (mode != FAST_COUNT)
+        }
-                    return i;
+        /* process pattern[-1] outside the loop */
-                count++;
+        mask |= (1 << (p[mlast] & 0x1F));
-                i = i + mlast;
+
-                continue;
+        for (i = 0; i <= w; i++) {
+            /* note: using mlast in the skip path slows things down on x86 */
+            if (s[i+m-1] == p[m-1]) {
+                /* candidate match */
+                for (j = 0; j < mlast; j++)
+                    if (s[i+j] != p[j])
+                        break;
+                if (j == mlast) {
+                    /* got a match! */
+                    if (mode != FAST_COUNT)
+                        return i;
+                    count++;
+                    i = i + mlast;
+                    continue;
+                }
+                /* miss: check if next character is part of pattern */
+                if (!(mask & (1 << (s[i+m] & 0x1F))))
+                    i = i + m;
+                else
+                    i = i + skip;
+            } else {
+                /* skip: check if next character is part of pattern */
+                if (!(mask & (1 << (s[i+m] & 0x1F))))
+                    i = i + m;
+            }
+        }
+    } else {    /* FAST_RSEARCH */
+
+        /* create compressed boyer-moore delta 1 table */
+
+        /* process pattern[0] outside the loop */
+        mask = (1 << (p[0] & 0x1F));
+        /* process pattern[:0:-1] */
+        for (i = mlast; i > 0; i--) {
+            mask |= (1 << (p[i] & 0x1F));
+            if (p[i] == p[0])
+                skip = i - 1;
+        }
+
+        for (i = w; i >= 0; i--) {
+            if (s[i] == p[0]) {
+                /* candidate match */
+                for (j = mlast; j > 0; j--)
+                    if (s[i+j] != p[j])
+                        break;
+                if (j == 0)
+                    /* got a match! */
+                    return i;
+                /* miss: check if previous character is part of pattern */
+                if (!(mask & (1 << (s[i-1] & 0x1F))))
+                    i = i - m;
+                else
+                    i = i - skip;
+            } else {
+                /* skip: check if previous character is part of pattern */
+                if (!(mask & (1 << (s[i-1] & 0x1F))))
+                    i = i - m;
             }
-            /* miss: check if next character is part of pattern */
-            if (!(mask & (1 << (s[i+m] & 0x1F))))
-                i = i + m;
-            else
-                i = i + skip;
-        } else {
-            /* skip: check if next character is part of pattern */
-            if (!(mask & (1 << (s[i+m] & 0x1F))))
-                i = i + m;
         }
     }
 

Objects/stringlib/find.h

@@ -32,20 +32,19 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
                 const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
                 Py_ssize_t offset)
 {
-    /* XXX - create reversefastsearch helper! */
+    Py_ssize_t pos;
-    if (sub_len == 0) {
+
-        if (str_len < 0)
+    if (str_len < 0)
-            return -1;
+        return -1;
-	return str_len + offset;
+    if (sub_len == 0)
-    } else {
+        return str_len + offset;
-	Py_ssize_t j, pos = -1;
+
-	for (j = str_len - sub_len; j >= 0; --j)
+    pos = fastsearch(str, str_len, sub, sub_len, FAST_RSEARCH);
-            if (STRINGLIB_CMP(str+j, sub, sub_len) == 0) {
+
-                pos = j + offset;
+    if (pos >= 0)
-                break;
+        pos += offset;
-            }
+
-        return pos;
+    return pos;
-    }
 }
 
 Py_LOCAL_INLINE(Py_ssize_t)
@@ -64,10 +63,7 @@ stringlib_find_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
     if (end < 0)
         end = 0;
 
-    return stringlib_find(
+    return stringlib_find(str + start, end - start, sub, sub_len, start);
-        str + start, end - start,
-        sub, sub_len, start
-        );
 }
 
 Py_LOCAL_INLINE(Py_ssize_t)

Objects/stringlib/partition.h

@@ -58,7 +58,7 @@ stringlib_rpartition(
     )
 {
     PyObject* out;
-    Py_ssize_t pos, j;
+    Py_ssize_t pos;
 
     if (sep_len == 0) {
         PyErr_SetString(PyExc_ValueError, "empty separator");
@@ -69,20 +69,14 @@ stringlib_rpartition(
     if (!out)
 	return NULL;
 
-    /* XXX - create reversefastsearch helper! */
+    pos = fastsearch(str, str_len, sep, sep_len, FAST_RSEARCH);
-        pos = -1;
-	for (j = str_len - sep_len; j >= 0; --j)
-            if (STRINGLIB_CMP(str+j, sep, sep_len) == 0) {
-                pos = j;
-                break;
-            }
 
     if (pos < 0) {
 	Py_INCREF(STRINGLIB_EMPTY);
 	PyTuple_SET_ITEM(out, 0, (PyObject*) STRINGLIB_EMPTY);
 	Py_INCREF(STRINGLIB_EMPTY);
 	PyTuple_SET_ITEM(out, 1, (PyObject*) STRINGLIB_EMPTY);
-	Py_INCREF(str_obj);        
+	Py_INCREF(str_obj);
 	PyTuple_SET_ITEM(out, 2, (PyObject*) str_obj);
 	return out;
     }

Objects/stringlib/stringdefs.h

@@ -21,7 +21,6 @@
 #define STRINGLIB_NEW            PyString_FromStringAndSize
 #define STRINGLIB_RESIZE         _PyString_Resize
 #define STRINGLIB_CHECK          PyString_Check
-#define STRINGLIB_CMP            memcmp
 #define STRINGLIB_TOSTR          PyObject_Str
 #define STRINGLIB_GROUPING       _PyString_InsertThousandsGrouping
 #define STRINGLIB_GROUPING_LOCALE _PyString_InsertThousandsGroupingLocale

Objects/stringlib/unicodedefs.h

@@ -31,23 +31,4 @@
 
 #define STRINGLIB_WANT_CONTAINS_OBJ 1
 
-/* STRINGLIB_CMP was defined as:
-
-Py_LOCAL_INLINE(int)
-STRINGLIB_CMP(const Py_UNICODE* str, const Py_UNICODE* other, Py_ssize_t len)
-{
-    if (str[0] != other[0])
-        return 1;
-    return memcmp((void*) str, (void*) other, len * sizeof(Py_UNICODE));
-}
-
-but unfortunately that gives a error if the function isn't used in a file that
-includes this file.  So, reluctantly convert it to a macro instead. */
-
-#define STRINGLIB_CMP(str, other, len) \
-    (((str)[0] != (other)[0]) ? \
-     1 : \
-     memcmp((void*) (str), (void*) (other), (len) * sizeof(Py_UNICODE)))
-
-
 #endif /* !STRINGLIB_UNICODEDEFS_H */

Objects/stringobject.c

@@ -1576,13 +1576,10 @@ from the result.");
 static PyObject *
 string_split(PyStringObject *self, PyObject *args)
 {
-	Py_ssize_t len = PyString_GET_SIZE(self), n, i, j;
+	Py_ssize_t len = PyString_GET_SIZE(self), n, i, j, pos;
 	Py_ssize_t maxsplit = -1, count=0;
 	const char *s = PyString_AS_STRING(self), *sub;
 	PyObject *list, *str, *subobj = Py_None;
-#ifdef USE_FAST
-	Py_ssize_t pos;
-#endif
 
 	if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit))
 		return NULL;
@@ -1612,28 +1609,15 @@ string_split(PyStringObject *self, PyObject *args)
 	if (list == NULL)
 		return NULL;
 
-#ifdef USE_FAST
 	i = j = 0;
 	while (maxsplit-- > 0) {
 		pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH);
 		if (pos < 0)
 			break;
-		j = i+pos;
+		j = i + pos;
 		SPLIT_ADD(s, i, j);
 		i = j + n;
 	}
-#else
-	i = j = 0;
-	while ((j+n <= len) && (maxsplit-- > 0)) {
-		for (; j+n <= len; j++) {
-			if (Py_STRING_MATCH(s, j, sub, n)) {
-				SPLIT_ADD(s, i, j);
-				i = j = j + n;
-				break;
-			}
-		}
-	}
-#endif
 	SPLIT_ADD(s, i, len);
 	FIX_PREALLOC_SIZE(list);
 	return list;
@@ -1801,9 +1785,9 @@ is a separator.");
 static PyObject *
 string_rsplit(PyStringObject *self, PyObject *args)
 {
-	Py_ssize_t len = PyString_GET_SIZE(self), n, i, j;
+	Py_ssize_t len = PyString_GET_SIZE(self), n, j, pos;
 	Py_ssize_t maxsplit = -1, count=0;
-	const char *s, *sub;
+	const char *s = PyString_AS_STRING(self), *sub;
 	PyObject *list, *str, *subobj = Py_None;
 
 	if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit))
@@ -1835,18 +1819,13 @@ string_rsplit(PyStringObject *self, PyObject *args)
 		return NULL;
 
 	j = len;
-	i = j - n;
 
-	s = PyString_AS_STRING(self);
+	while (maxsplit-- > 0) {
-	while ( (i >= 0) && (maxsplit-- > 0) ) {
+		pos = fastsearch(s, j, sub, n, FAST_RSEARCH);
-		for (; i>=0; i--) {
+		if (pos < 0)
-			if (Py_STRING_MATCH(s, i, sub, n)) {
+			break;
-				SPLIT_ADD(s, i + n, j);
+		SPLIT_ADD(s, pos + n, j);
-				j = i;
+		j = pos;
-				i -= n;
-				break;
-			}
-		}
 	}
 	SPLIT_ADD(s, 0, j);
 	FIX_PREALLOC_SIZE(list);