merge 3.2 (#14212)

Lib/test/test_re.py

@@ -1,4 +1,5 @@
-from test.support import verbose, run_unittest
+from test.support import verbose, run_unittest, gc_collect
+import io
 import re
 from re import Scanner
 import sys
@@ -16,6 +17,17 @@ import unittest
 
 class ReTests(unittest.TestCase):
 
+    def test_keep_buffer(self):
+        # See bug 14212
+        b = bytearray(b'x')
+        it = re.finditer(b'a', b)
+        with self.assertRaises(BufferError):
+            b.extend(b'x'*400)
+        list(it)
+        del it
+        gc_collect()
+        b.extend(b'x'*400)
+
     def test_weakref(self):
         s = 'QabbbcR'
         x = re.compile('ab+c')

Misc/NEWS

@@ -27,6 +27,12 @@ Library
   data or close method) for the Python implementation as well.
   Drop the no-op TreeBuilder().xml() method from the C implementation.
 
+Extension Modules
+-----------------
+
+- Issue #14212: The re module didn't retain a reference to buffers it was
+  scanning, resulting in segfaults.
+
 
 What's New in Python 3.3.0 Alpha 1?
 ===================================

Modules/_sre.c

@@ -1650,7 +1650,8 @@ state_reset(SRE_STATE* state)
 
 static void*
 getstring(PyObject* string, Py_ssize_t* p_length,
-          int* p_logical_charsize, int* p_charsize)
+          int* p_logical_charsize, int* p_charsize,
+          Py_buffer *view)
 {
     /* given a python object, return a data pointer, a length (in
        characters), and a character size.  return NULL if the object
@@ -1660,7 +1661,6 @@ getstring(PyObject* string, Py_ssize_t* p_length,
     Py_ssize_t size, bytes;
     int charsize;
     void* ptr;
-    Py_buffer view;
 
     /* Unicode objects do not support the buffer API. So, get the data
        directly instead. */
@@ -1675,26 +1675,21 @@ getstring(PyObject* string, Py_ssize_t* p_length,
     }
 
     /* get pointer to byte string buffer */
-    view.len = -1;
+    view->len = -1;
     buffer = Py_TYPE(string)->tp_as_buffer;
     if (!buffer || !buffer->bf_getbuffer ||
-        (*buffer->bf_getbuffer)(string, &view, PyBUF_SIMPLE) < 0) {
+        (*buffer->bf_getbuffer)(string, view, PyBUF_SIMPLE) < 0) {
             PyErr_SetString(PyExc_TypeError, "expected string or buffer");
             return NULL;
     }
 
     /* determine buffer size */
-    bytes = view.len;
+    bytes = view->len;
-    ptr = view.buf;
+    ptr = view->buf;
-
-    /* Release the buffer immediately --- possibly dangerous
-       but doing something else would require some re-factoring
-    */
-    PyBuffer_Release(&view);
 
     if (bytes < 0) {
         PyErr_SetString(PyExc_TypeError, "buffer has negative size");
-        return NULL;
+        goto err;
     }
 
     /* determine character size */
@@ -1704,7 +1699,7 @@ getstring(PyObject* string, Py_ssize_t* p_length,
         charsize = 1;
     else {
         PyErr_SetString(PyExc_TypeError, "buffer size mismatch");
-        return NULL;
+        goto err;
     }
 
     *p_length = size;
@@ -1714,8 +1709,13 @@ getstring(PyObject* string, Py_ssize_t* p_length,
     if (ptr == NULL) {
             PyErr_SetString(PyExc_ValueError,
                             "Buffer is NULL");
+            goto err;
     }
     return ptr;
+  err:
+    PyBuffer_Release(view);
+    view->buf = NULL;
+    return NULL;
 }
 
 LOCAL(PyObject*)
@@ -1733,20 +1733,21 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string,
     state->lastmark = -1;
     state->lastindex = -1;
 
-    ptr = getstring(string, &length, &logical_charsize, &charsize);
+    state->buffer.buf = NULL;
+    ptr = getstring(string, &length, &logical_charsize, &charsize, &state->buffer);
     if (!ptr)
-        return NULL;
+        goto err;
 
-        if (logical_charsize == 1 && pattern->logical_charsize > 1) {
+    if (logical_charsize == 1 && pattern->logical_charsize > 1) {
-                PyErr_SetString(PyExc_TypeError,
+        PyErr_SetString(PyExc_TypeError,
                         "can't use a string pattern on a bytes-like object");
-                return NULL;
+        goto err;
-        }
+    }
-        if (logical_charsize > 1 && pattern->logical_charsize == 1) {
+    if (logical_charsize > 1 && pattern->logical_charsize == 1) {
-                PyErr_SetString(PyExc_TypeError,
+        PyErr_SetString(PyExc_TypeError,
                         "can't use a bytes pattern on a string-like object");
-                return NULL;
+        goto err;
-        }
+    }
 
     /* adjust boundaries */
     if (start < 0)
@@ -1780,11 +1781,17 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string,
         state->lower = sre_lower;
 
     return string;
+  err:
+    if (state->buffer.buf)
+        PyBuffer_Release(&state->buffer);
+    return NULL;
 }
 
 LOCAL(void)
 state_fini(SRE_STATE* state)
 {
+    if (state->buffer.buf)
+        PyBuffer_Release(&state->buffer);
     Py_XDECREF(state->string);
     data_stack_dealloc(state);
 }
@@ -1846,6 +1853,8 @@ pattern_dealloc(PatternObject* self)
 {
     if (self->weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject *) self);
+    if (self->view.buf)
+        PyBuffer_Release(&self->view);
     Py_XDECREF(self->pattern);
     Py_XDECREF(self->groupindex);
     Py_XDECREF(self->indexgroup);
@@ -2272,6 +2281,7 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string,
     Py_ssize_t i, b, e;
     int logical_charsize, charsize;
     int filter_is_callable;
+    Py_buffer view;
 
     if (PyCallable_Check(ptemplate)) {
         /* sub/subn takes either a function or a template */
@@ -2281,7 +2291,8 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string,
     } else {
         /* if not callable, check if it's a literal string */
         int literal;
-        ptr = getstring(ptemplate, &n, &logical_charsize, &charsize);
+        view.buf = NULL;
+        ptr = getstring(ptemplate, &n, &logical_charsize, &charsize, &view);
         b = charsize;
         if (ptr) {
             literal = sre_literal_template(b, ptr, n);
@@ -2289,6 +2300,8 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string,
             PyErr_Clear();
             literal = 0;
         }
+        if (view.buf)
+            PyBuffer_Release(&view);
         if (literal) {
             filter = ptemplate;
             Py_INCREF(filter);
@@ -2628,6 +2641,7 @@ _compile(PyObject* self_, PyObject* args)
     Py_ssize_t groups = 0;
     PyObject* groupindex = NULL;
     PyObject* indexgroup = NULL;
+
     if (!PyArg_ParseTuple(args, "OiO!|nOO", &pattern, &flags,
                           &PyList_Type, &code, &groups,
                           &groupindex, &indexgroup))
@@ -2642,6 +2656,7 @@ _compile(PyObject* self_, PyObject* args)
     self->pattern = NULL;
     self->groupindex = NULL;
     self->indexgroup = NULL;
+    self->view.buf = NULL;
 
     self->codesize = n;
 
@@ -2668,7 +2683,7 @@ _compile(PyObject* self_, PyObject* args)
     else {
         Py_ssize_t p_length;
         if (!getstring(pattern, &p_length, &self->logical_charsize,
-                       &self->charsize)) {
+                       &self->charsize, &self->view)) {
             Py_DECREF(self);
             return NULL;
         }

Modules/sre.h

@@ -32,6 +32,7 @@ typedef struct {
     PyObject *weakreflist; /* List of weak references */
     int logical_charsize; /* pattern charsize (or -1) */
     int charsize;
+    Py_buffer view;
     /* pattern code */
     Py_ssize_t codesize;
     SRE_CODE code[1];
@@ -82,6 +83,7 @@ typedef struct {
     char* data_stack;
     size_t data_stack_size;
     size_t data_stack_base;
+    Py_buffer buffer;
     /* current repeat context */
     SRE_REPEAT *repeat;
     /* hooks */