Issue #10212: Support new buffer interface for struct.unpack and

cStringIO

Lib/test/test_StringIO.py

@@ -20,7 +20,6 @@ class TestGenericStringIO(unittest.TestCase):
     constructor = str
 
     def setUp(self):
-        self._line = self.constructor(self._line)
         self._lines = self.constructor((self._line + '\n') * 5)
         self._fp = self.MODULE.StringIO(self._lines)
 
@@ -210,12 +209,16 @@ class TestBufferStringIO(TestStringIO):
 class TestBuffercStringIO(TestcStringIO):
     constructor = buffer
 
+class TestMemoryviewcStringIO(TestcStringIO):
+    constructor = memoryview
+
 
 def test_main():
     test_support.run_unittest(TestStringIO, TestcStringIO)
     with test_support.check_py3k_warnings(("buffer.. not supported",
                                              DeprecationWarning)):
         test_support.run_unittest(TestBufferStringIO, TestBuffercStringIO)
+    test_support.run_unittest(TestMemoryviewcStringIO)
 
 if __name__ == '__main__':
     test_main()

Lib/test/test_struct.py

@@ -496,6 +496,17 @@ class StructTest(unittest.TestCase):
 
             self.test_unpack_from(cls=buffer)
 
+    def test_unpack_with_memoryview(self):
+        with check_py3k_warnings(("buffer.. not supported in 3.x",
+                                  DeprecationWarning)):
+            # SF bug 1563759: struct.unpack doesn't support buffer protocol objects
+            data1 = memoryview('\x12\x34\x56\x78')
+            for data in [data1,]:
+                value, = struct.unpack('>I', data)
+                self.assertEqual(value, 0x12345678)
+
+            self.test_unpack_from(cls=memoryview)
+
     def test_bool(self):
         class ExplodingBool(object):
             def __nonzero__(self):

Misc/NEWS

@@ -9,6 +9,8 @@ What's New in Python 2.7.4
 Core and Builtins
 -----------------
 
+- Issue #10211: Buffer objects expose the new buffer interface internally
+
 - Issue #16445: Fixed potential segmentation fault when deleting an exception
   message.
 
@@ -214,6 +216,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #10212: cStringIO and struct.unpack support new buffer objects.
+
 - Issue #12098: multiprocessing on Windows now starts child processes
   using the same sys.flags as the current process.  Initial patch by
   Sergey Mezentsev.

Modules/_struct.c

@@ -1439,6 +1439,7 @@ strings.");
 static PyObject *
 s_unpack(PyObject *self, PyObject *inputstr)
 {
+    Py_buffer buf;
     char *start;
     Py_ssize_t len;
     PyObject *args=NULL, *result;
@@ -1454,12 +1455,17 @@ s_unpack(PyObject *self, PyObject *inputstr)
     args = PyTuple_Pack(1, inputstr);
     if (args == NULL)
         return NULL;
-    if (!PyArg_ParseTuple(args, "s#:unpack", &start, &len))
+    if (!PyArg_ParseTuple(args, "s*:unpack", &buf))
         goto fail;
-    if (soself->s_size != len)
+    start = buf.buf;
+    len = buf.len;
+    if (soself->s_size != len) {
+        PyBuffer_Release(&buf);
         goto fail;
+    }
     result = s_unpack_internal(soself, start);
     Py_DECREF(args);
+    PyBuffer_Release(&buf);
     return result;
 
 fail:
@@ -1482,24 +1488,24 @@ static PyObject *
 s_unpack_from(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwlist[] = {"buffer", "offset", 0};
-#if (PY_VERSION_HEX < 0x02050000)
+    static char *fmt = "z*|n:unpack_from";
-    static char *fmt = "z#|i:unpack_from";
+    Py_buffer buf;
-#else
-    static char *fmt = "z#|n:unpack_from";
-#endif
     Py_ssize_t buffer_len = 0, offset = 0;
     char *buffer = NULL;
     PyStructObject *soself = (PyStructObject *)self;
+    PyObject *result;
     assert(PyStruct_Check(self));
     assert(soself->s_codes != NULL);
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, fmt, kwlist,
-                                     &buffer, &buffer_len, &offset))
+                                     &buf, &offset))
         return NULL;
-
+    buffer = buf.buf;
+    buffer_len = buf.len;
     if (buffer == NULL) {
         PyErr_Format(StructError,
             "unpack_from requires a buffer argument");
+        PyBuffer_Release(&buf);
         return NULL;
     }
 
@@ -1510,9 +1516,12 @@ s_unpack_from(PyObject *self, PyObject *args, PyObject *kwds)
         PyErr_Format(StructError,
             "unpack_from requires a buffer of at least %zd bytes",
             soself->s_size);
+        PyBuffer_Release(&buf);
         return NULL;
     }
-    return s_unpack_internal(soself, buffer + offset);
+    result = s_unpack_internal(soself, buffer + offset);
+    PyBuffer_Release(&buf);
+    return result;
 }
 
 

Modules/cStringIO.c

@@ -66,9 +66,7 @@ typedef struct { /* Subtype of IOobject */
   PyObject_HEAD
   char *buf;
   Py_ssize_t pos, string_size;
-  /* We store a reference to the object here in order to keep
+    Py_buffer pbuf;
-     the buffer alive during the lifetime of the Iobject. */
-  PyObject *pbuf;
 } Iobject;
 
 /* IOobject (common) methods */
@@ -448,12 +446,14 @@ O_cwrite(PyObject *self, const char *c, Py_ssize_t  len) {
 
 static PyObject *
 O_write(Oobject *self, PyObject *args) {
-    char *c;
+    Py_buffer buf;
-    int l;
+    int result;
 
-    if (!PyArg_ParseTuple(args, "t#:write", &c, &l)) return NULL;
+    if (!PyArg_ParseTuple(args, "s*:write", &buf)) return NULL;
 
-    if (O_cwrite((PyObject*)self,c,l) < 0) return NULL;
+    result = O_cwrite((PyObject*)self, buf.buf, buf.len);
+    PyBuffer_Release(&buf);
+    if (result < 0) return NULL;
 
     Py_INCREF(Py_None);
     return Py_None;
@@ -606,7 +606,7 @@ newOobject(int  size) {
 
 static PyObject *
 I_close(Iobject *self, PyObject *unused) {
-    Py_CLEAR(self->pbuf);
+    PyBuffer_Release(&self->pbuf);
     self->buf = NULL;
 
     self->pos = self->string_size = 0;
@@ -635,7 +635,7 @@ static struct PyMethodDef I_methods[] = {
 
 static void
 I_dealloc(Iobject *self) {
-  Py_XDECREF(self->pbuf);
+  PyBuffer_Release(&self->pbuf);
   PyObject_Del(self);
 }
 
@@ -680,25 +680,26 @@ static PyTypeObject Itype = {
 static PyObject *
 newIobject(PyObject *s) {
   Iobject *self;
-  char *buf;
+  Py_buffer buf;
-  Py_ssize_t size;
+  PyObject *args;
+  int result;
 
-  if (PyUnicode_Check(s)) {
+  args = Py_BuildValue("(O)", s);
-    if (PyObject_AsCharBuffer(s, (const char **)&buf, &size) != 0)
+  if (args == NULL)
       return NULL;
-  }
+  result = PyArg_ParseTuple(args, "s*:StringIO", &buf);
-  else if (PyObject_AsReadBuffer(s, (const void **)&buf, &size)) {
+  Py_DECREF(args);
-    PyErr_Format(PyExc_TypeError, "expected read buffer, %.200s found",
+  if (!result)
-                 s->ob_type->tp_name);
       return NULL;
-  }
 
   self = PyObject_New(Iobject, &Itype);
-  if (!self) return NULL;
+  if (!self) {
-  Py_INCREF(s);
+      PyBuffer_Release(&buf);
-  self->buf=buf;
+      return NULL;
-  self->string_size=size;
+  }
-  self->pbuf=s;
+  self->buf=buf.buf;
+  self->string_size=buf.len;
+  self->pbuf=buf;
   self->pos=0;
 
   return (PyObject*)self;