Issue #12213: Fix a buffering bug with interleaved reads and writes that

could appear on io.BufferedRandom streams.

Lib/test/test_io.py

@@ -1367,15 +1367,18 @@ class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest):
         rw.seek(0, 0)
         self.assertEqual(b"asdf", rw.read(4))
 
-        rw.write(b"asdf")
+        rw.write(b"123f")
         rw.seek(0, 0)
-        self.assertEqual(b"asdfasdfl", rw.read())
+        self.assertEqual(b"asdf123fl", rw.read())
         self.assertEqual(9, rw.tell())
         rw.seek(-4, 2)
         self.assertEqual(5, rw.tell())
         rw.seek(2, 1)
         self.assertEqual(7, rw.tell())
         self.assertEqual(b"fl", rw.read(11))
+        rw.flush()
+        self.assertEqual(b"asdf123fl", raw.getvalue())
+
         self.assertRaises(TypeError, rw.seek, 0.0)
 
     def check_flush_and_read(self, read_func):
@@ -1520,6 +1523,44 @@ class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest):
         BufferedReaderTest.test_misbehaved_io(self)
         BufferedWriterTest.test_misbehaved_io(self)
 
+    def test_interleaved_read_write(self):
+        # Test for issue #12213
+        with self.BytesIO(b'abcdefgh') as raw:
+            with self.tp(raw, 100) as f:
+                f.write(b"1")
+                self.assertEqual(f.read(1), b'b')
+                f.write(b'2')
+                self.assertEqual(f.read1(1), b'd')
+                f.write(b'3')
+                buf = bytearray(1)
+                f.readinto(buf)
+                self.assertEqual(buf, b'f')
+                f.write(b'4')
+                self.assertEqual(f.peek(1), b'h')
+                f.flush()
+                self.assertEqual(raw.getvalue(), b'1b2d3f4h')
+
+        with self.BytesIO(b'abc') as raw:
+            with self.tp(raw, 100) as f:
+                self.assertEqual(f.read(1), b'a')
+                f.write(b"2")
+                self.assertEqual(f.read(1), b'c')
+                f.flush()
+                self.assertEqual(raw.getvalue(), b'a2c')
+
+    def test_interleaved_readline_write(self):
+        with self.BytesIO(b'ab\ncdef\ng\n') as raw:
+            with self.tp(raw) as f:
+                f.write(b'1')
+                self.assertEqual(f.readline(), b'b\n')
+                f.write(b'2')
+                self.assertEqual(f.readline(), b'def\n')
+                f.write(b'3')
+                self.assertEqual(f.readline(), b'\n')
+                f.flush()
+                self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n')
+
+
 class CBufferedRandomTest(CBufferedReaderTest, CBufferedWriterTest, BufferedRandomTest):
     tp = io.BufferedRandom
 

Misc/NEWS

@@ -40,6 +40,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #12213: Fix a buffering bug with interleaved reads and writes that
+  could appear on io.BufferedRandom streams.
+
 - Issue #12326: sys.platform is now always 'linux2' on Linux, even if Python
   is compiled on Linux 3.
 

Modules/_io/bufferedio.c

@@ -720,6 +720,28 @@ _trap_eintr(void)
  * Shared methods and wrappers
  */
 
+static PyObject *
+buffered_flush_and_rewind_unlocked(buffered *self)
+{
+    PyObject *res;
+
+    res = _bufferedwriter_flush_unlocked(self, 0);
+    if (res == NULL)
+        return NULL;
+    Py_DECREF(res);
+
+    if (self->readable) {
+        /* Rewind the raw stream so that its position corresponds to
+           the current logical position. */
+        Py_off_t n;
+        n = _buffered_raw_seek(self, -RAW_OFFSET(self), 1);
+        _bufferedreader_reset_buf(self);
+        if (n == -1)
+            return NULL;
+    }
+    Py_RETURN_NONE;
+}
+
 static PyObject *
 buffered_flush(buffered *self, PyObject *args)
 {
@@ -730,16 +752,7 @@ buffered_flush(buffered *self, PyObject *args)
 
     if (!ENTER_BUFFERED(self))
         return NULL;
-    res = _bufferedwriter_flush_unlocked(self, 0);
+    res = buffered_flush_and_rewind_unlocked(self);
-    if (res != NULL && self->readable) {
-        /* Rewind the raw stream so that its position corresponds to
-           the current logical position. */
-        Py_off_t n;
-        n = _buffered_raw_seek(self, -RAW_OFFSET(self), 1);
-        if (n == -1)
-            Py_CLEAR(res);
-        _bufferedreader_reset_buf(self);
-    }
     LEAVE_BUFFERED(self)
 
     return res;
@@ -760,7 +773,7 @@ buffered_peek(buffered *self, PyObject *args)
         return NULL;
 
     if (self->writable) {
-        res = _bufferedwriter_flush_unlocked(self, 1);
+        res = buffered_flush_and_rewind_unlocked(self);
         if (res == NULL)
             goto end;
         Py_CLEAR(res);
@@ -795,19 +808,18 @@ buffered_read(buffered *self, PyObject *args)
         if (!ENTER_BUFFERED(self))
             return NULL;
         res = _bufferedreader_read_all(self);
-        LEAVE_BUFFERED(self)
     }
     else {
         res = _bufferedreader_read_fast(self, n);
-        if (res == Py_None) {
+        if (res != Py_None)
-            Py_DECREF(res);
+            return res;
-            if (!ENTER_BUFFERED(self))
+        Py_DECREF(res);
-                return NULL;
+        if (!ENTER_BUFFERED(self))
-            res = _bufferedreader_read_generic(self, n);
+            return NULL;
-            LEAVE_BUFFERED(self)
+        res = _bufferedreader_read_generic(self, n);
-        }
     }
 
+    LEAVE_BUFFERED(self)
     return res;
 }
 
@@ -833,13 +845,6 @@ buffered_read1(buffered *self, PyObject *args)
     if (!ENTER_BUFFERED(self))
         return NULL;
     
-    if (self->writable) {
-        res = _bufferedwriter_flush_unlocked(self, 1);
-        if (res == NULL)
-            goto end;
-        Py_CLEAR(res);
-    }
-
     /* Return up to n bytes.  If at least one byte is buffered, we
        only return buffered bytes.  Otherwise, we do one raw read. */
 
@@ -859,6 +864,13 @@ buffered_read1(buffered *self, PyObject *args)
         goto end;
     }
 
+    if (self->writable) {
+        res = buffered_flush_and_rewind_unlocked(self);
+        if (res == NULL)
+            goto end;
+        Py_DECREF(res);
+    }
+
     /* Fill the buffer from the raw stream, and copy it to the result. */
     _bufferedreader_reset_buf(self);
     r = _bufferedreader_fill_buffer(self);
@@ -881,24 +893,10 @@ end:
 static PyObject *
 buffered_readinto(buffered *self, PyObject *args)
 {
-    PyObject *res = NULL;
-
     CHECK_INITIALIZED(self)
     
-    /* TODO: use raw.readinto() instead! */
+    /* TODO: use raw.readinto() (or a direct copy from our buffer) instead! */
-    if (self->writable) {
+    return bufferediobase_readinto((PyObject *)self, args);
-        if (!ENTER_BUFFERED(self))
-            return NULL;
-        res = _bufferedwriter_flush_unlocked(self, 0);
-        LEAVE_BUFFERED(self)
-        if (res == NULL)
-            goto end;
-        Py_DECREF(res);
-    }
-    res = bufferediobase_readinto((PyObject *)self, args);
-
-end:
-    return res;
 }
 
 static PyObject *
@@ -936,12 +934,6 @@ _buffered_readline(buffered *self, Py_ssize_t limit)
         goto end_unlocked;
 
     /* Now we try to get some more from the raw stream */
-    if (self->writable) {
-        res = _bufferedwriter_flush_unlocked(self, 1);
-        if (res == NULL)
-            goto end;
-        Py_CLEAR(res);
-    }
     chunks = PyList_New(0);
     if (chunks == NULL)
         goto end;
@@ -955,9 +947,16 @@ _buffered_readline(buffered *self, Py_ssize_t limit)
         }
         Py_CLEAR(res);
         written += n;
+        self->pos += n;
         if (limit >= 0)
             limit -= n;
     }
+    if (self->writable) {
+        PyObject *r = buffered_flush_and_rewind_unlocked(self);
+        if (r == NULL)
+            goto end;
+        Py_DECREF(r);
+    }
 
     for (;;) {
         _bufferedreader_reset_buf(self);
@@ -1126,20 +1125,11 @@ buffered_truncate(buffered *self, PyObject *args)
         return NULL;
 
     if (self->writable) {
-        res = _bufferedwriter_flush_unlocked(self, 0);
+        res = buffered_flush_and_rewind_unlocked(self);
         if (res == NULL)
             goto end;
         Py_CLEAR(res);
     }
-    if (self->readable) {
-        if (pos == Py_None) {
-            /* Rewind the raw stream so that its position corresponds to
-               the current logical position. */
-            if (_buffered_raw_seek(self, -RAW_OFFSET(self), 1) == -1)
-                goto end;
-        }
-        _bufferedreader_reset_buf(self);
-    }
     res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_truncate, pos, NULL);
     if (res == NULL)
         goto end;
@@ -1341,17 +1331,18 @@ _bufferedreader_read_all(buffered *self)
             Py_DECREF(chunks);
             return NULL;
         }
+        self->pos += current_size;
     }
-    _bufferedreader_reset_buf(self);
     /* We're going past the buffer's bounds, flush it */
     if (self->writable) {
-        res = _bufferedwriter_flush_unlocked(self, 1);
+        res = buffered_flush_and_rewind_unlocked(self);
         if (res == NULL) {
             Py_DECREF(chunks);
             return NULL;
         }
         Py_CLEAR(res);
     }
+    _bufferedreader_reset_buf(self);
     while (1) {
         if (data) {
             if (PyList_Append(chunks, data) < 0) {
@@ -1434,6 +1425,14 @@ _bufferedreader_read_generic(buffered *self, Py_ssize_t n)
         memcpy(out, self->buffer + self->pos, current_size);
         remaining -= current_size;
         written += current_size;
+        self->pos += current_size;
+    }
+    /* Flush the write buffer if necessary */
+    if (self->writable) {
+        PyObject *r = buffered_flush_and_rewind_unlocked(self);
+        if (r == NULL)
+            goto error;
+        Py_DECREF(r);
     }
     _bufferedreader_reset_buf(self);
     while (remaining > 0) {