mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-11 01:12:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Issue #19881: Fix bad pickling of large bytes in cpickle.

Browse source
This commit is contained in:
Alexandre Vassalotti 2013-12-05 19:29:32 -08:00
parent c9a87e6bbe
commit 6e73ff1a31
3 changed files with 59 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

57
Lib/test/pickletester.py
View file

@ -3,6 +3,7 @@ import io
import pickle import pickle
import pickletools import pickletools
import random import random
import struct
import sys import sys
import unittest import unittest
import weakref import weakref
@ -1611,9 +1612,9 @@ class BigmemPickleTests(unittest.TestCase):
data = 1 << (8 * size) data = 1 << (8 * size)
try: try:
for proto in protocols: for proto in protocols:
with self.subTest(proto=proto):
if proto < 2: if proto < 2:
continue continue
with self.subTest(proto=proto):
with self.assertRaises((ValueError, OverflowError)): with self.assertRaises((ValueError, OverflowError)):
self.dumps(data, protocol=proto) self.dumps(data, protocol=proto)
finally: finally:
@ -1628,13 +1629,17 @@ class BigmemPickleTests(unittest.TestCase):
data = b"abcd" * (size // 4) data = b"abcd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
with self.subTest(proto=proto):
if proto < 3: if proto < 3:
continue continue
with self.subTest(proto=proto):
try: try:
pickled = self.dumps(data, protocol=proto) pickled = self.dumps(data, protocol=proto)
self.assertTrue(b"abcd" in pickled[:19]) header = (pickle.BINBYTES +
self.assertTrue(b"abcd" in pickled[-18:]) struct.pack("<I", len(data)))
data_start = pickled.index(data)
self.assertEqual(
header,
pickled[data_start-len(header):data_start])
finally: finally:
pickled = None pickled = None
finally: finally:
@ -1642,14 +1647,28 @@ class BigmemPickleTests(unittest.TestCase):
@bigmemtest(size=_4G, memuse=1 + 1, dry_run=False) @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False)
def test_huge_bytes_64b(self, size): def test_huge_bytes_64b(self, size):
data = b"a" * size data = b"acbd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
with self.subTest(proto=proto):
if proto < 3: if proto < 3:
continue continue
with self.subTest(proto=proto):
if proto == 3:
# Protocol 3 does not support large bytes objects.
# Verify that we do not crash when processing one.
with self.assertRaises((ValueError, OverflowError)): with self.assertRaises((ValueError, OverflowError)):
self.dumps(data, protocol=proto) self.dumps(data, protocol=proto)
continue
try:
pickled = self.dumps(data, protocol=proto)
header = (pickle.BINBYTES8 +
struct.pack("<Q", len(data)))
data_start = pickled.index(data)
self.assertEqual(
header,
pickled[data_start-len(header):data_start])
finally:
pickled = None
finally: finally:
data = None data = None
@ -1661,11 +1680,19 @@ class BigmemPickleTests(unittest.TestCase):
data = "abcd" * (size // 4) data = "abcd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
if proto == 0:
continue
with self.subTest(proto=proto): with self.subTest(proto=proto):
try: try:
pickled = self.dumps(data, protocol=proto) pickled = self.dumps(data, protocol=proto)
self.assertTrue(b"abcd" in pickled[:19]) header = (pickle.BINUNICODE +
self.assertTrue(b"abcd" in pickled[-18:]) struct.pack("<I", len(data)))
data_start = pickled.index(b'abcd')
self.assertEqual(
header,
pickled[data_start-len(header):data_start])
self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
pickled.index(b"abcd")), len(data))
finally: finally:
pickled = None pickled = None
finally: finally:
@ -1680,17 +1707,23 @@ class BigmemPickleTests(unittest.TestCase):
data = "abcd" * (size // 4) data = "abcd" * (size // 4)
try: try:
for proto in protocols: for proto in protocols:
with self.subTest(proto=proto):
if proto == 0: if proto == 0:
continue continue
with self.subTest(proto=proto):
if proto < 4: if proto < 4:
with self.assertRaises((ValueError, OverflowError)): with self.assertRaises((ValueError, OverflowError)):
self.dumps(data, protocol=proto) self.dumps(data, protocol=proto)
else: continue
try: try:
pickled = self.dumps(data, protocol=proto) pickled = self.dumps(data, protocol=proto)
self.assertTrue(b"abcd" in pickled[:19]) header = (pickle.BINUNICODE8 +
self.assertTrue(b"abcd" in pickled[-18:]) struct.pack("<Q", len(data)))
data_start = pickled.index(b'abcd')
self.assertEqual(
header,
pickled[data_start-len(header):data_start])
self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
pickled.index(b"abcd")), len(data))
finally: finally:
pickled = None pickled = None
finally: finally:

4
Misc/NEWS
View file

@ -18,11 +18,15 @@ Core and Builtins
Library Library
------- -------
- Issue #19296: Silence compiler warning in dbm_open - Issue #19296: Silence compiler warning in dbm_open
- Issue #19839: Fix regression in bz2 module's handling of non-bzip2 data at - Issue #19839: Fix regression in bz2 module's handling of non-bzip2 data at
EOF, and analogous bug in lzma module. EOF, and analogous bug in lzma module.
- Issue #19881: Fix pickling bug where cpickle would emit bad pickle data for
large bytes string (i.e., with size greater than 2**32-1).
- Issue #19138: doctest's IGNORE_EXCEPTION_DETAIL now allows a match when - Issue #19138: doctest's IGNORE_EXCEPTION_DETAIL now allows a match when
no exception detail exists (no colon following the exception's name, or no exception detail exists (no colon following the exception's name, or
a colon does follow but no text follows the colon). a colon does follow but no text follows the colon).

2
Modules/_pickle.c
View file

@ -2027,7 +2027,7 @@ save_bytes(PicklerObject *self, PyObject *obj)
else if (self->proto >= 4) { else if (self->proto >= 4) {
header[0] = BINBYTES8; header[0] = BINBYTES8;
_write_size64(header + 1, size); _write_size64(header + 1, size);
len = 8; len = 9;
} }
else { else {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,