mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

- A new pickle protocol (protocol 3) is added with explicit support

Browse source 
for bytes.  This is the default protocol.  It intentionally cannot
  be unpickled by Python 2.x.

- When a pickle	written	by Python 2.x contains an (8-bit) str
  instance, this is now decoded to a (Unicode) str instance.  The
  encoding used to do this defaults to ASCII, but can be overridden
  via two new keyword arguments to the Unpickler class.  Previously
  this would create bytes instances, which is usually wrong: str
  instances are often used to pickle attribute names etc., and text is
  more common than binary data anyway.
This commit is contained in:
Guido van Rossum 2008-03-17 22:56:06 +00:00
parent 953e4e52c4
commit f416981691
6 changed files with 165 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

64
Lib/pickle.py
View file

@ -42,19 +42,22 @@ __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
bytes_types = (bytes, bytearray, memoryview) bytes_types = (bytes, bytearray, memoryview)
# These are purely informational; no code uses these. # These are purely informational; no code uses these.
format_version = "2.0" # File format version we write format_version = "3.0" # File format version we write
compatible_formats = ["1.0", # Original protocol 0 compatible_formats = ["1.0", # Original protocol 0
"1.1", # Protocol 0 with INST added "1.1", # Protocol 0 with INST added
"1.2", # Original protocol 1 "1.2", # Original protocol 1
"1.3", # Protocol 1 with BINFLOAT added "1.3", # Protocol 1 with BINFLOAT added
"2.0", # Protocol 2 "2.0", # Protocol 2
"3.0", # Protocol 3
] # Old format versions we can read ] # Old format versions we can read
# This is the highest protocol number we know how to read. # This is the highest protocol number we know how to read.
HIGHEST_PROTOCOL = 2 HIGHEST_PROTOCOL = 3
# The protocol we write by default. May be less than HIGHEST_PROTOCOL. # The protocol we write by default. May be less than HIGHEST_PROTOCOL.
DEFAULT_PROTOCOL = 2 # We intentionally write a protocol that Python 2.x cannot read;
# there are too many issues with that.
DEFAULT_PROTOCOL = 3
# Why use struct.pack() for pickling but marshal.loads() for # Why use struct.pack() for pickling but marshal.loads() for
# unpickling? struct.pack() is 40% faster than marshal.dumps(), but # unpickling? struct.pack() is 40% faster than marshal.dumps(), but
@ -161,6 +164,10 @@ LONG4 = b'\x8b' # push really big long
_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3] _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
# Protocol 3 (Python 3.x)
BINBYTES = b'B' # push bytes; counted binary string argument
SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)]) __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
@ -494,20 +501,19 @@ class Pickler:
self.write(FLOAT + repr(obj).encode("ascii") + b'\n') self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
dispatch[float] = save_float dispatch[float] = save_float
def save_string(self, obj, pack=struct.pack): def save_bytes(self, obj, pack=struct.pack):
if self.bin: if self.proto < 3:
n = len(obj) self.save_reduce(bytes, (list(obj),))
if n < 256: return
self.write(SHORT_BINSTRING + bytes([n]) + bytes(obj)) n = len(obj)
else: if n < 256:
self.write(BINSTRING + pack("<i", n) + bytes(obj)) self.write(SHORT_BINBYTES + bytes([n]) + bytes(obj))
else: else:
# Strip leading 'b' due to repr() of bytes() returning b'...' self.write(BINBYTES + pack("<i", n) + bytes(obj))
self.write(STRING + repr(obj).lstrip("b").encode("ascii") + b'\n')
self.memoize(obj) self.memoize(obj)
dispatch[bytes] = save_string dispatch[bytes] = save_bytes
def save_unicode(self, obj, pack=struct.pack): def save_str(self, obj, pack=struct.pack):
if self.bin: if self.bin:
encoded = obj.encode('utf-8') encoded = obj.encode('utf-8')
n = len(encoded) n = len(encoded)
@ -518,7 +524,7 @@ class Pickler:
self.write(UNICODE + bytes(obj.encode('raw-unicode-escape')) + self.write(UNICODE + bytes(obj.encode('raw-unicode-escape')) +
b'\n') b'\n')
self.memoize(obj) self.memoize(obj)
dispatch[str] = save_unicode dispatch[str] = save_str
def save_tuple(self, obj): def save_tuple(self, obj):
write = self.write write = self.write
@ -775,7 +781,7 @@ def whichmodule(func, funcname):
class Unpickler: class Unpickler:
def __init__(self, file): def __init__(self, file, *, encoding="ASCII", errors="strict"):
"""This takes a binary file for reading a pickle data stream. """This takes a binary file for reading a pickle data stream.
The protocol version of the pickle is detected automatically, so no The protocol version of the pickle is detected automatically, so no
@ -787,10 +793,16 @@ class Unpickler:
Thus file-like object can be a binary file object opened for Thus file-like object can be a binary file object opened for
reading, a BytesIO object, or any other custom object that reading, a BytesIO object, or any other custom object that
meets this interface. meets this interface.
Optional keyword arguments are encoding and errors, which are
used to decode 8-bit string instances pickled by Python 2.x.
These default to 'ASCII' and 'strict', respectively.
""" """
self.readline = file.readline self.readline = file.readline
self.read = file.read self.read = file.read
self.memo = {} self.memo = {}
self.encoding = encoding
self.errors = errors
def load(self): def load(self):
"""Read a pickled object representation from the open file. """Read a pickled object representation from the open file.
@ -831,7 +843,7 @@ class Unpickler:
def load_proto(self): def load_proto(self):
proto = ord(self.read(1)) proto = ord(self.read(1))
if not 0 <= proto <= 2: if not 0 <= proto <= HIGHEST_PROTOCOL:
raise ValueError("unsupported pickle protocol: %d" % proto) raise ValueError("unsupported pickle protocol: %d" % proto)
dispatch[PROTO[0]] = load_proto dispatch[PROTO[0]] = load_proto
@ -924,9 +936,16 @@ class Unpickler:
def load_binstring(self): def load_binstring(self):
len = mloads(b'i' + self.read(4)) len = mloads(b'i' + self.read(4))
self.append(self.read(len)) data = self.read(len)
value = str(data, self.encoding, self.errors)
self.append(value)
dispatch[BINSTRING[0]] = load_binstring dispatch[BINSTRING[0]] = load_binstring
def load_binbytes(self):
len = mloads(b'i' + self.read(4))
self.append(self.read(len))
dispatch[BINBYTES[0]] = load_binbytes
def load_unicode(self): def load_unicode(self):
self.append(str(self.readline()[:-1], 'raw-unicode-escape')) self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
dispatch[UNICODE[0]] = load_unicode dispatch[UNICODE[0]] = load_unicode
@ -938,9 +957,16 @@ class Unpickler:
def load_short_binstring(self): def load_short_binstring(self):
len = ord(self.read(1)) len = ord(self.read(1))
self.append(bytes(self.read(len))) data = bytes(self.read(len))
value = str(data, self.encoding, self.errors)
self.append(value)
dispatch[SHORT_BINSTRING[0]] = load_short_binstring dispatch[SHORT_BINSTRING[0]] = load_short_binstring
def load_short_binbytes(self):
len = ord(self.read(1))
self.append(bytes(self.read(len)))
dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
def load_tuple(self): def load_tuple(self):
k = self.marker() k = self.marker()
self.stack[k:] = [tuple(self.stack[k+1:])] self.stack[k:] = [tuple(self.stack[k+1:])]

109
Lib/pickletools.py
View file

@ -746,6 +746,11 @@ pyfloat = StackObject(
doc="A Python float object.") doc="A Python float object.")
pystring = StackObject( pystring = StackObject(
name='string',
obtype=bytes,
doc="A Python (8-bit) string object.")
pybytes = StackObject(
name='bytes', name='bytes',
obtype=bytes, obtype=bytes,
doc="A Python bytes object.") doc="A Python bytes object.")
@ -753,7 +758,7 @@ pystring = StackObject(
pyunicode = StackObject( pyunicode = StackObject(
name='str', name='str',
obtype=str, obtype=str,
doc="A Python string object.") doc="A Python (Unicode) string object.")
pynone = StackObject( pynone = StackObject(
name="None", name="None",
@ -868,7 +873,7 @@ class OpcodeInfo(object):
assert isinstance(x, StackObject) assert isinstance(x, StackObject)
self.stack_after = stack_after self.stack_after = stack_after
assert isinstance(proto, int) and 0 <= proto <= 2 assert isinstance(proto, int) and 0 <= proto <= 3
self.proto = proto self.proto = proto
assert isinstance(doc, str) assert isinstance(doc, str)
@ -995,7 +1000,9 @@ opcodes = [
The argument is a repr-style string, with bracketing quote characters, The argument is a repr-style string, with bracketing quote characters,
and perhaps embedded escapes. The argument extends until the next and perhaps embedded escapes. The argument extends until the next
newline character. newline character. (Actually, they are decoded into a str instance
using the encoding given to the Unpickler constructor. or the default,
'ASCII'.)
"""), """),
I(name='BINSTRING', I(name='BINSTRING',
@ -1008,7 +1015,9 @@ opcodes = [
There are two arguments: the first is a 4-byte little-endian signed int There are two arguments: the first is a 4-byte little-endian signed int
giving the number of bytes in the string, and the second is that many giving the number of bytes in the string, and the second is that many
bytes, which are taken literally as the string content. bytes, which are taken literally as the string content. (Actually,
they are decoded into a str instance using the encoding given to the
Unpickler constructor. or the default, 'ASCII'.)
"""), """),
I(name='SHORT_BINSTRING', I(name='SHORT_BINSTRING',
@ -1019,6 +1028,36 @@ opcodes = [
proto=1, proto=1,
doc="""Push a Python string object. doc="""Push a Python string object.
There are two arguments: the first is a 1-byte unsigned int giving
the number of bytes in the string, and the second is that many bytes,
which are taken literally as the string content. (Actually, they
are decoded into a str instance using the encoding given to the
Unpickler constructor. or the default, 'ASCII'.)
"""),
# Bytes (protocol 3 only; older protocols don't support bytes at all)
I(name='BINBYTES',
code='B',
arg=string4,
stack_before=[],
stack_after=[pybytes],
proto=3,
doc="""Push a Python bytes object.
There are two arguments: the first is a 4-byte little-endian signed int
giving the number of bytes in the string, and the second is that many
bytes, which are taken literally as the bytes content.
"""),
I(name='SHORT_BINBYTES',
code='C',
arg=string1,
stack_before=[],
stack_after=[pybytes],
proto=1,
doc="""Push a Python string object.
There are two arguments: the first is a 1-byte unsigned int giving There are two arguments: the first is a 1-byte unsigned int giving
the number of bytes in the string, and the second is that many bytes, the number of bytes in the string, and the second is that many bytes,
which are taken literally as the string content. which are taken literally as the string content.
@ -2006,9 +2045,9 @@ class _Example:
_dis_test = r""" _dis_test = r"""
>>> import pickle >>> import pickle
>>> x = [1, 2, (3, 4), {bytes(b'abc'): "def"}] >>> x = [1, 2, (3, 4), {b'abc': "def"}]
>>> pkl = pickle.dumps(x, 0) >>> pkl0 = pickle.dumps(x, 0)
>>> dis(pkl) >>> dis(pkl0)
0: ( MARK 0: ( MARK
1: l LIST (MARK at 0) 1: l LIST (MARK at 0)
2: p PUT 0 2: p PUT 0
@ -2025,19 +2064,32 @@ _dis_test = r"""
25: ( MARK 25: ( MARK
26: d DICT (MARK at 25) 26: d DICT (MARK at 25)
27: p PUT 2 27: p PUT 2
30: S STRING 'abc' 30: c GLOBAL 'builtins bytes'
37: p PUT 3 46: p PUT 3
40: V UNICODE 'def' 49: ( MARK
45: p PUT 4 50: ( MARK
48: s SETITEM 51: l LIST (MARK at 50)
49: a APPEND 52: p PUT 4
50: . STOP 55: L LONG 97
59: a APPEND
60: L LONG 98
64: a APPEND
65: L LONG 99
69: a APPEND
70: t TUPLE (MARK at 49)
71: p PUT 5
74: R REDUCE
75: V UNICODE 'def'
80: p PUT 6
83: s SETITEM
84: a APPEND
85: . STOP
highest protocol among opcodes = 0 highest protocol among opcodes = 0
Try again with a "binary" pickle. Try again with a "binary" pickle.
>>> pkl = pickle.dumps(x, 1) >>> pkl1 = pickle.dumps(x, 1)
>>> dis(pkl) >>> dis(pkl1)
0: ] EMPTY_LIST 0: ] EMPTY_LIST
1: q BINPUT 0 1: q BINPUT 0
3: ( MARK 3: ( MARK
@ -2050,13 +2102,24 @@ Try again with a "binary" pickle.
14: q BINPUT 1 14: q BINPUT 1
16: } EMPTY_DICT 16: } EMPTY_DICT
17: q BINPUT 2 17: q BINPUT 2
19: U SHORT_BINSTRING 'abc' 19: c GLOBAL 'builtins bytes'
24: q BINPUT 3 35: q BINPUT 3
26: X BINUNICODE 'def' 37: ( MARK
34: q BINPUT 4 38: ] EMPTY_LIST
36: s SETITEM 39: q BINPUT 4
37: e APPENDS (MARK at 3) 41: ( MARK
38: . STOP 42: K BININT1 97
44: K BININT1 98
46: K BININT1 99
48: e APPENDS (MARK at 41)
49: t TUPLE (MARK at 37)
50: q BINPUT 5
52: R REDUCE
53: X BINUNICODE 'def'
61: q BINPUT 6
63: s SETITEM
64: e APPENDS (MARK at 3)
65: . STOP
highest protocol among opcodes = 1 highest protocol among opcodes = 1
Exercise the INST/OBJ/BUILD family. Exercise the INST/OBJ/BUILD family.

21
Lib/test/pickletester.py
View file

@ -490,6 +490,12 @@ class AbstractPickleTests(unittest.TestCase):
u2 = self.loads(p) u2 = self.loads(p)
self.assertEqual(u2, u) self.assertEqual(u2, u)
def test_bytes(self):
for proto in protocols:
for u in b'', b'xyz', b'xyz'*100:
p = self.dumps(u)
self.assertEqual(self.loads(p), u)
def test_ints(self): def test_ints(self):
import sys import sys
for proto in protocols: for proto in protocols:
@ -532,8 +538,8 @@ class AbstractPickleTests(unittest.TestCase):
@run_with_locale('LC_ALL', 'de_DE', 'fr_FR') @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
def test_float_format(self): def test_float_format(self):
# make sure that floats are formatted locale independent # make sure that floats are formatted locale independent with proto 0
self.assertEqual(self.dumps(1.2)[0:3], b'F1.') self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
def test_reduce(self): def test_reduce(self):
pass pass
@ -624,6 +630,12 @@ class AbstractPickleTests(unittest.TestCase):
(2, 2): pickle.TUPLE2, (2, 2): pickle.TUPLE2,
(2, 3): pickle.TUPLE3, (2, 3): pickle.TUPLE3,
(2, 4): pickle.TUPLE, (2, 4): pickle.TUPLE,
(3, 0): pickle.EMPTY_TUPLE,
(3, 1): pickle.TUPLE1,
(3, 2): pickle.TUPLE2,
(3, 3): pickle.TUPLE3,
(3, 4): pickle.TUPLE,
} }
a = () a = ()
b = (1,) b = (1,)
@ -643,14 +655,17 @@ class AbstractPickleTests(unittest.TestCase):
expected_opcode = {(0, None): pickle.NONE, expected_opcode = {(0, None): pickle.NONE,
(1, None): pickle.NONE, (1, None): pickle.NONE,
(2, None): pickle.NONE, (2, None): pickle.NONE,
(3, None): pickle.NONE,
(0, True): pickle.INT, (0, True): pickle.INT,
(1, True): pickle.INT, (1, True): pickle.INT,
(2, True): pickle.NEWTRUE, (2, True): pickle.NEWTRUE,
(3, True): pickle.NEWTRUE,
(0, False): pickle.INT, (0, False): pickle.INT,
(1, False): pickle.INT, (1, False): pickle.INT,
(2, False): pickle.NEWFALSE, (2, False): pickle.NEWFALSE,
(3, False): pickle.NEWFALSE,
} }
for proto in protocols: for proto in protocols:
for x in None, False, True: for x in None, False, True:
@ -955,7 +970,7 @@ class AbstractPickleModuleTests(unittest.TestCase):
def test_highest_protocol(self): def test_highest_protocol(self):
# Of course this needs to be changed when HIGHEST_PROTOCOL changes. # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
self.assertEqual(self.module.HIGHEST_PROTOCOL, 2) self.assertEqual(self.module.HIGHEST_PROTOCOL, 3)
def test_callapi(self): def test_callapi(self):
from io import BytesIO from io import BytesIO

12
Lib/test/test_pickle.py
View file

@ -12,23 +12,19 @@ class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
module = pickle module = pickle
error = KeyError error = KeyError
def dumps(self, arg, proto=0, fast=0): def dumps(self, arg, proto=None):
# Ignore fast
return pickle.dumps(arg, proto) return pickle.dumps(arg, proto)
def loads(self, buf): def loads(self, buf):
# Ignore fast
return pickle.loads(buf) return pickle.loads(buf)
class PicklerTests(AbstractPickleTests): class PicklerTests(AbstractPickleTests):
error = KeyError error = KeyError
def dumps(self, arg, proto=0, fast=0): def dumps(self, arg, proto=None):
f = io.BytesIO() f = io.BytesIO()
p = pickle.Pickler(f, proto) p = pickle.Pickler(f, proto)
if fast:
p.fast = fast
p.dump(arg) p.dump(arg)
f.seek(0) f.seek(0)
return bytes(f.read()) return bytes(f.read())
@ -40,14 +36,12 @@ class PicklerTests(AbstractPickleTests):
class PersPicklerTests(AbstractPersistentPicklerTests): class PersPicklerTests(AbstractPersistentPicklerTests):
def dumps(self, arg, proto=0, fast=0): def dumps(self, arg, proto=None):
class PersPickler(pickle.Pickler): class PersPickler(pickle.Pickler):
def persistent_id(subself, obj): def persistent_id(subself, obj):
return self.persistent_id(obj) return self.persistent_id(obj)
f = io.BytesIO() f = io.BytesIO()
p = PersPickler(f, proto) p = PersPickler(f, proto)
if fast:
p.fast = fast
p.dump(arg) p.dump(arg)
f.seek(0) f.seek(0)
return f.read() return f.read()

2
Lib/test/test_pickletools.py
View file

@ -6,7 +6,7 @@ from test.pickletester import AbstractPickleModuleTests
class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
def dumps(self, arg, proto=0, fast=0): def dumps(self, arg, proto=None):
return pickletools.optimize(pickle.dumps(arg, proto)) return pickletools.optimize(pickle.dumps(arg, proto))
def loads(self, buf): def loads(self, buf):

12
Misc/NEWS
View file

@ -23,6 +23,18 @@ Extension Modules
Library Library
------- -------
- A new pickle protocol (protocol 3) is added with explicit support
for bytes. This is the default protocol. It intentionally cannot
be unpickled by Python 2.x.
- When a pickle written by Python 2.x contains an (8-bit) str
instance, this is now decoded to a (Unicode) str instance. The
encoding used to do this defaults to ASCII, but can be overridden
via two new keyword arguments to the Unpickler class. Previously
this would create bytes instances, which is usually wrong: str
instances are often used to pickle attribute names etc., and text is
more common than binary data anyway.
- Default to ASCII as the locale.getpreferredencoding, if the POSIX - Default to ASCII as the locale.getpreferredencoding, if the POSIX
system doesn't support CODESET and LANG isn't set or doesn't system doesn't support CODESET and LANG isn't set or doesn't
allow deduction of an encoding. allow deduction of an encoding.