Issue #14455: fix handling of unsigned long long values for binary plist files

Values in the range of an unsigned long long, but outside of the range
of a signed long long were serialized as a negative value.

Due to a bug in PyObjC my test scripts indicated that the previous behavior
matched Apple's plist code, instead the handle large unsigned values correctly.

The change to plistlib.py is from a patch by Serhiy.

Doc/library/plistlib.rst

@@ -107,13 +107,6 @@ This module defines the following functions:
    An :exc:`OverflowError` will be raised for integer values that cannot
    be represented in (binary) plist files.
 
-   .. warning::
-
-      For compatibility with Apple's libraries it is possible to write
-      an integer in the range from 2 ** 63 upto (and including) 2 ** 64
-      to binary plists, even though these will be read back as negative
-      values.
-
    .. versionadded: 3.4
 
 

Lib/plistlib.py

@@ -879,18 +879,19 @@ class _BinaryPlistWriter (object):
                 try:
                     self._fp.write(struct.pack('>Bq', 0x13, value))
                 except struct.error:
-                    raise OverflowError(value)
+                    raise OverflowError(value) from None
             elif value < 1 << 8:
                 self._fp.write(struct.pack('>BB', 0x10, value))
             elif value < 1 << 16:
                 self._fp.write(struct.pack('>BH', 0x11, value))
             elif value < 1 << 32:
                 self._fp.write(struct.pack('>BL', 0x12, value))
+            elif value < 1 << 63:
+                self._fp.write(struct.pack('>BQ', 0x13, value))
+            elif value < 1 << 64:
+                self._fp.write(b'\x14' + value.to_bytes(16, 'big', signed=True))
             else:
-                try:
+                raise OverflowError(value)
-                    self._fp.write(struct.pack('>BQ', 0x13, value))
-                except struct.error:
-                    raise OverflowError(value)
 
         elif isinstance(value, float):
             self._fp.write(struct.pack('>Bd', 0x23, value))

Lib/test/test_plistlib.py

@@ -22,72 +22,73 @@ TESTDATA={
         IiAiaHR0cDovL3d3dy5hcHBsZS5jb20vRFREcy9Qcm9wZXJ0eUxpc3QtMS4w
         LmR0ZCI+CjxwbGlzdCB2ZXJzaW9uPSIxLjAiPgo8ZGljdD4KCTxrZXk+YUJp
         Z0ludDwva2V5PgoJPGludGVnZXI+OTIyMzM3MjAzNjg1NDc3NTc2NDwvaW50
-        ZWdlcj4KCTxrZXk+YURhdGU8L2tleT4KCTxkYXRlPjIwMDQtMTAtMjZUMTA6
+        ZWdlcj4KCTxrZXk+YUJpZ0ludDI8L2tleT4KCTxpbnRlZ2VyPjkyMjMzNzIw
-        MzM6MzNaPC9kYXRlPgoJPGtleT5hRGljdDwva2V5PgoJPGRpY3Q+CgkJPGtl
+        MzY4NTQ3NzU4NTI8L2ludGVnZXI+Cgk8a2V5PmFEYXRlPC9rZXk+Cgk8ZGF0
-        eT5hRmFsc2VWYWx1ZTwva2V5PgoJCTxmYWxzZS8+CgkJPGtleT5hVHJ1ZVZh
+        ZT4yMDA0LTEwLTI2VDEwOjMzOjMzWjwvZGF0ZT4KCTxrZXk+YURpY3Q8L2tl
-        bHVlPC9rZXk+CgkJPHRydWUvPgoJCTxrZXk+YVVuaWNvZGVWYWx1ZTwva2V5
+        eT4KCTxkaWN0PgoJCTxrZXk+YUZhbHNlVmFsdWU8L2tleT4KCQk8ZmFsc2Uv
-        PgoJCTxzdHJpbmc+TcOkc3NpZywgTWHDnzwvc3RyaW5nPgoJCTxrZXk+YW5v
+        PgoJCTxrZXk+YVRydWVWYWx1ZTwva2V5PgoJCTx0cnVlLz4KCQk8a2V5PmFV
-        dGhlclN0cmluZzwva2V5PgoJCTxzdHJpbmc+Jmx0O2hlbGxvICZhbXA7ICdo
+        bmljb2RlVmFsdWU8L2tleT4KCQk8c3RyaW5nPk3DpHNzaWcsIE1hw588L3N0
-        aScgdGhlcmUhJmd0Ozwvc3RyaW5nPgoJCTxrZXk+ZGVlcGVyRGljdDwva2V5
+        cmluZz4KCQk8a2V5PmFub3RoZXJTdHJpbmc8L2tleT4KCQk8c3RyaW5nPiZs
-        PgoJCTxkaWN0PgoJCQk8a2V5PmE8L2tleT4KCQkJPGludGVnZXI+MTc8L2lu
+        dDtoZWxsbyAmYW1wOyAnaGknIHRoZXJlISZndDs8L3N0cmluZz4KCQk8a2V5
-        dGVnZXI+CgkJCTxrZXk+Yjwva2V5PgoJCQk8cmVhbD4zMi41PC9yZWFsPgoJ
+        PmRlZXBlckRpY3Q8L2tleT4KCQk8ZGljdD4KCQkJPGtleT5hPC9rZXk+CgkJ
-        CQk8a2V5PmM8L2tleT4KCQkJPGFycmF5PgoJCQkJPGludGVnZXI+MTwvaW50
+        CTxpbnRlZ2VyPjE3PC9pbnRlZ2VyPgoJCQk8a2V5PmI8L2tleT4KCQkJPHJl
-        ZWdlcj4KCQkJCTxpbnRlZ2VyPjI8L2ludGVnZXI+CgkJCQk8c3RyaW5nPnRl
+        YWw+MzIuNTwvcmVhbD4KCQkJPGtleT5jPC9rZXk+CgkJCTxhcnJheT4KCQkJ
-        eHQ8L3N0cmluZz4KCQkJPC9hcnJheT4KCQk8L2RpY3Q+Cgk8L2RpY3Q+Cgk8
+        CTxpbnRlZ2VyPjE8L2ludGVnZXI+CgkJCQk8aW50ZWdlcj4yPC9pbnRlZ2Vy
-        a2V5PmFGbG9hdDwva2V5PgoJPHJlYWw+MC41PC9yZWFsPgoJPGtleT5hTGlz
+        PgoJCQkJPHN0cmluZz50ZXh0PC9zdHJpbmc+CgkJCTwvYXJyYXk+CgkJPC9k
-        dDwva2V5PgoJPGFycmF5PgoJCTxzdHJpbmc+QTwvc3RyaW5nPgoJCTxzdHJp
+        aWN0PgoJPC9kaWN0PgoJPGtleT5hRmxvYXQ8L2tleT4KCTxyZWFsPjAuNTwv
-        bmc+Qjwvc3RyaW5nPgoJCTxpbnRlZ2VyPjEyPC9pbnRlZ2VyPgoJCTxyZWFs
+        cmVhbD4KCTxrZXk+YUxpc3Q8L2tleT4KCTxhcnJheT4KCQk8c3RyaW5nPkE8
-        PjMyLjU8L3JlYWw+CgkJPGFycmF5PgoJCQk8aW50ZWdlcj4xPC9pbnRlZ2Vy
+        L3N0cmluZz4KCQk8c3RyaW5nPkI8L3N0cmluZz4KCQk8aW50ZWdlcj4xMjwv
-        PgoJCQk8aW50ZWdlcj4yPC9pbnRlZ2VyPgoJCQk8aW50ZWdlcj4zPC9pbnRl
+        aW50ZWdlcj4KCQk8cmVhbD4zMi41PC9yZWFsPgoJCTxhcnJheT4KCQkJPGlu
-        Z2VyPgoJCTwvYXJyYXk+Cgk8L2FycmF5PgoJPGtleT5hTmVnYXRpdmVCaWdJ
+        dGVnZXI+MTwvaW50ZWdlcj4KCQkJPGludGVnZXI+MjwvaW50ZWdlcj4KCQkJ
-        bnQ8L2tleT4KCTxpbnRlZ2VyPi04MDAwMDAwMDAwMDwvaW50ZWdlcj4KCTxr
+        PGludGVnZXI+MzwvaW50ZWdlcj4KCQk8L2FycmF5PgoJPC9hcnJheT4KCTxr
-        ZXk+YU5lZ2F0aXZlSW50PC9rZXk+Cgk8aW50ZWdlcj4tNTwvaW50ZWdlcj4K
+        ZXk+YU5lZ2F0aXZlQmlnSW50PC9rZXk+Cgk8aW50ZWdlcj4tODAwMDAwMDAw
-        CTxrZXk+YVN0cmluZzwva2V5PgoJPHN0cmluZz5Eb29kYWg8L3N0cmluZz4K
+        MDA8L2ludGVnZXI+Cgk8a2V5PmFOZWdhdGl2ZUludDwva2V5PgoJPGludGVn
-        CTxrZXk+YW5FbXB0eURpY3Q8L2tleT4KCTxkaWN0Lz4KCTxrZXk+YW5FbXB0
+        ZXI+LTU8L2ludGVnZXI+Cgk8a2V5PmFTdHJpbmc8L2tleT4KCTxzdHJpbmc+
-        eUxpc3Q8L2tleT4KCTxhcnJheS8+Cgk8a2V5PmFuSW50PC9rZXk+Cgk8aW50
+        RG9vZGFoPC9zdHJpbmc+Cgk8a2V5PmFuRW1wdHlEaWN0PC9rZXk+Cgk8ZGlj
-        ZWdlcj43Mjg8L2ludGVnZXI+Cgk8a2V5Pm5lc3RlZERhdGE8L2tleT4KCTxh
+        dC8+Cgk8a2V5PmFuRW1wdHlMaXN0PC9rZXk+Cgk8YXJyYXkvPgoJPGtleT5h
-        cnJheT4KCQk8ZGF0YT4KCQlQR3h2ZEhNZ2IyWWdZbWx1WVhKNUlHZDFibXMr
+        bkludDwva2V5PgoJPGludGVnZXI+NzI4PC9pbnRlZ2VyPgoJPGtleT5uZXN0
-        QUFFQ0F6eHNiM1J6SUc5bUlHSnBibUZ5ZVNCbmRXNXIKCQlQZ0FCQWdNOGJH
+        ZWREYXRhPC9rZXk+Cgk8YXJyYXk+CgkJPGRhdGE+CgkJUEd4dmRITWdiMlln
-        OTBjeUJ2WmlCaWFXNWhjbmtnWjNWdWF6NEFBUUlEUEd4dmRITWdiMllnWW1s
-        dVlYSjUKCQlJR2QxYm1zK0FBRUNBenhzYjNSeklHOW1JR0pwYm1GeWVTQm5k
-        VzVyUGdBQkFnTThiRzkwY3lCdlppQmkKCQlhVzVoY25rZ1ozVnVhejRBQVFJ
-        RFBHeHZkSE1nYjJZZ1ltbHVZWEo1SUdkMWJtcytBQUVDQXp4c2IzUnoKCQlJ
-        RzltSUdKcGJtRnllU0JuZFc1clBnQUJBZ004Ykc5MGN5QnZaaUJpYVc1aGNu
-        a2daM1Z1YXo0QUFRSUQKCQlQR3h2ZEhNZ2IyWWdZbWx1WVhKNUlHZDFibXMr
-        QUFFQ0F3PT0KCQk8L2RhdGE+Cgk8L2FycmF5PgoJPGtleT5zb21lRGF0YTwv
-        a2V5PgoJPGRhdGE+CglQR0pwYm1GeWVTQm5kVzVyUGc9PQoJPC9kYXRhPgoJ
-        PGtleT5zb21lTW9yZURhdGE8L2tleT4KCTxkYXRhPgoJUEd4dmRITWdiMlln
         WW1sdVlYSjVJR2QxYm1zK0FBRUNBenhzYjNSeklHOW1JR0pwYm1GeWVTQm5k
-        VzVyUGdBQkFnTTgKCWJHOTBjeUJ2WmlCaWFXNWhjbmtnWjNWdWF6NEFBUUlE
+        VzVyCgkJUGdBQkFnTThiRzkwY3lCdlppQmlhVzVoY25rZ1ozVnVhejRBQVFJ
-        UEd4dmRITWdiMllnWW1sdVlYSjVJR2QxYm1zK0FBRUNBenhzCgliM1J6SUc5
+        RFBHeHZkSE1nYjJZZ1ltbHVZWEo1CgkJSUdkMWJtcytBQUVDQXp4c2IzUnpJ
-        bUlHSnBibUZ5ZVNCbmRXNXJQZ0FCQWdNOGJHOTBjeUJ2WmlCaWFXNWhjbmtn
+        RzltSUdKcGJtRnllU0JuZFc1clBnQUJBZ004Ykc5MGN5QnZaaUJpCgkJYVc1
-        WjNWdWF6NEFBUUlEUEd4dgoJZEhNZ2IyWWdZbWx1WVhKNUlHZDFibXMrQUFF
+        aGNua2daM1Z1YXo0QUFRSURQR3h2ZEhNZ2IyWWdZbWx1WVhKNUlHZDFibXMr
-        Q0F6eHNiM1J6SUc5bUlHSnBibUZ5ZVNCbmRXNXJQZ0FCQWdNOGJHOTAKCWN5
+        QUFFQ0F6eHNiM1J6CgkJSUc5bUlHSnBibUZ5ZVNCbmRXNXJQZ0FCQWdNOGJH
-        QnZaaUJpYVc1aGNua2daM1Z1YXo0QUFRSURQR3h2ZEhNZ2IyWWdZbWx1WVhK
+        OTBjeUJ2WmlCaWFXNWhjbmtnWjNWdWF6NEFBUUlECgkJUEd4dmRITWdiMlln
-        NUlHZDFibXMrQUFFQ0F3PT0KCTwvZGF0YT4KCTxrZXk+w4ViZW5yYWE8L2tl
+        WW1sdVlYSjVJR2QxYm1zK0FBRUNBdz09CgkJPC9kYXRhPgoJPC9hcnJheT4K
-        eT4KCTxzdHJpbmc+VGhhdCB3YXMgYSB1bmljb2RlIGtleS48L3N0cmluZz4K
+        CTxrZXk+c29tZURhdGE8L2tleT4KCTxkYXRhPgoJUEdKcGJtRnllU0JuZFc1
-        PC9kaWN0Pgo8L3BsaXN0Pgo='''),
+        clBnPT0KCTwvZGF0YT4KCTxrZXk+c29tZU1vcmVEYXRhPC9rZXk+Cgk8ZGF0
+        YT4KCVBHeHZkSE1nYjJZZ1ltbHVZWEo1SUdkMWJtcytBQUVDQXp4c2IzUnpJ
+        RzltSUdKcGJtRnllU0JuZFc1clBnQUJBZ004CgliRzkwY3lCdlppQmlhVzVo
+        Y25rZ1ozVnVhejRBQVFJRFBHeHZkSE1nYjJZZ1ltbHVZWEo1SUdkMWJtcytB
+        QUVDQXp4cwoJYjNSeklHOW1JR0pwYm1GeWVTQm5kVzVyUGdBQkFnTThiRzkw
+        Y3lCdlppQmlhVzVoY25rZ1ozVnVhejRBQVFJRFBHeHYKCWRITWdiMllnWW1s
+        dVlYSjVJR2QxYm1zK0FBRUNBenhzYjNSeklHOW1JR0pwYm1GeWVTQm5kVzVy
+        UGdBQkFnTThiRzkwCgljeUJ2WmlCaWFXNWhjbmtnWjNWdWF6NEFBUUlEUEd4
+        dmRITWdiMllnWW1sdVlYSjVJR2QxYm1zK0FBRUNBdz09Cgk8L2RhdGE+Cgk8
+        a2V5PsOFYmVucmFhPC9rZXk+Cgk8c3RyaW5nPlRoYXQgd2FzIGEgdW5pY29k
+        ZSBrZXkuPC9zdHJpbmc+CjwvZGljdD4KPC9wbGlzdD4K'''),
     plistlib.FMT_BINARY: binascii.a2b_base64(b'''
-        YnBsaXN0MDDfEA8BAgMEBQYHCAkKCwwNDg8QERImJy0uLzAxMjM1NDZXYUJp
+        YnBsaXN0MDDfEBABAgMEBQYHCAkKCwwNDg8QERITFCgpLzAxMjM0NTc2OFdh
-        Z0ludFVhRGF0ZVVhRGljdFZhRmxvYXRVYUxpc3RfEA9hTmVnYXRpdmVCaWdJ
+        QmlnSW50WGFCaWdJbnQyVWFEYXRlVWFEaWN0VmFGbG9hdFVhTGlzdF8QD2FO
-        bnRcYU5lZ2F0aXZlSW50V2FTdHJpbmdbYW5FbXB0eURpY3RbYW5FbXB0eUxp
+        ZWdhdGl2ZUJpZ0ludFxhTmVnYXRpdmVJbnRXYVN0cmluZ1thbkVtcHR5RGlj
-        c3RVYW5JbnRabmVzdGVkRGF0YVhzb21lRGF0YVxzb21lTW9yZURhdGFnAMUA
+        dFthbkVtcHR5TGlzdFVhbkludFpuZXN0ZWREYXRhWHNvbWVEYXRhXHNvbWVN
-        YgBlAG4AcgBhAGETf////////9QzQZy5ffQAAADVExQVFhcYGRobHFthRmFs
+        b3JlRGF0YWcAxQBiAGUAbgByAGEAYRN/////////1BQAAAAAAAAAAIAAAAAA
-        c2VWYWx1ZVphVHJ1ZVZhbHVlXWFVbmljb2RlVmFsdWVdYW5vdGhlclN0cmlu
+        AAAsM0GcuX30AAAA1RUWFxgZGhscHR5bYUZhbHNlVmFsdWVaYVRydWVWYWx1
-        Z1pkZWVwZXJEaWN0CAlrAE0A5ABzAHMAaQBnACwAIABNAGEA318QFTxoZWxs
+        ZV1hVW5pY29kZVZhbHVlXWFub3RoZXJTdHJpbmdaZGVlcGVyRGljdAgJawBN
-        byAmICdoaScgdGhlcmUhPtMdHh8gISJRYVFiUWMQESNAQEAAAAAAAKMjJCUQ
+        AOQAcwBzAGkAZwAsACAATQBhAN9fEBU8aGVsbG8gJiAnaGknIHRoZXJlIT7T
-        ARACVHRleHQjP+AAAAAAAAClKCkqIStRQVFCEAyjIyQsEAMT////7V+g4AAT
+        HyAhIiMkUWFRYlFjEBEjQEBAAAAAAACjJSYnEAEQAlR0ZXh0Iz/gAAAAAAAA
-        //////////tWRG9vZGFo0KARAtihNE8Q+jxsb3RzIG9mIGJpbmFyeSBndW5r
+        pSorLCMtUUFRQhAMoyUmLhADE////+1foOAAE//////////7VkRvb2RhaNCg
-        PgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5
+        EQLYoTZPEPo8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmlu
-        IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBvZiBi
+        YXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBv
-        aW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3Rz
+        ZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxs
-        IG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQID
+        b3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4A
-        PGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5r
+        AQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBn
-        PgABAgNNPGJpbmFyeSBndW5rPl8QF1RoYXQgd2FzIGEgdW5pY29kZSBrZXku
+        dW5rPgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDTTxiaW5hcnkgZ3Vu
-        AAgAKQAxADcAPQBEAEoAXABpAHEAfQCJAI8AmgCjALAAvwDIANEA3ADoAPMB
+        az5fEBdUaGF0IHdhcyBhIHVuaWNvZGUga2V5LgAIACsAMwA8AEIASABPAFUA
-        AQEPARoBGwEcATMBSwFSAVQBVgFYAVoBYwFnAWkBawFwAXkBfwGBAYMBhQGJ
+        ZwB0AHwAiACUAJoApQCuALsAygDTAOQA7QD4AQQBDwEdASsBNgE3ATgBTwFn
-        AYsBlAGdAaQBpQGmAakBqwKoArYAAAAAAAACAQAAAAAAAAA3AAAAAAAAAAAA
+        AW4BcAFyAXQBdgF/AYMBhQGHAYwBlQGbAZ0BnwGhAaUBpwGwAbkBwAHBAcIB
-        AAAAAAAC0A=='''),
+        xQHHAsQC0gAAAAAAAAIBAAAAAAAAADkAAAAAAAAAAAAAAAAAAALs'''),
 }
 
 
@@ -106,6 +107,7 @@ class TestPlistlib(unittest.TestCase):
             aFloat = 0.5,
             anInt = 728,
             aBigInt = 2 ** 63 - 44,
+            aBigInt2 = 2 ** 63 + 44,
             aNegativeInt = -5,
             aNegativeBigInt = -80000000000,
             aDict=dict(
@@ -152,7 +154,7 @@ class TestPlistlib(unittest.TestCase):
 
     def test_int(self):
         for pl in [0, 2**8-1, 2**8, 2**16-1, 2**16, 2**32-1, 2**32,
-                   2**63-1, 1, -2**63]:
+                   2**63-1, 2**64-1, 1, -2**63]:
             for fmt in ALL_FORMATS:
                 with self.subTest(pl=pl, fmt=fmt):
                     data = plistlib.dumps(pl, fmt=fmt)

Mac/Tools/plistlib_generate_testdata.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from Cocoa import NSMutableDictionary, NSMutableArray, NSString, NSDate
+from Cocoa import NSMutableDictionary, NSMutableArray, NSString, NSDate, NSNumber
 from Cocoa import NSPropertyListSerialization, NSPropertyListOpenStepFormat
 from Cocoa import NSPropertyListXMLFormat_v1_0, NSPropertyListBinaryFormat_v1_0
 from Cocoa import CFUUIDCreateFromString, NSNull, NSUUID, CFPropertyListCreateData
@@ -30,6 +30,7 @@ def main():
 
     seconds = datetime.datetime(2004, 10, 26, 10, 33, 33, tzinfo=datetime.timezone(datetime.timedelta(0))).timestamp()
     pl[nsstr('aBigInt')] = 2 ** 63 - 44
+    pl[nsstr('aBigInt2')] = NSNumber.numberWithUnsignedLongLong_(2 ** 63 + 44)
     pl[nsstr('aDate')] = NSDate.dateWithTimeIntervalSince1970_(seconds)
 
     pl[nsstr('aDict')] = d = OrderedDict()

Misc/NEWS

@@ -109,6 +109,10 @@ Library
 - Issue #15304: concurrent.futures.wait() can block forever even if
   Futures have completed. Patch by Glenn Langford.
 
+- Issue #14455: plistlib: fix serializing integers integers in the range
+  of an unsigned long long but outside of the range of signed long long for
+  binary plist files.
+
 IDLE
 ----