mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-08 20:05:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

gh-132775: Support Fallbacks in _PyObject_GetXIData() (gh-133482)

Browse source 
It now supports a "full" fallback to _PyFunction_GetXIData() and then `_PyPickle_GetXIData()`.  There's also room for other fallback modes if that later makes sense.
This commit is contained in:
Eric Snow 2025-05-21 07:23:48 -06:00 • committed by GitHub
parent 0c5a8b0b55
commit 88f8102a8f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 581 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

25
Include/internal/pycore_crossinterp.h
View file

@ -131,7 +131,23 @@ PyAPI_FUNC(void) _PyXIData_Clear(PyInterpreterState *, _PyXIData_t *);
/* getting cross-interpreter data */
typedef int (*xidatafunc)(PyThreadState *tstate, PyObject *, _PyXIData_t *);
typedef int xidata_fallback_t;
#define _PyXIDATA_XIDATA_ONLY (0)
#define _PyXIDATA_FULL_FALLBACK (1)
// Technically, we don't need two different function types;
// we could go with just the fallback one. However, only container
// types like tuple need it, so always having the extra arg would be
// a bit unfortunate. It's also nice to be able to clearly distinguish
// between types that might call _PyObject_GetXIData() and those that won't.
//
typedef int (*xidatafunc)(PyThreadState *, PyObject *, _PyXIData_t *);
typedef int (*xidatafbfunc)(
PyThreadState *, PyObject *, xidata_fallback_t, _PyXIData_t *);
typedef struct {
xidatafunc basic;
xidatafbfunc fallback;
} _PyXIData_getdata_t;
PyAPI_FUNC(PyObject *) _PyXIData_GetNotShareableErrorType(PyThreadState *);
PyAPI_FUNC(void) _PyXIData_SetNotShareableError(PyThreadState *, const char *);
@ -140,16 +156,21 @@ PyAPI_FUNC(void) _PyXIData_FormatNotShareableError(
const char *,
...);
PyAPI_FUNC(xidatafunc) _PyXIData_Lookup(
PyAPI_FUNC(_PyXIData_getdata_t) _PyXIData_Lookup(
PyThreadState *,
PyObject *);
PyAPI_FUNC(int) _PyObject_CheckXIData(
PyThreadState *,
PyObject *);
PyAPI_FUNC(int) _PyObject_GetXIDataNoFallback(
PyThreadState *,
PyObject *,
_PyXIData_t *);
PyAPI_FUNC(int) _PyObject_GetXIData(
PyThreadState *,
PyObject *,
xidata_fallback_t,
_PyXIData_t *);
// _PyObject_GetXIData() for bytes

4
Include/internal/pycore_crossinterp_data_registry.h
View file

@ -17,7 +17,7 @@ typedef struct _xid_regitem {
/* This is NULL for builtin types. */
PyObject *weakref;
size_t refcount;
xidatafunc getdata;
_PyXIData_getdata_t getdata;
} _PyXIData_regitem_t;
typedef struct {
@ -30,7 +30,7 @@ typedef struct {
PyAPI_FUNC(int) _PyXIData_RegisterClass(
PyThreadState *,
PyTypeObject *,
xidatafunc);
_PyXIData_getdata_t);
PyAPI_FUNC(int) _PyXIData_UnregisterClass(
PyThreadState *,
PyTypeObject *);

2
Lib/test/support/import_helper.py
View file

@ -438,5 +438,5 @@ def ensure_module_imported(name, *, clearnone=True):
if sys.modules.get(name) is not None:
mod = sys.modules[name]
else:
mod, _, _ = _force_import(name, False, True, clearnone)
mod, _, _ = _ensure_module(name, False, True, clearnone)
return mod

511
Lib/test/test_crossinterp.py
View file

@ -5,6 +5,7 @@ import itertools
import sys
import types
import unittest
import warnings
from test.support import import_helper
@ -16,13 +17,281 @@ from test import _code_definitions as code_defs
from test import _crossinterp_definitions as defs
BUILTIN_TYPES = [o for _, o in __builtins__.items()
if isinstance(o, type)]
EXCEPTION_TYPES = [cls for cls in BUILTIN_TYPES
@contextlib.contextmanager
def ignore_byteswarning():
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=BytesWarning)
yield
# builtin types
BUILTINS_TYPES = [o for _, o in __builtins__.items() if isinstance(o, type)]
EXCEPTION_TYPES = [cls for cls in BUILTINS_TYPES
if issubclass(cls, BaseException)]
OTHER_TYPES = [o for n, o in vars(types).items()
if (isinstance(o, type) and
n not in ('DynamicClassAttribute', '_GeneratorWrapper'))]
n not in ('DynamicClassAttribute', '_GeneratorWrapper'))]
BUILTIN_TYPES = [
*BUILTINS_TYPES,
*OTHER_TYPES,
]
# builtin exceptions
try:
raise Exception
except Exception as exc:
CAUGHT = exc
EXCEPTIONS_WITH_SPECIAL_SIG = {
BaseExceptionGroup: (lambda msg: (msg, [CAUGHT])),
ExceptionGroup: (lambda msg: (msg, [CAUGHT])),
UnicodeError: (lambda msg: (None, msg, None, None, None)),
UnicodeEncodeError: (lambda msg: ('utf-8', '', 1, 3, msg)),
UnicodeDecodeError: (lambda msg: ('utf-8', b'', 1, 3, msg)),
UnicodeTranslateError: (lambda msg: ('', 1, 3, msg)),
}
BUILTIN_EXCEPTIONS = [
*(cls(*sig('error!')) for cls, sig in EXCEPTIONS_WITH_SPECIAL_SIG.items()),
*(cls('error!') for cls in EXCEPTION_TYPES
if cls not in EXCEPTIONS_WITH_SPECIAL_SIG),
]
# other builtin objects
METHOD = defs.SpamOkay().okay
BUILTIN_METHOD = [].append
METHOD_DESCRIPTOR_WRAPPER = str.join
METHOD_WRAPPER = object().__str__
WRAPPER_DESCRIPTOR = object.__init__
BUILTIN_WRAPPERS = {
METHOD: types.MethodType,
BUILTIN_METHOD: types.BuiltinMethodType,
dict.__dict__['fromkeys']: types.ClassMethodDescriptorType,
types.FunctionType.__code__: types.GetSetDescriptorType,
types.FunctionType.__globals__: types.MemberDescriptorType,
METHOD_DESCRIPTOR_WRAPPER: types.MethodDescriptorType,
METHOD_WRAPPER: types.MethodWrapperType,
WRAPPER_DESCRIPTOR: types.WrapperDescriptorType,
staticmethod(defs.SpamOkay.okay): None,
classmethod(defs.SpamOkay.okay): None,
property(defs.SpamOkay.okay): None,
}
BUILTIN_FUNCTIONS = [
# types.BuiltinFunctionType
len,
sys.is_finalizing,
sys.exit,
_testinternalcapi.get_crossinterp_data,
]
assert 'emptymod' not in sys.modules
with import_helper.ready_to_import('emptymod', ''):
import emptymod as EMPTYMOD
MODULES = [
sys,
defs,
unittest,
EMPTYMOD,
]
OBJECT = object()
EXCEPTION = Exception()
LAMBDA = (lambda: None)
BUILTIN_SIMPLE = [
OBJECT,
# singletons
None,
True,
False,
Ellipsis,
NotImplemented,
# bytes
*(i.to_bytes(2, 'little', signed=True)
for i in range(-1, 258)),
# str
'hello world',
'你好世界',
'',
# int
sys.maxsize + 1,
sys.maxsize,
-sys.maxsize - 1,
-sys.maxsize - 2,
*range(-1, 258),
2**1000,
# float
0.0,
1.1,
-1.0,
0.12345678,
-0.12345678,
]
TUPLE_EXCEPTION = (0, 1.0, EXCEPTION)
TUPLE_OBJECT = (0, 1.0, OBJECT)
TUPLE_NESTED_EXCEPTION = (0, 1.0, (EXCEPTION,))
TUPLE_NESTED_OBJECT = (0, 1.0, (OBJECT,))
MEMORYVIEW_EMPTY = memoryview(b'')
MEMORYVIEW_NOT_EMPTY = memoryview(b'spam'*42)
MAPPING_PROXY_EMPTY = types.MappingProxyType({})
BUILTIN_CONTAINERS = [
# tuple (flat)
(),
(1,),
("hello", "world", ),
(1, True, "hello"),
TUPLE_EXCEPTION,
TUPLE_OBJECT,
# tuple (nested)
((1,),),
((1, 2), (3, 4)),
((1, 2), (3, 4), (5, 6)),
TUPLE_NESTED_EXCEPTION,
TUPLE_NESTED_OBJECT,
# buffer
MEMORYVIEW_EMPTY,
MEMORYVIEW_NOT_EMPTY,
# list
[],
[1, 2, 3],
[[1], (2,), {3: 4}],
# dict
{},
{1: 7, 2: 8, 3: 9},
{1: [1], 2: (2,), 3: {3: 4}},
# set
set(),
{1, 2, 3},
{frozenset({1}), (2,)},
# frozenset
frozenset([]),
frozenset({frozenset({1}), (2,)}),
# bytearray
bytearray(b''),
# other
MAPPING_PROXY_EMPTY,
types.SimpleNamespace(),
]
ns = {}
exec("""
try:
raise Exception
except Exception as exc:
TRACEBACK = exc.__traceback__
FRAME = TRACEBACK.tb_frame
""", ns, ns)
BUILTIN_OTHER = [
# types.CellType
types.CellType(),
# types.FrameType
ns['FRAME'],
# types.TracebackType
ns['TRACEBACK'],
]
del ns
# user-defined objects
USER_TOP_INSTANCES = [c(*a) for c, a in defs.TOP_CLASSES.items()]
USER_NESTED_INSTANCES = [c(*a) for c, a in defs.NESTED_CLASSES.items()]
USER_INSTANCES = [
*USER_TOP_INSTANCES,
*USER_NESTED_INSTANCES,
]
USER_EXCEPTIONS = [
defs.MimimalError('error!'),
]
# shareable objects
TUPLES_WITHOUT_EQUALITY = [
TUPLE_EXCEPTION,
TUPLE_OBJECT,
TUPLE_NESTED_EXCEPTION,
TUPLE_NESTED_OBJECT,
]
_UNSHAREABLE_SIMPLE = [
Ellipsis,
NotImplemented,
OBJECT,
sys.maxsize + 1,
-sys.maxsize - 2,
2**1000,
]
with ignore_byteswarning():
_SHAREABLE_SIMPLE = [o for o in BUILTIN_SIMPLE
if o not in _UNSHAREABLE_SIMPLE]
_SHAREABLE_CONTAINERS = [
*(o for o in BUILTIN_CONTAINERS if type(o) is memoryview),
*(o for o in BUILTIN_CONTAINERS
if type(o) is tuple and o not in TUPLES_WITHOUT_EQUALITY),
]
_UNSHAREABLE_CONTAINERS = [o for o in BUILTIN_CONTAINERS
if o not in _SHAREABLE_CONTAINERS]
SHAREABLE = [
*_SHAREABLE_SIMPLE,
*_SHAREABLE_CONTAINERS,
]
NOT_SHAREABLE = [
*_UNSHAREABLE_SIMPLE,
*_UNSHAREABLE_CONTAINERS,
*BUILTIN_TYPES,
*BUILTIN_WRAPPERS,
*BUILTIN_EXCEPTIONS,
*BUILTIN_FUNCTIONS,
*MODULES,
*BUILTIN_OTHER,
# types.CodeType
*(f.__code__ for f in defs.FUNCTIONS),
*(f.__code__ for f in defs.FUNCTION_LIKE),
# types.FunctionType
*defs.FUNCTIONS,
defs.SpamOkay.okay,
LAMBDA,
*defs.FUNCTION_LIKE,
# coroutines and generators
*defs.FUNCTION_LIKE_APPLIED,
# user classes
*defs.CLASSES,
*USER_INSTANCES,
# user exceptions
*USER_EXCEPTIONS,
]
# pickleable objects
PICKLEABLE = [
*BUILTIN_SIMPLE,
*(o for o in BUILTIN_CONTAINERS if o not in [
MEMORYVIEW_EMPTY,
MEMORYVIEW_NOT_EMPTY,
MAPPING_PROXY_EMPTY,
] or type(o) is dict),
*BUILTINS_TYPES,
*BUILTIN_EXCEPTIONS,
*BUILTIN_FUNCTIONS,
*defs.TOP_FUNCTIONS,
defs.SpamOkay.okay,
*defs.FUNCTION_LIKE,
*defs.TOP_CLASSES,
*USER_TOP_INSTANCES,
*USER_EXCEPTIONS,
# from OTHER_TYPES
types.NoneType,
types.EllipsisType,
types.NotImplementedType,
types.GenericAlias,
types.UnionType,
types.SimpleNamespace,
# from BUILTIN_WRAPPERS
METHOD,
BUILTIN_METHOD,
METHOD_DESCRIPTOR_WRAPPER,
METHOD_WRAPPER,
WRAPPER_DESCRIPTOR,
]
assert not any(isinstance(o, types.MappingProxyType) for o in PICKLEABLE)
# helpers
DEFS = defs
with open(code_defs.__file__) as infile:
@ -111,6 +380,77 @@ class _GetXIDataTests(unittest.TestCase):
MODE = None
def assert_functions_equal(self, func1, func2):
assert type(func1) is types.FunctionType, repr(func1)
assert type(func2) is types.FunctionType, repr(func2)
self.assertEqual(func1.__name__, func2.__name__)
self.assertEqual(func1.__code__, func2.__code__)
self.assertEqual(func1.__defaults__, func2.__defaults__)
self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__)
# We don't worry about __globals__ for now.
def assert_exc_args_equal(self, exc1, exc2):
args1 = exc1.args
args2 = exc2.args
if isinstance(exc1, ExceptionGroup):
self.assertIs(type(args1), type(args2))
self.assertEqual(len(args1), 2)
self.assertEqual(len(args1), len(args2))
self.assertEqual(args1[0], args2[0])
group1 = args1[1]
group2 = args2[1]
self.assertEqual(len(group1), len(group2))
for grouped1, grouped2 in zip(group1, group2):
# Currently the "extra" attrs are not preserved
# (via __reduce__).
self.assertIs(type(exc1), type(exc2))
self.assert_exc_equal(grouped1, grouped2)
else:
self.assertEqual(args1, args2)
def assert_exc_equal(self, exc1, exc2):
self.assertIs(type(exc1), type(exc2))
if type(exc1).__eq__ is not object.__eq__:
self.assertEqual(exc1, exc2)
self.assert_exc_args_equal(exc1, exc2)
# XXX For now we do not preserve tracebacks.
if exc1.__traceback__ is not None:
self.assertEqual(exc1.__traceback__, exc2.__traceback__)
self.assertEqual(
getattr(exc1, '__notes__', None),
getattr(exc2, '__notes__', None),
)
# We assume there are no cycles.
if exc1.__cause__ is None:
self.assertIs(exc1.__cause__, exc2.__cause__)
else:
self.assert_exc_equal(exc1.__cause__, exc2.__cause__)
if exc1.__context__ is None:
self.assertIs(exc1.__context__, exc2.__context__)
else:
self.assert_exc_equal(exc1.__context__, exc2.__context__)
def assert_equal_or_equalish(self, obj, expected):
cls = type(expected)
if cls.__eq__ is not object.__eq__:
self.assertEqual(obj, expected)
elif cls is types.FunctionType:
self.assert_functions_equal(obj, expected)
elif isinstance(expected, BaseException):
self.assert_exc_equal(obj, expected)
elif cls is types.MethodType:
raise NotImplementedError(cls)
elif cls is types.BuiltinMethodType:
raise NotImplementedError(cls)
elif cls is types.MethodWrapperType:
raise NotImplementedError(cls)
elif cls.__bases__ == (object,):
self.assertEqual(obj.__dict__, expected.__dict__)
else:
raise NotImplementedError(cls)
def get_xidata(self, obj, *, mode=None):
mode = self._resolve_mode(mode)
return _testinternalcapi.get_crossinterp_data(obj, mode)
@ -126,35 +466,37 @@ class _GetXIDataTests(unittest.TestCase):
def assert_roundtrip_identical(self, values, *, mode=None):
mode = self._resolve_mode(mode)
for obj in values:
with self.subTest(obj):
with self.subTest(repr(obj)):
got = self._get_roundtrip(obj, mode)
self.assertIs(got, obj)
def assert_roundtrip_equal(self, values, *, mode=None, expecttype=None):
mode = self._resolve_mode(mode)
for obj in values:
with self.subTest(obj):
with self.subTest(repr(obj)):
got = self._get_roundtrip(obj, mode)
self.assertEqual(got, obj)
if got is obj:
continue
self.assertIs(type(got),
type(obj) if expecttype is None else expecttype)
self.assert_equal_or_equalish(got, obj)
def assert_roundtrip_equal_not_identical(self, values, *,
mode=None, expecttype=None):
mode = self._resolve_mode(mode)
for obj in values:
with self.subTest(obj):
with self.subTest(repr(obj)):
got = self._get_roundtrip(obj, mode)
self.assertIsNot(got, obj)
self.assertIs(type(got),
type(obj) if expecttype is None else expecttype)
self.assertEqual(got, obj)
self.assert_equal_or_equalish(got, obj)
def assert_roundtrip_not_equal(self, values, *,
mode=None, expecttype=None):
mode = self._resolve_mode(mode)
for obj in values:
with self.subTest(obj):
with self.subTest(repr(obj)):
got = self._get_roundtrip(obj, mode)
self.assertIsNot(got, obj)
self.assertIs(type(got),
@ -164,7 +506,7 @@ class _GetXIDataTests(unittest.TestCase):
def assert_not_shareable(self, values, exctype=None, *, mode=None):
mode = self._resolve_mode(mode)
for obj in values:
with self.subTest(obj):
with self.subTest(repr(obj)):
with self.assertRaises(NotShareableError) as cm:
_testinternalcapi.get_crossinterp_data(obj, mode)
if exctype is not None:
@ -182,49 +524,26 @@ class PickleTests(_GetXIDataTests):
MODE = 'pickle'
def test_shareable(self):
self.assert_roundtrip_equal([
# singletons
None,
True,
False,
# bytes
*(i.to_bytes(2, 'little', signed=True)
for i in range(-1, 258)),
# str
'hello world',
'你好世界',
'',
# int
sys.maxsize,
-sys.maxsize - 1,
*range(-1, 258),
# float
0.0,
1.1,
-1.0,
0.12345678,
-0.12345678,
# tuple
(),
(1,),
("hello", "world", ),
(1, True, "hello"),
((1,),),
((1, 2), (3, 4)),
((1, 2), (3, 4), (5, 6)),
])
# not shareable using xidata
self.assert_roundtrip_equal([
# int
sys.maxsize + 1,
-sys.maxsize - 2,
2**1000,
# tuple
(0, 1.0, []),
(0, 1.0, {}),
(0, 1.0, ([],)),
(0, 1.0, ({},)),
])
with ignore_byteswarning():
for obj in SHAREABLE:
if obj in PICKLEABLE:
self.assert_roundtrip_equal([obj])
else:
self.assert_not_shareable([obj])
def test_not_shareable(self):
with ignore_byteswarning():
for obj in NOT_SHAREABLE:
if type(obj) is types.MappingProxyType:
self.assert_not_shareable([obj])
elif obj in PICKLEABLE:
with self.subTest(repr(obj)):
# We don't worry about checking the actual value.
# The other tests should cover that well enough.
got = self.get_roundtrip(obj)
self.assertIs(type(got), type(obj))
else:
self.assert_not_shareable([obj])
def test_list(self):
self.assert_roundtrip_equal_not_identical([
@ -266,7 +585,7 @@ class PickleTests(_GetXIDataTests):
if cls not in defs.CLASSES_WITHOUT_EQUALITY:
continue
instances.append(cls(*args))
self.assert_roundtrip_not_equal(instances)
self.assert_roundtrip_equal(instances)
def assert_class_defs_other_pickle(self, defs, mod):
# Pickle relative to a different module than the original.
@ -286,7 +605,7 @@ class PickleTests(_GetXIDataTests):
instances = []
for cls, args in defs.TOP_CLASSES.items():
with self.subTest(cls):
with self.subTest(repr(cls)):
setattr(mod, cls.__name__, cls)
xid = self.get_xidata(cls)
inst = cls(*args)
@ -295,7 +614,7 @@ class PickleTests(_GetXIDataTests):
(cls, xid, inst, instxid))
for cls, xid, inst, instxid in instances:
with self.subTest(cls):
with self.subTest(repr(cls)):
delattr(mod, cls.__name__)
if fail:
with self.assertRaises(NotShareableError):
@ -403,13 +722,13 @@ class PickleTests(_GetXIDataTests):
def assert_func_defs_other_pickle(self, defs, mod):
# Pickle relative to a different module than the original.
for func in defs.TOP_FUNCTIONS:
assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__))
assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),)
self.assert_not_shareable(defs.TOP_FUNCTIONS)
def assert_func_defs_other_unpickle(self, defs, mod, *, fail=False):
# Unpickle relative to a different module than the original.
for func in defs.TOP_FUNCTIONS:
assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__))
assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),)
captured = []
for func in defs.TOP_FUNCTIONS:
@ -434,7 +753,7 @@ class PickleTests(_GetXIDataTests):
self.assert_not_shareable(defs.TOP_FUNCTIONS)
def test_user_function_normal(self):
# self.assert_roundtrip_equal(defs.TOP_FUNCTIONS)
self.assert_roundtrip_equal(defs.TOP_FUNCTIONS)
self.assert_func_defs_same(defs)
def test_user_func_in___main__(self):
@ -505,7 +824,7 @@ class PickleTests(_GetXIDataTests):
# exceptions
def test_user_exception_normal(self):
self.assert_roundtrip_not_equal([
self.assert_roundtrip_equal([
defs.MimimalError('error!'),
])
self.assert_roundtrip_equal_not_identical([
@ -521,7 +840,7 @@ class PickleTests(_GetXIDataTests):
special = {
BaseExceptionGroup: (msg, [caught]),
ExceptionGroup: (msg, [caught]),
# UnicodeError: (None, msg, None, None, None),
UnicodeError: (None, msg, None, None, None),
UnicodeEncodeError: ('utf-8', '', 1, 3, msg),
UnicodeDecodeError: ('utf-8', b'', 1, 3, msg),
UnicodeTranslateError: ('', 1, 3, msg),
@ -531,7 +850,7 @@ class PickleTests(_GetXIDataTests):
args = special.get(cls) or (msg,)
exceptions.append(cls(*args))
self.assert_roundtrip_not_equal(exceptions)
self.assert_roundtrip_equal(exceptions)
class MarshalTests(_GetXIDataTests):
@ -576,7 +895,7 @@ class MarshalTests(_GetXIDataTests):
'',
])
self.assert_not_shareable([
object(),
OBJECT,
types.SimpleNamespace(),
])
@ -647,10 +966,7 @@ class MarshalTests(_GetXIDataTests):
shareable = [
StopIteration,
]
types = [
*BUILTIN_TYPES,
*OTHER_TYPES,
]
types = BUILTIN_TYPES
self.assert_not_shareable(cls for cls in types
if cls not in shareable)
self.assert_roundtrip_identical(cls for cls in types
@ -763,7 +1079,7 @@ class ShareableFuncTests(_GetXIDataTests):
MODE = 'func'
def test_stateless(self):
self.assert_roundtrip_not_equal([
self.assert_roundtrip_equal([
*defs.STATELESS_FUNCTIONS,
# Generators can be stateless too.
*defs.FUNCTION_LIKE,
@ -912,10 +1228,49 @@ class ShareableScriptTests(PureShareableScriptTests):
], expecttype=types.CodeType)
class ShareableFallbackTests(_GetXIDataTests):
MODE = 'fallback'
def test_shareable(self):
self.assert_roundtrip_equal(SHAREABLE)
def test_not_shareable(self):
okay = [
*PICKLEABLE,
*defs.STATELESS_FUNCTIONS,
LAMBDA,
]
ignored = [
*TUPLES_WITHOUT_EQUALITY,
OBJECT,
METHOD,
BUILTIN_METHOD,
METHOD_WRAPPER,
]
with ignore_byteswarning():
self.assert_roundtrip_equal([
*(o for o in NOT_SHAREABLE
if o in okay and o not in ignored
and o is not MAPPING_PROXY_EMPTY),
])
self.assert_roundtrip_not_equal([
*(o for o in NOT_SHAREABLE
if o in ignored and o is not MAPPING_PROXY_EMPTY),
])
self.assert_not_shareable([
*(o for o in NOT_SHAREABLE if o not in okay),
MAPPING_PROXY_EMPTY,
])
class ShareableTypeTests(_GetXIDataTests):
MODE = 'xidata'
def test_shareable(self):
self.assert_roundtrip_equal(SHAREABLE)
def test_singletons(self):
self.assert_roundtrip_identical([
None,
@ -983,8 +1338,8 @@ class ShareableTypeTests(_GetXIDataTests):
def test_tuples_containing_non_shareable_types(self):
non_shareables = [
Exception(),
object(),
EXCEPTION,
OBJECT,
]
for s in non_shareables:
value = tuple([0, 1.0, s])
@ -999,6 +1354,9 @@ class ShareableTypeTests(_GetXIDataTests):
# The rest are not shareable.
def test_not_shareable(self):
self.assert_not_shareable(NOT_SHAREABLE)
def test_object(self):
self.assert_not_shareable([
object(),
@ -1015,12 +1373,12 @@ class ShareableTypeTests(_GetXIDataTests):
for func in defs.FUNCTIONS:
assert type(func) is types.FunctionType, func
assert type(defs.SpamOkay.okay) is types.FunctionType, func
assert type(lambda: None) is types.LambdaType
assert type(LAMBDA) is types.LambdaType
self.assert_not_shareable([
*defs.FUNCTIONS,
defs.SpamOkay.okay,
(lambda: None),
LAMBDA,
])
def test_builtin_function(self):
@ -1085,10 +1443,7 @@ class ShareableTypeTests(_GetXIDataTests):
self.assert_not_shareable(instances)
def test_builtin_type(self):
self.assert_not_shareable([
*BUILTIN_TYPES,
*OTHER_TYPES,
])
self.assert_not_shareable(BUILTIN_TYPES)
def test_exception(self):
self.assert_not_shareable([
@ -1127,7 +1482,7 @@ class ShareableTypeTests(_GetXIDataTests):
""", ns, ns)
self.assert_not_shareable([
types.MappingProxyType({}),
MAPPING_PROXY_EMPTY,
types.SimpleNamespace(),
# types.CellType
types.CellType(),

8
Modules/_interpchannelsmodule.c
View file

@ -1779,7 +1779,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj,
PyThread_release_lock(mutex);
return -1;
}
if (_PyObject_GetXIData(tstate, obj, data) != 0) {
if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) {
PyThread_release_lock(mutex);
GLOBAL_FREE(data);
return -1;
@ -2694,7 +2694,7 @@ add_channelid_type(PyObject *mod)
Py_DECREF(cls);
return NULL;
}
if (ensure_xid_class(cls, _channelid_shared) < 0) {
if (ensure_xid_class(cls, GETDATA(_channelid_shared)) < 0) {
Py_DECREF(cls);
return NULL;
}
@ -2797,12 +2797,12 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv)
// Add and register the types.
state->send_channel_type = (PyTypeObject *)Py_NewRef(send);
state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv);
if (ensure_xid_class(send, _channelend_shared) < 0) {
if (ensure_xid_class(send, GETDATA(_channelend_shared)) < 0) {
Py_CLEAR(state->send_channel_type);
Py_CLEAR(state->recv_channel_type);
return -1;
}
if (ensure_xid_class(recv, _channelend_shared) < 0) {
if (ensure_xid_class(recv, GETDATA(_channelend_shared)) < 0) {
(void)clear_xid_class(state->send_channel_type);
Py_CLEAR(state->send_channel_type);
Py_CLEAR(state->recv_channel_type);

4
Modules/_interpqueuesmodule.c
View file

@ -1143,7 +1143,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop)
_queue_unmark_waiter(queue, queues->mutex);
return -1;
}
if (_PyObject_GetXIData(tstate, obj, data) != 0) {
if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) {
_queue_unmark_waiter(queue, queues->mutex);
GLOBAL_FREE(data);
return -1;
@ -1270,7 +1270,7 @@ set_external_queue_type(module_state *state, PyTypeObject *queue_type)
}
// Add and register the new type.
if (ensure_xid_class(queue_type, _queueobj_shared) < 0) {
if (ensure_xid_class(queue_type, GETDATA(_queueobj_shared)) < 0) {
return -1;
}
state->queue_type = (PyTypeObject *)Py_NewRef(queue_type);

4
Modules/_interpreters_common.h
View file

@ -5,8 +5,10 @@
_RESOLVE_MODINIT_FUNC_NAME(NAME)
#define GETDATA(FUNC) ((_PyXIData_getdata_t){.basic=FUNC})
static int
ensure_xid_class(PyTypeObject *cls, xidatafunc getdata)
ensure_xid_class(PyTypeObject *cls, _PyXIData_getdata_t getdata)
{
PyThreadState *tstate = PyThreadState_Get();
return _PyXIData_RegisterClass(tstate, cls, getdata);

2
Modules/_interpretersmodule.c
View file

@ -286,7 +286,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state)
*p_state = cls;
// Register XID for the builtin memoryview type.
if (ensure_xid_class(&PyMemoryView_Type, _pybuffer_shared) < 0) {
if (ensure_xid_class(&PyMemoryView_Type, GETDATA(_pybuffer_shared)) < 0) {
return -1;
}
// We don't ever bother un-registering memoryview.

9
Modules/_testinternalcapi.c
View file

@ -1991,7 +1991,14 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs)
return NULL;
}
if (strcmp(mode, "xidata") == 0) {
if (_PyObject_GetXIData(tstate, obj, xidata) != 0) {
if (_PyObject_GetXIDataNoFallback(tstate, obj, xidata) != 0) {
goto error;
}
}
else if (strcmp(mode, "fallback") == 0) {
xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK;
if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0)
{
goto error;
}
}

89
Python/crossinterp.c
View file

@ -210,16 +210,16 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp,
/* cross-interpreter data */
/**************************/
/* registry of {type -> xidatafunc} */
/* registry of {type -> _PyXIData_getdata_t} */
/* For now we use a global registry of shareable classes. An
alternative would be to add a tp_* slot for a class's
xidatafunc. It would be simpler and more efficient. */
/* For now we use a global registry of shareable classes.
An alternative would be to add a tp_* slot for a class's
_PyXIData_getdata_t. It would be simpler and more efficient. */
static void xid_lookup_init(_PyXIData_lookup_t *);
static void xid_lookup_fini(_PyXIData_lookup_t *);
struct _dlcontext;
static xidatafunc lookup_getdata(struct _dlcontext *, PyObject *);
static _PyXIData_getdata_t lookup_getdata(struct _dlcontext *, PyObject *);
#include "crossinterp_data_lookup.h"
@ -343,7 +343,7 @@ _set_xid_lookup_failure(PyThreadState *tstate, PyObject *obj, const char *msg,
set_notshareableerror(tstate, cause, 0, msg);
}
else {
msg = "%S does not support cross-interpreter data";
msg = "%R does not support cross-interpreter data";
format_notshareableerror(tstate, cause, 0, msg, obj);
}
}
@ -356,8 +356,8 @@ _PyObject_CheckXIData(PyThreadState *tstate, PyObject *obj)
if (get_lookup_context(tstate, &ctx) < 0) {
return -1;
}
xidatafunc getdata = lookup_getdata(&ctx, obj);
if (getdata == NULL) {
_PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj);
if (getdata.basic == NULL && getdata.fallback == NULL) {
if (!_PyErr_Occurred(tstate)) {
_set_xid_lookup_failure(tstate, obj, NULL, NULL);
}
@ -388,9 +388,9 @@ _check_xidata(PyThreadState *tstate, _PyXIData_t *xidata)
return 0;
}
int
_PyObject_GetXIData(PyThreadState *tstate,
PyObject *obj, _PyXIData_t *xidata)
static int
_get_xidata(PyThreadState *tstate,
PyObject *obj, xidata_fallback_t fallback, _PyXIData_t *xidata)
{
PyInterpreterState *interp = tstate->interp;
@ -398,6 +398,7 @@ _PyObject_GetXIData(PyThreadState *tstate,
assert(xidata->obj == NULL);
if (xidata->data != NULL || xidata->obj != NULL) {
_PyErr_SetString(tstate, PyExc_ValueError, "xidata not cleared");
return -1;
}
// Call the "getdata" func for the object.
@ -406,8 +407,8 @@ _PyObject_GetXIData(PyThreadState *tstate,
return -1;
}
Py_INCREF(obj);
xidatafunc getdata = lookup_getdata(&ctx, obj);
if (getdata == NULL) {
_PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj);
if (getdata.basic == NULL && getdata.fallback == NULL) {
if (PyErr_Occurred()) {
Py_DECREF(obj);
return -1;
@ -419,7 +420,9 @@ _PyObject_GetXIData(PyThreadState *tstate,
}
return -1;
}
int res = getdata(tstate, obj, xidata);
int res = getdata.basic != NULL
? getdata.basic(tstate, obj, xidata)
: getdata.fallback(tstate, obj, fallback, xidata);
Py_DECREF(obj);
if (res != 0) {
PyObject *cause = _PyErr_GetRaisedException(tstate);
@ -439,6 +442,51 @@ _PyObject_GetXIData(PyThreadState *tstate,
return 0;
}
int
_PyObject_GetXIDataNoFallback(PyThreadState *tstate,
PyObject *obj, _PyXIData_t *xidata)
{
return _get_xidata(tstate, obj, _PyXIDATA_XIDATA_ONLY, xidata);
}
int
_PyObject_GetXIData(PyThreadState *tstate,
PyObject *obj, xidata_fallback_t fallback,
_PyXIData_t *xidata)
{
switch (fallback) {
case _PyXIDATA_XIDATA_ONLY:
return _get_xidata(tstate, obj, fallback, xidata);
case _PyXIDATA_FULL_FALLBACK:
if (_get_xidata(tstate, obj, fallback, xidata) == 0) {
return 0;
}
PyObject *exc = _PyErr_GetRaisedException(tstate);
if (PyFunction_Check(obj)) {
if (_PyFunction_GetXIData(tstate, obj, xidata) == 0) {
Py_DECREF(exc);
return 0;
}
_PyErr_Clear(tstate);
}
// We could try _PyMarshal_GetXIData() but we won't for now.
if (_PyPickle_GetXIData(tstate, obj, xidata) == 0) {
Py_DECREF(exc);
return 0;
}
// Raise the original exception.
_PyErr_SetRaisedException(tstate, exc);
return -1;
default:
#ifdef Py_DEBUG
Py_FatalError("unsupported xidata fallback option");
#endif
_PyErr_SetString(tstate, PyExc_SystemError,
"unsupported xidata fallback option");
return -1;
}
}
/* pickle C-API */
@ -1617,14 +1665,9 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp)
PyThreadState *tstate = _PyThreadState_GET();
assert(!PyErr_Occurred());
assert(code != _PyXI_ERR_NO_ERROR);
assert(code != _PyXI_ERR_UNCAUGHT_EXCEPTION);
switch (code) {
case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH;
case _PyXI_ERR_UNCAUGHT_EXCEPTION:
// There is nothing to apply.
#ifdef Py_DEBUG
Py_UNREACHABLE();
#endif
return 0;
case _PyXI_ERR_OTHER:
// XXX msg?
PyErr_SetNone(PyExc_InterpreterError);
@ -1649,7 +1692,7 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp)
break;
default:
#ifdef Py_DEBUG
Py_UNREACHABLE();
Py_FatalError("unsupported error code");
#else
PyErr_Format(PyExc_RuntimeError, "unsupported error code %d", code);
#endif
@ -1796,7 +1839,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value)
return -1;
}
PyThreadState *tstate = PyThreadState_Get();
if (_PyObject_GetXIData(tstate, value, item->xidata) != 0) {
if (_PyObject_GetXIDataNoFallback(tstate, value, item->xidata) != 0) {
PyMem_RawFree(item->xidata);
item->xidata = NULL;
// The caller may want to propagate PyExc_NotShareableError

62
Python/crossinterp_data_lookup.h
View file

@ -12,7 +12,8 @@ typedef _PyXIData_regitem_t dlregitem_t;
// forward
static void _xidregistry_init(dlregistry_t *);
static void _xidregistry_fini(dlregistry_t *);
static xidatafunc _lookup_getdata_from_registry(dlcontext_t *, PyObject *);
static _PyXIData_getdata_t _lookup_getdata_from_registry(
dlcontext_t *, PyObject *);
/* used in crossinterp.c */
@ -49,7 +50,7 @@ get_lookup_context(PyThreadState *tstate, dlcontext_t *res)
return 0;
}
static xidatafunc
static _PyXIData_getdata_t
lookup_getdata(dlcontext_t *ctx, PyObject *obj)
{
/* Cross-interpreter objects are looked up by exact match on the class.
@ -88,24 +89,24 @@ _PyXIData_FormatNotShareableError(PyThreadState *tstate,
}
xidatafunc
_PyXIData_getdata_t
_PyXIData_Lookup(PyThreadState *tstate, PyObject *obj)
{
dlcontext_t ctx;
if (get_lookup_context(tstate, &ctx) < 0) {
return NULL;
return (_PyXIData_getdata_t){0};
}
return lookup_getdata(&ctx, obj);
}
/***********************************************/
/* a registry of {type -> xidatafunc} */
/* a registry of {type -> _PyXIData_getdata_t} */
/***********************************************/
/* For now we use a global registry of shareable classes. An
alternative would be to add a tp_* slot for a class's
xidatafunc. It would be simpler and more efficient. */
/* For now we use a global registry of shareable classes.
An alternative would be to add a tp_* slot for a class's
_PyXIData_getdata_t. It would be simpler and more efficient. */
/* registry lifecycle */
@ -200,7 +201,7 @@ _xidregistry_find_type(dlregistry_t *xidregistry, PyTypeObject *cls)
return NULL;
}
static xidatafunc
static _PyXIData_getdata_t
_lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj)
{
PyTypeObject *cls = Py_TYPE(obj);
@ -209,10 +210,12 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj)
_xidregistry_lock(xidregistry);
dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls);
xidatafunc func = matched != NULL ? matched->getdata : NULL;
_PyXIData_getdata_t getdata = matched != NULL
? matched->getdata
: (_PyXIData_getdata_t){0};
_xidregistry_unlock(xidregistry);
return func;
return getdata;
}
@ -220,12 +223,13 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj)
static int
_xidregistry_add_type(dlregistry_t *xidregistry,
PyTypeObject *cls, xidatafunc getdata)
PyTypeObject *cls, _PyXIData_getdata_t getdata)
{
dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t));
if (newhead == NULL) {
return -1;
}
assert((getdata.basic == NULL) != (getdata.fallback == NULL));
*newhead = (dlregitem_t){
// We do not keep a reference, to avoid keeping the class alive.
.cls = cls,
@ -283,13 +287,13 @@ _xidregistry_clear(dlregistry_t *xidregistry)
int
_PyXIData_RegisterClass(PyThreadState *tstate,
PyTypeObject *cls, xidatafunc getdata)
PyTypeObject *cls, _PyXIData_getdata_t getdata)
{
if (!PyType_Check(cls)) {
PyErr_Format(PyExc_ValueError, "only classes may be registered");
return -1;
}
if (getdata == NULL) {
if (getdata.basic == NULL && getdata.fallback == NULL) {
PyErr_Format(PyExc_ValueError, "missing 'getdata' func");
return -1;
}
@ -304,7 +308,8 @@ _PyXIData_RegisterClass(PyThreadState *tstate,
dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls);
if (matched != NULL) {
assert(matched->getdata == getdata);
assert(matched->getdata.basic == getdata.basic);
assert(matched->getdata.fallback == getdata.fallback);
matched->refcount += 1;
goto finally;
}
@ -608,7 +613,8 @@ _tuple_shared_free(void* data)
}
static int
_tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata)
_tuple_shared(PyThreadState *tstate, PyObject *obj, xidata_fallback_t fallback,
_PyXIData_t *xidata)
{
Py_ssize_t len = PyTuple_GET_SIZE(obj);
if (len < 0) {
@ -636,7 +642,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata)
int res = -1;
if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) {
res = _PyObject_GetXIData(tstate, item, xidata_i);
res = _PyObject_GetXIData(tstate, item, fallback, xidata_i);
_Py_LeaveRecursiveCallTstate(tstate);
}
if (res < 0) {
@ -737,40 +743,48 @@ _PyFunction_GetXIData(PyThreadState *tstate, PyObject *func,
static void
_register_builtins_for_crossinterpreter_data(dlregistry_t *xidregistry)
{
#define REGISTER(TYPE, GETDATA) \
_xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \
((_PyXIData_getdata_t){.basic=(GETDATA)}))
#define REGISTER_FALLBACK(TYPE, GETDATA) \
_xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \
((_PyXIData_getdata_t){.fallback=(GETDATA)}))
// None
if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
if (REGISTER(Py_TYPE(Py_None), _none_shared) != 0) {
Py_FatalError("could not register None for cross-interpreter sharing");
}
// int
if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) {
if (REGISTER(&PyLong_Type, _long_shared) != 0) {
Py_FatalError("could not register int for cross-interpreter sharing");
}
// bytes
if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _PyBytes_GetXIData) != 0) {
if (REGISTER(&PyBytes_Type, _PyBytes_GetXIData) != 0) {
Py_FatalError("could not register bytes for cross-interpreter sharing");
}
// str
if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) {
if (REGISTER(&PyUnicode_Type, _str_shared) != 0) {
Py_FatalError("could not register str for cross-interpreter sharing");
}
// bool
if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) {
if (REGISTER(&PyBool_Type, _bool_shared) != 0) {
Py_FatalError("could not register bool for cross-interpreter sharing");
}
// float
if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) {
if (REGISTER(&PyFloat_Type, _float_shared) != 0) {
Py_FatalError("could not register float for cross-interpreter sharing");
}
// tuple
if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) {
if (REGISTER_FALLBACK(&PyTuple_Type, _tuple_shared) != 0) {
Py_FatalError("could not register tuple for cross-interpreter sharing");
}
// For now, we do not register PyCode_Type or PyFunction_Type.
#undef REGISTER
#undef REGISTER_FALLBACK
}