mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-09 20:35:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

gh-132775: Expand the Capability of Interpreter.call() (gh-133484)

Browse source 
It now supports most callables, full args, and return values.
This commit is contained in:
Eric Snow 2025-05-30 09:15:00 -06:00 • committed by GitHub
parent eb145fabbd
commit 52deabefd0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 1259 additions and 303 deletions
Download patch file Download diff file Expand all files Collapse all files

31
Include/internal/pycore_crossinterp.h
View file

@ -317,7 +317,9 @@ typedef enum error_code {
_PyXI_ERR_ALREADY_RUNNING = -4,
_PyXI_ERR_MAIN_NS_FAILURE = -5,
_PyXI_ERR_APPLY_NS_FAILURE = -6,
_PyXI_ERR_NOT_SHAREABLE = -7,
_PyXI_ERR_PRESERVE_FAILURE = -7,
_PyXI_ERR_EXC_PROPAGATION_FAILURE = -8,
_PyXI_ERR_NOT_SHAREABLE = -9,
} _PyXI_errcode;
@ -350,16 +352,33 @@ typedef struct xi_session _PyXI_session;
PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void);
PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *);
typedef struct {
PyObject *preserved;
PyObject *excinfo;
_PyXI_errcode errcode;
} _PyXI_session_result;
PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *);
PyAPI_FUNC(int) _PyXI_Enter(
_PyXI_session *session,
PyInterpreterState *interp,
PyObject *nsupdates);
PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session);
PyObject *nsupdates,
_PyXI_session_result *);
PyAPI_FUNC(int) _PyXI_Exit(
_PyXI_session *,
_PyXI_errcode,
_PyXI_session_result *);
PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *);
PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(
_PyXI_session *,
_PyXI_errcode *);
PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session);
PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session);
PyAPI_FUNC(int) _PyXI_Preserve(
_PyXI_session *,
const char *,
PyObject *,
_PyXI_errcode *);
PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *);
/*************/

20
Lib/test/_code_definitions.py
View file

@ -57,6 +57,15 @@ def spam_with_globals_and_builtins():
print(res)
def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs):
return (a, b, c, d, e, f, args, kwargs)
def spam_full_args_with_defaults(a=-1, b=-2, /, c=-3, d=-4, *args,
e=-5, f=-6, **kwargs):
return (a, b, c, d, e, f, args, kwargs)
def spam_args_attrs_and_builtins(a, b, /, c, d, *args, e, f, **kwargs):
if args.__len__() > 2:
return None
@ -67,6 +76,10 @@ def spam_returns_arg(x):
return x
def spam_raises():
raise Exception('spam!')
def spam_with_inner_not_closure():
def eggs():
pass
@ -177,8 +190,11 @@ TOP_FUNCTIONS = [
spam_minimal,
spam_with_builtins,
spam_with_globals_and_builtins,
spam_full_args,
spam_full_args_with_defaults,
spam_args_attrs_and_builtins,
spam_returns_arg,
spam_raises,
spam_with_inner_not_closure,
spam_with_inner_closure,
spam_annotated,
@ -219,8 +235,10 @@ STATELESS_FUNCTIONS = [
spam,
spam_minimal,
spam_with_builtins,
spam_full_args,
spam_args_attrs_and_builtins,
spam_returns_arg,
spam_raises,
spam_annotated,
spam_with_inner_not_closure,
spam_with_inner_closure,
@ -238,6 +256,7 @@ STATELESS_FUNCTIONS = [
STATELESS_CODE = [
*STATELESS_FUNCTIONS,
script_with_globals,
spam_full_args_with_defaults,
spam_with_globals_and_builtins,
spam_full,
]
@ -248,6 +267,7 @@ PURE_SCRIPT_FUNCTIONS = [
script_with_explicit_empty_return,
spam_minimal,
spam_with_builtins,
spam_raises,
spam_with_inner_not_closure,
spam_with_inner_closure,
]

31
Lib/test/support/interpreters/__init__.py
View file

@ -226,33 +226,32 @@ class Interpreter:
if excinfo is not None:
raise ExecutionFailed(excinfo)
def call(self, callable, /):
def _call(self, callable, args, kwargs):
res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True)
if excinfo is not None:
raise ExecutionFailed(excinfo)
return res
def call(self, callable, /, *args, **kwargs):
"""Call the object in the interpreter with given args/kwargs.
Only functions that take no arguments and have no closure
are supported.
The return value is discarded.
Nearly all callables, args, kwargs, and return values are
supported. All "shareable" objects are supported, as are
"stateless" functions (meaning non-closures that do not use
any globals). This method will fall back to pickle.
If the callable raises an exception then the error display
(including full traceback) is send back between the interpreters
(including full traceback) is sent back between the interpreters
and an ExecutionFailed exception is raised, much like what
happens with Interpreter.exec().
"""
# XXX Support args and kwargs.
# XXX Support arbitrary callables.
# XXX Support returning the return value (e.g. via pickle).
excinfo = _interpreters.call(self._id, callable, restrict=True)
if excinfo is not None:
raise ExecutionFailed(excinfo)
return self._call(callable, args, kwargs)
def call_in_thread(self, callable, /):
def call_in_thread(self, callable, /, *args, **kwargs):
"""Return a new thread that calls the object in the interpreter.
The return value and any raised exception are discarded.
"""
def task():
self.call(callable)
t = threading.Thread(target=task)
t = threading.Thread(target=self._call, args=(callable, args, kwargs))
t.start()
return t

48
Lib/test/test_code.py
View file

@ -701,6 +701,26 @@ class CodeTest(unittest.TestCase):
'checks': CO_FAST_LOCAL,
'res': CO_FAST_LOCAL,
},
defs.spam_full_args: {
'a': POSONLY,
'b': POSONLY,
'c': POSORKW,
'd': POSORKW,
'e': KWONLY,
'f': KWONLY,
'args': VARARGS,
'kwargs': VARKWARGS,
},
defs.spam_full_args_with_defaults: {
'a': POSONLY,
'b': POSONLY,
'c': POSORKW,
'd': POSORKW,
'e': KWONLY,
'f': KWONLY,
'args': VARARGS,
'kwargs': VARKWARGS,
},
defs.spam_args_attrs_and_builtins: {
'a': POSONLY,
'b': POSONLY,
@ -714,6 +734,7 @@ class CodeTest(unittest.TestCase):
defs.spam_returns_arg: {
'x': POSORKW,
},
defs.spam_raises: {},
defs.spam_with_inner_not_closure: {
'eggs': CO_FAST_LOCAL,
},
@ -934,6 +955,20 @@ class CodeTest(unittest.TestCase):
purelocals=5,
globalvars=6,
),
defs.spam_full_args: new_var_counts(
posonly=2,
posorkw=2,
kwonly=2,
varargs=1,
varkwargs=1,
),
defs.spam_full_args_with_defaults: new_var_counts(
posonly=2,
posorkw=2,
kwonly=2,
varargs=1,
varkwargs=1,
),
defs.spam_args_attrs_and_builtins: new_var_counts(
posonly=2,
posorkw=2,
@ -945,6 +980,9 @@ class CodeTest(unittest.TestCase):
defs.spam_returns_arg: new_var_counts(
posorkw=1,
),
defs.spam_raises: new_var_counts(
globalvars=1,
),
defs.spam_with_inner_not_closure: new_var_counts(
purelocals=1,
),
@ -1097,10 +1135,16 @@ class CodeTest(unittest.TestCase):
def test_stateless(self):
self.maxDiff = None
STATELESS_FUNCTIONS = [
*defs.STATELESS_FUNCTIONS,
# stateless with defaults
defs.spam_full_args_with_defaults,
]
for func in defs.STATELESS_CODE:
with self.subTest((func, '(code)')):
_testinternalcapi.verify_stateless_code(func.__code__)
for func in defs.STATELESS_FUNCTIONS:
for func in STATELESS_FUNCTIONS:
with self.subTest((func, '(func)')):
_testinternalcapi.verify_stateless_code(func)
@ -1110,7 +1154,7 @@ class CodeTest(unittest.TestCase):
with self.assertRaises(Exception):
_testinternalcapi.verify_stateless_code(func.__code__)
if func not in defs.STATELESS_FUNCTIONS:
if func not in STATELESS_FUNCTIONS:
with self.subTest((func, '(func)')):
with self.assertRaises(Exception):
_testinternalcapi.verify_stateless_code(func)

549
Lib/test/test_interpreters/test_api.py
View file

@ -1,17 +1,22 @@
import contextlib
import os
import pickle
import sys
from textwrap import dedent
import threading
import types
import unittest
from test import support
from test.support import os_helper
from test.support import script_helper
from test.support import import_helper
# Raise SkipTest if subinterpreters not supported.
_interpreters = import_helper.import_module('_interpreters')
from test.support import Py_GIL_DISABLED
from test.support import interpreters
from test.support import force_not_colorized
import test._crossinterp_definitions as defs
from test.support.interpreters import (
InterpreterError, InterpreterNotFoundError, ExecutionFailed,
)
@ -29,6 +34,59 @@ WHENCE_STR_XI = 'cross-interpreter C-API'
WHENCE_STR_STDLIB = '_interpreters module'
def is_pickleable(obj):
try:
pickle.dumps(obj)
except Exception:
return False
return True
@contextlib.contextmanager
def defined_in___main__(name, script, *, remove=False):
import __main__ as mainmod
mainns = vars(mainmod)
assert name not in mainns
exec(script, mainns, mainns)
if remove:
yield mainns.pop(name)
else:
try:
yield mainns[name]
finally:
mainns.pop(name, None)
def build_excinfo(exctype, msg=None, formatted=None, errdisplay=None):
if isinstance(exctype, type):
assert issubclass(exctype, BaseException), exctype
exctype = types.SimpleNamespace(
__name__=exctype.__name__,
__qualname__=exctype.__qualname__,
__module__=exctype.__module__,
)
elif isinstance(exctype, str):
module, _, name = exctype.rpartition(exctype)
if not module and name in __builtins__:
module = 'builtins'
exctype = types.SimpleNamespace(
__name__=name,
__qualname__=exctype,
__module__=module or None,
)
else:
assert isinstance(exctype, types.SimpleNamespace)
assert msg is None or isinstance(msg, str), msg
assert formatted is None or isinstance(formatted, str), formatted
assert errdisplay is None or isinstance(errdisplay, str), errdisplay
return types.SimpleNamespace(
type=exctype,
msg=msg,
formatted=formatted,
errdisplay=errdisplay,
)
class ModuleTests(TestBase):
def test_queue_aliases(self):
@ -890,24 +948,26 @@ class TestInterpreterExec(TestBase):
# Interpreter.exec() behavior.
def call_func_noop():
pass
call_func_noop = defs.spam_minimal
call_func_ident = defs.spam_returns_arg
call_func_failure = defs.spam_raises
def call_func_return_shareable():
return (1, None)
def call_func_return_not_shareable():
def call_func_return_stateless_func():
return (lambda x: x)
def call_func_return_pickleable():
return [1, 2, 3]
def call_func_failure():
raise Exception('spam!')
def call_func_ident(value):
return value
def call_func_return_unpickleable():
x = 42
return (lambda: x)
def get_call_func_closure(value):
@ -916,6 +976,11 @@ def get_call_func_closure(value):
return call_func_closure
def call_func_exec_wrapper(script, ns):
res = exec(script, ns, ns)
return res, ns, id(ns)
class Spam:
@staticmethod
@ -1012,86 +1077,375 @@ class TestInterpreterCall(TestBase):
# - preserves info (e.g. SyntaxError)
# - matching error display
def test_call(self):
@contextlib.contextmanager
def assert_fails(self, expected):
with self.assertRaises(ExecutionFailed) as cm:
yield cm
uncaught = cm.exception.excinfo
self.assertEqual(uncaught.type.__name__, expected.__name__)
def assert_fails_not_shareable(self):
return self.assert_fails(interpreters.NotShareableError)
def assert_code_equal(self, code1, code2):
if code1 == code2:
return
self.assertEqual(code1.co_name, code2.co_name)
self.assertEqual(code1.co_flags, code2.co_flags)
self.assertEqual(code1.co_consts, code2.co_consts)
self.assertEqual(code1.co_varnames, code2.co_varnames)
self.assertEqual(code1.co_cellvars, code2.co_cellvars)
self.assertEqual(code1.co_freevars, code2.co_freevars)
self.assertEqual(code1.co_names, code2.co_names)
self.assertEqual(
_testinternalcapi.get_code_var_counts(code1),
_testinternalcapi.get_code_var_counts(code2),
)
self.assertEqual(code1.co_code, code2.co_code)
def assert_funcs_equal(self, func1, func2):
if func1 == func2:
return
self.assertIs(type(func1), type(func2))
self.assertEqual(func1.__name__, func2.__name__)
self.assertEqual(func1.__defaults__, func2.__defaults__)
self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__)
self.assertEqual(func1.__closure__, func2.__closure__)
self.assert_code_equal(func1.__code__, func2.__code__)
self.assertEqual(
_testinternalcapi.get_code_var_counts(func1),
_testinternalcapi.get_code_var_counts(func2),
)
def assert_exceptions_equal(self, exc1, exc2):
assert isinstance(exc1, Exception)
assert isinstance(exc2, Exception)
if exc1 == exc2:
return
self.assertIs(type(exc1), type(exc2))
self.assertEqual(exc1.args, exc2.args)
def test_stateless_funcs(self):
interp = interpreters.create()
for i, (callable, args, kwargs) in enumerate([
(call_func_noop, (), {}),
(Spam.noop, (), {}),
]):
with self.subTest(f'success case #{i+1}'):
res = interp.call(callable)
self.assertIs(res, None)
func = call_func_noop
with self.subTest('no args, no return'):
res = interp.call(func)
self.assertIsNone(res)
for i, (callable, args, kwargs) in enumerate([
(call_func_ident, ('spamspamspam',), {}),
(get_call_func_closure, (42,), {}),
(get_call_func_closure(42), (), {}),
(Spam.from_values, (), {}),
(Spam.from_values, (1, 2, 3), {}),
(Spam, ('???'), {}),
(Spam(101), (), {}),
(Spam(10101).run, (), {}),
(call_func_complex, ('ident', 'spam'), {}),
(call_func_complex, ('full-ident', 'spam'), {}),
(call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}),
(call_func_complex, ('globals',), {}),
(call_func_complex, ('interpid',), {}),
(call_func_complex, ('closure',), {'value': '~~~'}),
(call_func_complex, ('custom', 'spam!'), {}),
(call_func_complex, ('custom-inner', 'eggs!'), {}),
(call_func_complex, ('???',), {'exc': ValueError('spam')}),
(call_func_return_shareable, (), {}),
(call_func_return_not_shareable, (), {}),
]):
with self.subTest(f'invalid case #{i+1}'):
with self.assertRaises(Exception):
if args or kwargs:
raise Exception((args, kwargs))
interp.call(callable)
func = call_func_return_shareable
with self.subTest('no args, returns shareable'):
res = interp.call(func)
self.assertEqual(res, (1, None))
func = call_func_return_stateless_func
expected = (lambda x: x)
with self.subTest('no args, returns stateless func'):
res = interp.call(func)
self.assert_funcs_equal(res, expected)
func = call_func_return_pickleable
with self.subTest('no args, returns pickleable'):
res = interp.call(func)
self.assertEqual(res, [1, 2, 3])
func = call_func_return_unpickleable
with self.subTest('no args, returns unpickleable'):
with self.assertRaises(interpreters.NotShareableError):
interp.call(func)
def test_stateless_func_returns_arg(self):
interp = interpreters.create()
for arg in [
None,
10,
'spam!',
b'spam!',
(1, 2, 'spam!'),
memoryview(b'spam!'),
]:
with self.subTest(f'shareable {arg!r}'):
assert _interpreters.is_shareable(arg)
res = interp.call(defs.spam_returns_arg, arg)
self.assertEqual(res, arg)
for arg in defs.STATELESS_FUNCTIONS:
with self.subTest(f'stateless func {arg!r}'):
res = interp.call(defs.spam_returns_arg, arg)
self.assert_funcs_equal(res, arg)
for arg in defs.TOP_FUNCTIONS:
if arg in defs.STATELESS_FUNCTIONS:
continue
with self.subTest(f'stateful func {arg!r}'):
res = interp.call(defs.spam_returns_arg, arg)
self.assert_funcs_equal(res, arg)
assert is_pickleable(arg)
for arg in [
Ellipsis,
NotImplemented,
object(),
2**1000,
[1, 2, 3],
{'a': 1, 'b': 2},
types.SimpleNamespace(x=42),
# builtin types
object,
type,
Exception,
ModuleNotFoundError,
# builtin exceptions
Exception('uh-oh!'),
ModuleNotFoundError('mymodule'),
# builtin fnctions
len,
sys.exit,
# user classes
*defs.TOP_CLASSES,
*(c(*a) for c, a in defs.TOP_CLASSES.items()
if c not in defs.CLASSES_WITHOUT_EQUALITY),
]:
with self.subTest(f'pickleable {arg!r}'):
res = interp.call(defs.spam_returns_arg, arg)
if type(arg) is object:
self.assertIs(type(res), object)
elif isinstance(arg, BaseException):
self.assert_exceptions_equal(res, arg)
else:
self.assertEqual(res, arg)
assert is_pickleable(arg)
for arg in [
types.MappingProxyType({}),
*(f for f in defs.NESTED_FUNCTIONS
if f not in defs.STATELESS_FUNCTIONS),
]:
with self.subTest(f'unpickleable {arg!r}'):
assert not _interpreters.is_shareable(arg)
assert not is_pickleable(arg)
with self.assertRaises(interpreters.NotShareableError):
interp.call(defs.spam_returns_arg, arg)
def test_full_args(self):
interp = interpreters.create()
expected = (1, 2, 3, 4, 5, 6, ('?',), {'g': 7, 'h': 8})
func = defs.spam_full_args
res = interp.call(func, 1, 2, 3, 4, '?', e=5, f=6, g=7, h=8)
self.assertEqual(res, expected)
def test_full_defaults(self):
# pickleable, but not stateless
interp = interpreters.create()
expected = (-1, -2, -3, -4, -5, -6, (), {'g': 8, 'h': 9})
res = interp.call(defs.spam_full_args_with_defaults, g=8, h=9)
self.assertEqual(res, expected)
def test_modified_arg(self):
interp = interpreters.create()
script = dedent("""
a = 7
b = 2
c = a ** b
""")
ns = {}
expected = {'a': 7, 'b': 2, 'c': 49}
res = interp.call(call_func_exec_wrapper, script, ns)
obj, resns, resid = res
del resns['__builtins__']
self.assertIsNone(obj)
self.assertEqual(ns, {})
self.assertEqual(resns, expected)
self.assertNotEqual(resid, id(ns))
self.assertNotEqual(resid, id(resns))
def test_func_in___main___valid(self):
# pickleable, already there'
with os_helper.temp_dir() as tempdir:
def new_mod(name, text):
script_helper.make_script(tempdir, name, dedent(text))
def run(text):
name = 'myscript'
text = dedent(f"""
import sys
sys.path.insert(0, {tempdir!r})
""") + dedent(text)
filename = script_helper.make_script(tempdir, name, text)
res = script_helper.assert_python_ok(filename)
return res.out.decode('utf-8').strip()
# no module indirection
with self.subTest('no indirection'):
text = run(f"""
from test.support import interpreters
def spam():
# This a global var...
return __name__
if __name__ == '__main__':
interp = interpreters.create()
res = interp.call(spam)
print(res)
""")
self.assertEqual(text, '<fake __main__>')
# indirect as func, direct interp
new_mod('mymod', f"""
def run(interp, func):
return interp.call(func)
""")
with self.subTest('indirect as func, direct interp'):
text = run(f"""
from test.support import interpreters
import mymod
def spam():
# This a global var...
return __name__
if __name__ == '__main__':
interp = interpreters.create()
res = mymod.run(interp, spam)
print(res)
""")
self.assertEqual(text, '<fake __main__>')
# indirect as func, indirect interp
new_mod('mymod', f"""
from test.support import interpreters
def run(func):
interp = interpreters.create()
return interp.call(func)
""")
with self.subTest('indirect as func, indirect interp'):
text = run(f"""
import mymod
def spam():
# This a global var...
return __name__
if __name__ == '__main__':
res = mymod.run(spam)
print(res)
""")
self.assertEqual(text, '<fake __main__>')
def test_func_in___main___invalid(self):
interp = interpreters.create()
funcname = f'{__name__.replace(".", "_")}_spam_okay'
script = dedent(f"""
def {funcname}():
# This a global var...
return __name__
""")
with self.subTest('pickleable, added dynamically'):
with defined_in___main__(funcname, script) as arg:
with self.assertRaises(interpreters.NotShareableError):
interp.call(defs.spam_returns_arg, arg)
with self.subTest('lying about __main__'):
with defined_in___main__(funcname, script, remove=True) as arg:
with self.assertRaises(interpreters.NotShareableError):
interp.call(defs.spam_returns_arg, arg)
def test_raises(self):
interp = interpreters.create()
with self.assertRaises(ExecutionFailed):
interp.call(call_func_failure)
with self.assert_fails(ValueError):
interp.call(call_func_complex, '???', exc=ValueError('spam'))
def test_call_valid(self):
interp = interpreters.create()
for i, (callable, args, kwargs, expected) in enumerate([
(call_func_noop, (), {}, None),
(call_func_ident, ('spamspamspam',), {}, 'spamspamspam'),
(call_func_return_shareable, (), {}, (1, None)),
(call_func_return_pickleable, (), {}, [1, 2, 3]),
(Spam.noop, (), {}, None),
(Spam.from_values, (), {}, Spam(())),
(Spam.from_values, (1, 2, 3), {}, Spam((1, 2, 3))),
(Spam, ('???',), {}, Spam('???')),
(Spam(101), (), {}, (101, (), {})),
(Spam(10101).run, (), {}, (10101, (), {})),
(call_func_complex, ('ident', 'spam'), {}, 'spam'),
(call_func_complex, ('full-ident', 'spam'), {}, ('spam', (), {})),
(call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'},
('spam', ('ham',), {'eggs': '!!!'})),
(call_func_complex, ('globals',), {}, __name__),
(call_func_complex, ('interpid',), {}, interp.id),
(call_func_complex, ('custom', 'spam!'), {}, Spam('spam!')),
]):
with self.subTest(f'success case #{i+1}'):
res = interp.call(callable, *args, **kwargs)
self.assertEqual(res, expected)
def test_call_invalid(self):
interp = interpreters.create()
func = get_call_func_closure
with self.subTest(func):
with self.assertRaises(interpreters.NotShareableError):
interp.call(func, 42)
func = get_call_func_closure(42)
with self.subTest(func):
with self.assertRaises(interpreters.NotShareableError):
interp.call(func)
func = call_func_complex
op = 'closure'
with self.subTest(f'{func} ({op})'):
with self.assertRaises(interpreters.NotShareableError):
interp.call(func, op, value='~~~')
op = 'custom-inner'
with self.subTest(f'{func} ({op})'):
with self.assertRaises(interpreters.NotShareableError):
interp.call(func, op, 'eggs!')
def test_call_in_thread(self):
interp = interpreters.create()
for i, (callable, args, kwargs) in enumerate([
(call_func_noop, (), {}),
(Spam.noop, (), {}),
]):
with self.subTest(f'success case #{i+1}'):
with self.captured_thread_exception() as ctx:
t = interp.call_in_thread(callable)
t.join()
self.assertIsNone(ctx.caught)
for i, (callable, args, kwargs) in enumerate([
(call_func_ident, ('spamspamspam',), {}),
(get_call_func_closure, (42,), {}),
(get_call_func_closure(42), (), {}),
(call_func_return_shareable, (), {}),
(call_func_return_pickleable, (), {}),
(Spam.from_values, (), {}),
(Spam.from_values, (1, 2, 3), {}),
(Spam, ('???'), {}),
(Spam(101), (), {}),
(Spam(10101).run, (), {}),
(Spam.noop, (), {}),
(call_func_complex, ('ident', 'spam'), {}),
(call_func_complex, ('full-ident', 'spam'), {}),
(call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}),
(call_func_complex, ('globals',), {}),
(call_func_complex, ('interpid',), {}),
(call_func_complex, ('closure',), {'value': '~~~'}),
(call_func_complex, ('custom', 'spam!'), {}),
(call_func_complex, ('custom-inner', 'eggs!'), {}),
(call_func_complex, ('???',), {'exc': ValueError('spam')}),
(call_func_return_shareable, (), {}),
(call_func_return_not_shareable, (), {}),
]):
with self.subTest(f'success case #{i+1}'):
with self.captured_thread_exception() as ctx:
t = interp.call_in_thread(callable, *args, **kwargs)
t.join()
self.assertIsNone(ctx.caught)
for i, (callable, args, kwargs) in enumerate([
(get_call_func_closure, (42,), {}),
(get_call_func_closure(42), (), {}),
]):
with self.subTest(f'invalid case #{i+1}'):
if args or kwargs:
continue
with self.captured_thread_exception() as ctx:
t = interp.call_in_thread(callable)
t = interp.call_in_thread(callable, *args, **kwargs)
t.join()
self.assertIsNotNone(ctx.caught)
@ -1600,18 +1954,14 @@ class LowLevelTests(TestBase):
with results:
exc = _interpreters.exec(interpid, script)
out = results.stdout()
self.assertEqual(out, '')
self.assert_ns_equal(exc, types.SimpleNamespace(
type=types.SimpleNamespace(
__name__='Exception',
__qualname__='Exception',
__module__='builtins',
),
msg='uh-oh!',
expected = build_excinfo(
Exception, 'uh-oh!',
# We check these in other tests.
formatted=exc.formatted,
errdisplay=exc.errdisplay,
))
)
self.assertEqual(out, '')
self.assert_ns_equal(exc, expected)
with self.subTest('from C-API'):
with self.interpreter_from_capi() as interpid:
@ -1623,25 +1973,50 @@ class LowLevelTests(TestBase):
self.assertEqual(exc.msg, 'it worked!')
def test_call(self):
with self.subTest('no args'):
interpid = _interpreters.create()
with self.assertRaises(ValueError):
_interpreters.call(interpid, call_func_return_shareable)
interpid = _interpreters.create()
# Here we focus on basic args and return values.
# See TestInterpreterCall for full operational coverage,
# including supported callables.
with self.subTest('no args, return None'):
func = defs.spam_minimal
res, exc = _interpreters.call(interpid, func)
self.assertIsNone(exc)
self.assertIsNone(res)
with self.subTest('empty args, return None'):
func = defs.spam_minimal
res, exc = _interpreters.call(interpid, func, (), {})
self.assertIsNone(exc)
self.assertIsNone(res)
with self.subTest('no args, return non-None'):
func = defs.script_with_return
res, exc = _interpreters.call(interpid, func)
self.assertIsNone(exc)
self.assertIs(res, True)
with self.subTest('full args, return non-None'):
expected = (1, 2, 3, 4, 5, 6, (7, 8), {'g': 9, 'h': 0})
func = defs.spam_full_args
args = (1, 2, 3, 4, 7, 8)
kwargs = dict(e=5, f=6, g=9, h=0)
res, exc = _interpreters.call(interpid, func, args, kwargs)
self.assertIsNone(exc)
self.assertEqual(res, expected)
with self.subTest('uncaught exception'):
interpid = _interpreters.create()
exc = _interpreters.call(interpid, call_func_failure)
self.assertEqual(exc, types.SimpleNamespace(
type=types.SimpleNamespace(
__name__='Exception',
__qualname__='Exception',
__module__='builtins',
),
msg='spam!',
func = defs.spam_raises
res, exc = _interpreters.call(interpid, func)
expected = build_excinfo(
Exception, 'spam!',
# We check these in other tests.
formatted=exc.formatted,
errdisplay=exc.errdisplay,
))
)
self.assertIsNone(res)
self.assertEqual(exc, expected)
@requires_test_modules
def test_set___main___attrs(self):

8
Modules/_interpchannelsmodule.c
View file

@ -254,10 +254,10 @@ _get_current_module_state(void)
{
PyObject *mod = _get_current_module();
if (mod == NULL) {
// XXX import it?
PyErr_SetString(PyExc_RuntimeError,
MODULE_NAME_STR " module not imported yet");
return NULL;
mod = PyImport_ImportModule(MODULE_NAME_STR);
if (mod == NULL) {
return NULL;
}
}
module_state *state = get_module_state(mod);
Py_DECREF(mod);

8
Modules/_interpqueuesmodule.c
View file

@ -1356,10 +1356,10 @@ _queueobj_from_xid(_PyXIData_t *data)
PyObject *mod = _get_current_module();
if (mod == NULL) {
// XXX import it?
PyErr_SetString(PyExc_RuntimeError,
MODULE_NAME_STR " module not imported yet");
return NULL;
mod = PyImport_ImportModule(MODULE_NAME_STR);
if (mod == NULL) {
return NULL;
}
}
PyTypeObject *cls = get_external_queue_type(mod);

414
Modules/_interpretersmodule.c
View file

@ -72,6 +72,32 @@ is_running_main(PyInterpreterState *interp)
}
static inline int
is_notshareable_raised(PyThreadState *tstate)
{
PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate);
return _PyErr_ExceptionMatches(tstate, exctype);
}
static void
unwrap_not_shareable(PyThreadState *tstate)
{
if (!is_notshareable_raised(tstate)) {
return;
}
PyObject *exc = _PyErr_GetRaisedException(tstate);
PyObject *cause = PyException_GetCause(exc);
if (cause != NULL) {
Py_DECREF(exc);
exc = cause;
}
else {
assert(PyException_GetContext(exc) == NULL);
}
_PyErr_SetRaisedException(tstate, exc);
}
/* Cross-interpreter Buffer Views *******************************************/
/* When a memoryview object is "shared" between interpreters,
@ -320,10 +346,10 @@ _get_current_module_state(void)
{
PyObject *mod = _get_current_module();
if (mod == NULL) {
// XXX import it?
PyErr_SetString(PyExc_RuntimeError,
MODULE_NAME_STR " module not imported yet");
return NULL;
mod = PyImport_ImportModule(MODULE_NAME_STR);
if (mod == NULL) {
return NULL;
}
}
module_state *state = get_module_state(mod);
Py_DECREF(mod);
@ -422,76 +448,265 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config)
}
struct interp_call {
_PyXIData_t *func;
_PyXIData_t *args;
_PyXIData_t *kwargs;
struct {
_PyXIData_t func;
_PyXIData_t args;
_PyXIData_t kwargs;
} _preallocated;
};
static void
_interp_call_clear(struct interp_call *call)
{
if (call->func != NULL) {
_PyXIData_Clear(NULL, call->func);
}
if (call->args != NULL) {
_PyXIData_Clear(NULL, call->args);
}
if (call->kwargs != NULL) {
_PyXIData_Clear(NULL, call->kwargs);
}
*call = (struct interp_call){0};
}
static int
_run_script(_PyXIData_t *script, PyObject *ns)
_interp_call_pack(PyThreadState *tstate, struct interp_call *call,
PyObject *func, PyObject *args, PyObject *kwargs)
{
xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK;
assert(call->func == NULL);
assert(call->args == NULL);
assert(call->kwargs == NULL);
// Handle the func.
if (!PyCallable_Check(func)) {
_PyErr_Format(tstate, PyExc_TypeError,
"expected a callable, got %R", func);
return -1;
}
if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) {
PyObject *exc = _PyErr_GetRaisedException(tstate);
if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) {
_PyErr_SetRaisedException(tstate, exc);
return -1;
}
Py_DECREF(exc);
}
call->func = &call->_preallocated.func;
// Handle the args.
if (args == NULL || args == Py_None) {
// Leave it empty.
}
else {
assert(PyTuple_Check(args));
if (PyTuple_GET_SIZE(args) > 0) {
if (_PyObject_GetXIData(
tstate, args, fallback, &call->_preallocated.args) < 0)
{
_interp_call_clear(call);
return -1;
}
call->args = &call->_preallocated.args;
}
}
// Handle the kwargs.
if (kwargs == NULL || kwargs == Py_None) {
// Leave it empty.
}
else {
assert(PyDict_Check(kwargs));
if (PyDict_GET_SIZE(kwargs) > 0) {
if (_PyObject_GetXIData(
tstate, kwargs, fallback, &call->_preallocated.kwargs) < 0)
{
_interp_call_clear(call);
return -1;
}
call->kwargs = &call->_preallocated.kwargs;
}
}
return 0;
}
static int
_interp_call_unpack(struct interp_call *call,
PyObject **p_func, PyObject **p_args, PyObject **p_kwargs)
{
// Unpack the func.
PyObject *func = _PyXIData_NewObject(call->func);
if (func == NULL) {
return -1;
}
// Unpack the args.
PyObject *args;
if (call->args == NULL) {
args = PyTuple_New(0);
if (args == NULL) {
Py_DECREF(func);
return -1;
}
}
else {
args = _PyXIData_NewObject(call->args);
if (args == NULL) {
Py_DECREF(func);
return -1;
}
assert(PyTuple_Check(args));
}
// Unpack the kwargs.
PyObject *kwargs = NULL;
if (call->kwargs != NULL) {
kwargs = _PyXIData_NewObject(call->kwargs);
if (kwargs == NULL) {
Py_DECREF(func);
Py_DECREF(args);
return -1;
}
assert(PyDict_Check(kwargs));
}
*p_func = func;
*p_args = args;
*p_kwargs = kwargs;
return 0;
}
static int
_make_call(struct interp_call *call,
PyObject **p_result, _PyXI_errcode *p_errcode)
{
assert(call != NULL && call->func != NULL);
PyThreadState *tstate = _PyThreadState_GET();
// Get the func and args.
PyObject *func = NULL, *args = NULL, *kwargs = NULL;
if (_interp_call_unpack(call, &func, &args, &kwargs) < 0) {
assert(func == NULL);
assert(args == NULL);
assert(kwargs == NULL);
*p_errcode = is_notshareable_raised(tstate)
? _PyXI_ERR_NOT_SHAREABLE
: _PyXI_ERR_OTHER;
return -1;
}
*p_errcode = _PyXI_ERR_NO_ERROR;
// Make the call.
PyObject *resobj = PyObject_Call(func, args, kwargs);
Py_DECREF(func);
Py_XDECREF(args);
Py_XDECREF(kwargs);
if (resobj == NULL) {
return -1;
}
*p_result = resobj;
return 0;
}
static int
_run_script(_PyXIData_t *script, PyObject *ns, _PyXI_errcode *p_errcode)
{
PyObject *code = _PyXIData_NewObject(script);
if (code == NULL) {
*p_errcode = _PyXI_ERR_NOT_SHAREABLE;
return -1;
}
PyObject *result = PyEval_EvalCode(code, ns, ns);
Py_DECREF(code);
if (result == NULL) {
*p_errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION;
return -1;
}
assert(result == Py_None);
Py_DECREF(result); // We throw away the result.
return 0;
}
struct run_result {
PyObject *result;
PyObject *excinfo;
};
static void
_run_result_clear(struct run_result *runres)
{
Py_CLEAR(runres->result);
Py_CLEAR(runres->excinfo);
}
static int
_exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp,
_PyXIData_t *script, PyObject *shareables,
PyObject **p_excinfo)
_run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp,
_PyXIData_t *script, struct interp_call *call,
PyObject *shareables, struct run_result *runres)
{
assert(!_PyErr_Occurred(tstate));
_PyXI_session *session = _PyXI_NewSession();
if (session == NULL) {
return -1;
}
_PyXI_session_result result = {0};
// Prep and switch interpreters.
if (_PyXI_Enter(session, interp, shareables) < 0) {
if (_PyErr_Occurred(tstate)) {
// If an error occured at this step, it means that interp
// was not prepared and switched.
_PyXI_FreeSession(session);
return -1;
}
// Now, apply the error from another interpreter:
PyObject *excinfo = _PyXI_ApplyCapturedException(session);
if (excinfo != NULL) {
*p_excinfo = excinfo;
}
assert(PyErr_Occurred());
if (_PyXI_Enter(session, interp, shareables, &result) < 0) {
// If an error occured at this step, it means that interp
// was not prepared and switched.
_PyXI_FreeSession(session);
assert(result.excinfo == NULL);
return -1;
}
// Run the script.
// Run in the interpreter.
int res = -1;
PyObject *mainns = _PyXI_GetMainNamespace(session);
if (mainns == NULL) {
goto finally;
_PyXI_errcode errcode = _PyXI_ERR_NO_ERROR;
if (script != NULL) {
assert(call == NULL);
PyObject *mainns = _PyXI_GetMainNamespace(session, &errcode);
if (mainns == NULL) {
goto finally;
}
res = _run_script(script, mainns, &errcode);
}
res = _run_script(script, mainns);
else {
assert(call != NULL);
PyObject *resobj;
res = _make_call(call, &resobj, &errcode);
if (res == 0) {
res = _PyXI_Preserve(session, "resobj", resobj, &errcode);
Py_DECREF(resobj);
if (res < 0) {
goto finally;
}
}
}
int exitres;
finally:
// Clean up and switch back.
_PyXI_Exit(session);
exitres = _PyXI_Exit(session, errcode, &result);
assert(res == 0 || exitres != 0);
_PyXI_FreeSession(session);
// Propagate any exception out to the caller.
assert(!PyErr_Occurred());
if (res < 0) {
PyObject *excinfo = _PyXI_ApplyCapturedException(session);
if (excinfo != NULL) {
*p_excinfo = excinfo;
}
res = exitres;
if (_PyErr_Occurred(tstate)) {
assert(res < 0);
}
else if (res < 0) {
assert(result.excinfo != NULL);
runres->excinfo = Py_NewRef(result.excinfo);
res = -1;
}
else {
assert(!_PyXI_HasCapturedException(session));
assert(result.excinfo == NULL);
runres->result = _PyXI_GetPreserved(&result, "resobj");
if (_PyErr_Occurred(tstate)) {
res = -1;
}
}
_PyXI_FreeSession(session);
_PyXI_ClearResult(&result);
return res;
}
@ -842,21 +1057,23 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs)
}
// Prep and switch interpreters, including apply the updates.
if (_PyXI_Enter(session, interp, updates) < 0) {
if (!PyErr_Occurred()) {
_PyXI_ApplyCapturedException(session);
assert(PyErr_Occurred());
}
else {
assert(!_PyXI_HasCapturedException(session));
}
if (_PyXI_Enter(session, interp, updates, NULL) < 0) {
_PyXI_FreeSession(session);
return NULL;
}
// Clean up and switch back.
_PyXI_Exit(session);
assert(!PyErr_Occurred());
int res = _PyXI_Exit(session, _PyXI_ERR_NO_ERROR, NULL);
_PyXI_FreeSession(session);
assert(res == 0);
if (res < 0) {
// unreachable
if (!PyErr_Occurred()) {
PyErr_SetString(PyExc_RuntimeError, "unresolved error");
}
return NULL;
}
Py_RETURN_NONE;
}
@ -867,23 +1084,16 @@ PyDoc_STRVAR(set___main___attrs_doc,
Bind the given attributes in the interpreter's __main__ module.");
static void
unwrap_not_shareable(PyThreadState *tstate)
static PyObject *
_handle_script_error(struct run_result *runres)
{
PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate);
if (!_PyErr_ExceptionMatches(tstate, exctype)) {
return;
assert(runres->result == NULL);
if (runres->excinfo == NULL) {
assert(PyErr_Occurred());
return NULL;
}
PyObject *exc = _PyErr_GetRaisedException(tstate);
PyObject *cause = PyException_GetCause(exc);
if (cause != NULL) {
Py_DECREF(exc);
exc = cause;
}
else {
assert(PyException_GetContext(exc) == NULL);
}
_PyErr_SetRaisedException(tstate, exc);
assert(!PyErr_Occurred());
return runres->excinfo;
}
static PyObject *
@ -918,13 +1128,14 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
PyObject *excinfo = NULL;
int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo);
struct run_result runres = {0};
int res = _run_in_interpreter(
tstate, interp, &xidata, NULL, shared, &runres);
_PyXIData_Release(&xidata);
if (res < 0) {
assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
return excinfo;
return _handle_script_error(&runres);
}
assert(runres.result == NULL);
Py_RETURN_NONE;
#undef FUNCNAME
}
@ -981,13 +1192,14 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
PyObject *excinfo = NULL;
int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo);
struct run_result runres = {0};
int res = _run_in_interpreter(
tstate, interp, &xidata, NULL, shared, &runres);
_PyXIData_Release(&xidata);
if (res < 0) {
assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
return excinfo;
return _handle_script_error(&runres);
}
assert(runres.result == NULL);
Py_RETURN_NONE;
#undef FUNCNAME
}
@ -1043,13 +1255,14 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
PyObject *excinfo = NULL;
int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo);
struct run_result runres = {0};
int res = _run_in_interpreter(
tstate, interp, &xidata, NULL, shared, &runres);
_PyXIData_Release(&xidata);
if (res < 0) {
assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
return excinfo;
return _handle_script_error(&runres);
}
assert(runres.result == NULL);
Py_RETURN_NONE;
#undef FUNCNAME
}
@ -1069,15 +1282,18 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds)
#define FUNCNAME MODULE_NAME_STR ".call"
PyThreadState *tstate = _PyThreadState_GET();
static char *kwlist[] = {"id", "callable", "args", "kwargs",
"restrict", NULL};
"preserve_exc", "restrict", NULL};
PyObject *id, *callable;
PyObject *args_obj = NULL;
PyObject *kwargs_obj = NULL;
int preserve_exc = 0;
int restricted = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"OO|OO$p:" FUNCNAME, kwlist,
&id, &callable, &args_obj, &kwargs_obj,
&restricted))
"OO|O!O!$pp:" FUNCNAME, kwlist,
&id, &callable,
&PyTuple_Type, &args_obj,
&PyDict_Type, &kwargs_obj,
&preserve_exc, &restricted))
{
return NULL;
}
@ -1089,29 +1305,29 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
if (args_obj != NULL) {
_PyErr_SetString(tstate, PyExc_ValueError, "got unexpected args");
return NULL;
}
if (kwargs_obj != NULL) {
_PyErr_SetString(tstate, PyExc_ValueError, "got unexpected kwargs");
struct interp_call call = {0};
if (_interp_call_pack(tstate, &call, callable, args_obj, kwargs_obj) < 0) {
return NULL;
}
_PyXIData_t xidata = {0};
if (_PyCode_GetPureScriptXIData(tstate, callable, &xidata) < 0) {
unwrap_not_shareable(tstate);
return NULL;
PyObject *res_and_exc = NULL;
struct run_result runres = {0};
if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) {
if (runres.excinfo == NULL) {
assert(_PyErr_Occurred(tstate));
goto finally;
}
assert(!_PyErr_Occurred(tstate));
}
assert(runres.result == NULL || runres.excinfo == NULL);
res_and_exc = Py_BuildValue("OO",
(runres.result ? runres.result : Py_None),
(runres.excinfo ? runres.excinfo : Py_None));
PyObject *excinfo = NULL;
int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &excinfo);
_PyXIData_Release(&xidata);
if (res < 0) {
assert((excinfo == NULL) != (PyErr_Occurred() == NULL));
return excinfo;
}
Py_RETURN_NONE;
finally:
_interp_call_clear(&call);
_run_result_clear(&runres);
return res_and_exc;
#undef FUNCNAME
}
@ -1119,13 +1335,7 @@ PyDoc_STRVAR(call_doc,
"call(id, callable, args=None, kwargs=None, *, restrict=False)\n\
\n\
Call the provided object in the identified interpreter.\n\
Pass the given args and kwargs, if possible.\n\
\n\
\"callable\" may be a plain function with no free vars that takes\n\
no arguments.\n\
\n\
The function's code object is used and all its state\n\
is ignored, including its __globals__ dict.");
Pass the given args and kwargs, if possible.");
static PyObject *

449
Python/crossinterp.c
View file

@ -70,6 +70,17 @@ runpy_run_path(const char *filename, const char *modname)
}
static void
set_exc_with_cause(PyObject *exctype, const char *msg)
{
PyObject *cause = PyErr_GetRaisedException();
PyErr_SetString(exctype, msg);
PyObject *exc = PyErr_GetRaisedException();
PyException_SetCause(exc, cause);
PyErr_SetRaisedException(exc);
}
static PyObject *
pyerr_get_message(PyObject *exc)
{
@ -1314,7 +1325,7 @@ _excinfo_normalize_type(struct _excinfo_type *info,
}
static void
_PyXI_excinfo_Clear(_PyXI_excinfo *info)
_PyXI_excinfo_clear(_PyXI_excinfo *info)
{
_excinfo_clear_type(&info->type);
if (info->msg != NULL) {
@ -1364,7 +1375,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc)
assert(exc != NULL);
if (PyErr_GivenExceptionMatches(exc, PyExc_MemoryError)) {
_PyXI_excinfo_Clear(info);
_PyXI_excinfo_clear(info);
return NULL;
}
const char *failure = NULL;
@ -1410,7 +1421,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc)
error:
assert(failure != NULL);
_PyXI_excinfo_Clear(info);
_PyXI_excinfo_clear(info);
return failure;
}
@ -1461,7 +1472,7 @@ _PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj)
error:
assert(failure != NULL);
_PyXI_excinfo_Clear(info);
_PyXI_excinfo_clear(info);
return failure;
}
@ -1656,7 +1667,7 @@ _PyXI_ExcInfoAsObject(_PyXI_excinfo *info)
void
_PyXI_ClearExcInfo(_PyXI_excinfo *info)
{
_PyXI_excinfo_Clear(info);
_PyXI_excinfo_clear(info);
}
@ -1694,6 +1705,14 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp)
PyErr_SetString(PyExc_InterpreterError,
"failed to apply namespace to __main__");
break;
case _PyXI_ERR_PRESERVE_FAILURE:
PyErr_SetString(PyExc_InterpreterError,
"failed to preserve objects across session");
break;
case _PyXI_ERR_EXC_PROPAGATION_FAILURE:
PyErr_SetString(PyExc_InterpreterError,
"failed to transfer exception between interpreters");
break;
case _PyXI_ERR_NOT_SHAREABLE:
_set_xid_lookup_failure(tstate, NULL, NULL, NULL);
break;
@ -1743,7 +1762,7 @@ _PyXI_InitError(_PyXI_error *error, PyObject *excobj, _PyXI_errcode code)
assert(excobj == NULL);
assert(code != _PyXI_ERR_NO_ERROR);
error->code = code;
_PyXI_excinfo_Clear(&error->uncaught);
_PyXI_excinfo_clear(&error->uncaught);
}
return failure;
}
@ -1753,7 +1772,7 @@ _PyXI_ApplyError(_PyXI_error *error)
{
PyThreadState *tstate = PyThreadState_Get();
if (error->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) {
// Raise an exception that proxies the propagated exception.
// We will raise an exception that proxies the propagated exception.
return _PyXI_excinfo_AsObject(&error->uncaught);
}
else if (error->code == _PyXI_ERR_NOT_SHAREABLE) {
@ -1839,7 +1858,8 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid)
}
static int
_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value)
_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value,
xidata_fallback_t fallback)
{
assert(_sharednsitem_is_initialized(item));
assert(item->xidata == NULL);
@ -1848,8 +1868,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value)
return -1;
}
PyThreadState *tstate = PyThreadState_Get();
// XXX Use _PyObject_GetXIDataWithFallback()?
if (_PyObject_GetXIDataNoFallback(tstate, value, item->xidata) != 0) {
if (_PyObject_GetXIData(tstate, value, fallback, item->xidata) < 0) {
PyMem_RawFree(item->xidata);
item->xidata = NULL;
// The caller may want to propagate PyExc_NotShareableError
@ -1881,7 +1900,8 @@ _sharednsitem_clear(_PyXI_namespace_item *item)
}
static int
_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns)
_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns,
xidata_fallback_t fallback)
{
assert(item->name != NULL);
assert(item->xidata == NULL);
@ -1893,7 +1913,7 @@ _sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns)
// When applied, this item will be set to the default (or fail).
return 0;
}
if (_sharednsitem_set_value(item, value) < 0) {
if (_sharednsitem_set_value(item, value, fallback) < 0) {
return -1;
}
return 0;
@ -2144,18 +2164,21 @@ error:
return NULL;
}
static void _propagate_not_shareable_error(_PyXI_session *);
static void _propagate_not_shareable_error(_PyXI_errcode *);
static int
_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, _PyXI_session *session)
_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj,
xidata_fallback_t fallback, _PyXI_errcode *p_errcode)
{
// All items are expected to be shareable.
assert(_sharedns_check_counts(ns));
assert(ns->numnames == ns->maxitems);
assert(ns->numvalues == 0);
for (Py_ssize_t i=0; i < ns->maxitems; i++) {
if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj) < 0) {
_propagate_not_shareable_error(session);
if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj, fallback) < 0) {
if (p_errcode != NULL) {
_propagate_not_shareable_error(p_errcode);
}
// Clear out the ones we set so far.
for (Py_ssize_t j=0; j < i; j++) {
_sharednsitem_clear_value(&ns->items[j]);
@ -2221,6 +2244,18 @@ _apply_sharedns(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt)
/* switched-interpreter sessions */
/*********************************/
struct xi_session_error {
// This is set if the interpreter is entered and raised an exception
// that needs to be handled in some special way during exit.
_PyXI_errcode *override;
// This is set if exit captured an exception to propagate.
_PyXI_error *info;
// -- pre-allocated memory --
_PyXI_error _info;
_PyXI_errcode _override;
};
struct xi_session {
#define SESSION_UNUSED 0
#define SESSION_ACTIVE 1
@ -2249,18 +2284,14 @@ struct xi_session {
// beginning of the session as a convenience.
PyObject *main_ns;
// This is set if the interpreter is entered and raised an exception
// that needs to be handled in some special way during exit.
_PyXI_errcode *error_override;
// This is set if exit captured an exception to propagate.
_PyXI_error *error;
// This is a dict of objects that will be available (via sharing)
// once the session exits. Do not access this directly; use
// _PyXI_Preserve() and _PyXI_GetPreserved() instead;
PyObject *_preserved;
// -- pre-allocated memory --
_PyXI_error _error;
_PyXI_errcode _error_override;
struct xi_session_error error;
};
_PyXI_session *
_PyXI_NewSession(void)
{
@ -2286,9 +2317,25 @@ _session_is_active(_PyXI_session *session)
return session->status == SESSION_ACTIVE;
}
static int _ensure_main_ns(_PyXI_session *);
static int
_session_pop_error(_PyXI_session *session, struct xi_session_error *err)
{
if (session->error.info == NULL) {
assert(session->error.override == NULL);
*err = (struct xi_session_error){0};
return 0;
}
*err = session->error;
err->info = &err->_info;
if (err->override != NULL) {
err->override = &err->_override;
}
session->error = (struct xi_session_error){0};
return 1;
}
static int _ensure_main_ns(_PyXI_session *, _PyXI_errcode *);
static inline void _session_set_error(_PyXI_session *, _PyXI_errcode);
static void _capture_current_exception(_PyXI_session *);
/* enter/exit a cross-interpreter session */
@ -2305,9 +2352,9 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp)
assert(!session->running);
assert(session->main_ns == NULL);
// Set elsewhere and cleared in _capture_current_exception().
assert(session->error_override == NULL);
// Set elsewhere and cleared in _PyXI_ApplyCapturedException().
assert(session->error == NULL);
assert(session->error.override == NULL);
// Set elsewhere and cleared in _PyXI_Exit().
assert(session->error.info == NULL);
// Switch to interpreter.
PyThreadState *tstate = PyThreadState_Get();
@ -2336,14 +2383,16 @@ _exit_session(_PyXI_session *session)
PyThreadState *tstate = session->init_tstate;
assert(tstate != NULL);
assert(PyThreadState_Get() == tstate);
assert(!_PyErr_Occurred(tstate));
// Release any of the entered interpreters resources.
Py_CLEAR(session->main_ns);
Py_CLEAR(session->_preserved);
// Ensure this thread no longer owns __main__.
if (session->running) {
_PyInterpreterState_SetNotRunningMain(tstate->interp);
assert(!PyErr_Occurred());
assert(!_PyErr_Occurred(tstate));
session->running = 0;
}
@ -2360,21 +2409,16 @@ _exit_session(_PyXI_session *session)
assert(!session->own_init_tstate);
}
// For now the error data persists past the exit.
*session = (_PyXI_session){
.error_override = session->error_override,
.error = session->error,
._error = session->_error,
._error_override = session->_error_override,
};
assert(session->error.info == NULL);
assert(session->error.override == _PyXI_ERR_NO_ERROR);
*session = (_PyXI_session){0};
}
static void
_propagate_not_shareable_error(_PyXI_session *session)
_propagate_not_shareable_error(_PyXI_errcode *p_errcode)
{
if (session == NULL) {
return;
}
assert(p_errcode != NULL);
PyThreadState *tstate = PyThreadState_Get();
PyObject *exctype = get_notshareableerror_type(tstate);
if (exctype == NULL) {
@ -2384,46 +2428,46 @@ _propagate_not_shareable_error(_PyXI_session *session)
}
if (PyErr_ExceptionMatches(exctype)) {
// We want to propagate the exception directly.
_session_set_error(session, _PyXI_ERR_NOT_SHAREABLE);
*p_errcode = _PyXI_ERR_NOT_SHAREABLE;
}
}
PyObject *
_PyXI_ApplyCapturedException(_PyXI_session *session)
{
assert(!PyErr_Occurred());
assert(session->error != NULL);
PyObject *res = _PyXI_ApplyError(session->error);
assert((res == NULL) != (PyErr_Occurred() == NULL));
session->error = NULL;
return res;
}
int
_PyXI_HasCapturedException(_PyXI_session *session)
{
return session->error != NULL;
}
int
_PyXI_Enter(_PyXI_session *session,
PyInterpreterState *interp, PyObject *nsupdates)
PyInterpreterState *interp, PyObject *nsupdates,
_PyXI_session_result *result)
{
// Convert the attrs for cross-interpreter use.
_PyXI_namespace *sharedns = NULL;
if (nsupdates != NULL) {
Py_ssize_t len = PyDict_Size(nsupdates);
if (len < 0) {
if (result != NULL) {
result->errcode = _PyXI_ERR_APPLY_NS_FAILURE;
}
return -1;
}
if (len > 0) {
sharedns = _create_sharedns(nsupdates);
if (sharedns == NULL) {
if (result != NULL) {
result->errcode = _PyXI_ERR_APPLY_NS_FAILURE;
}
return -1;
}
if (_fill_sharedns(sharedns, nsupdates, NULL) < 0) {
assert(session->error == NULL);
// For now we limit it to shareable objects.
xidata_fallback_t fallback = _PyXIDATA_XIDATA_ONLY;
_PyXI_errcode errcode = _PyXI_ERR_NO_ERROR;
if (_fill_sharedns(sharedns, nsupdates, fallback, &errcode) < 0) {
assert(PyErr_Occurred());
assert(session->error.info == NULL);
if (errcode == _PyXI_ERR_NO_ERROR) {
errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION;
}
_destroy_sharedns(sharedns);
if (result != NULL) {
result->errcode = errcode;
}
return -1;
}
}
@ -2445,8 +2489,7 @@ _PyXI_Enter(_PyXI_session *session,
// Apply the cross-interpreter data.
if (sharedns != NULL) {
if (_ensure_main_ns(session) < 0) {
errcode = _PyXI_ERR_MAIN_NS_FAILURE;
if (_ensure_main_ns(session, &errcode) < 0) {
goto error;
}
if (_apply_sharedns(sharedns, session->main_ns, NULL) < 0) {
@ -2462,19 +2505,124 @@ _PyXI_Enter(_PyXI_session *session,
error:
// We want to propagate all exceptions here directly (best effort).
assert(errcode != _PyXI_ERR_NO_ERROR);
_session_set_error(session, errcode);
assert(!PyErr_Occurred());
// Exit the session.
struct xi_session_error err;
(void)_session_pop_error(session, &err);
_exit_session(session);
if (sharedns != NULL) {
_destroy_sharedns(sharedns);
}
// Apply the error from the other interpreter.
PyObject *excinfo = _PyXI_ApplyError(err.info);
_PyXI_excinfo_clear(&err.info->uncaught);
if (excinfo != NULL) {
if (result != NULL) {
result->excinfo = excinfo;
}
else {
#ifdef Py_DEBUG
fprintf(stderr, "_PyXI_Enter(): uncaught exception discarded");
#endif
}
}
assert(PyErr_Occurred());
return -1;
}
void
_PyXI_Exit(_PyXI_session *session)
static int _pop_preserved(_PyXI_session *, _PyXI_namespace **, PyObject **,
_PyXI_errcode *);
static int _finish_preserved(_PyXI_namespace *, PyObject **);
int
_PyXI_Exit(_PyXI_session *session, _PyXI_errcode errcode,
_PyXI_session_result *result)
{
_capture_current_exception(session);
int res = 0;
// Capture the raised exception, if any.
assert(session->error.info == NULL);
if (PyErr_Occurred()) {
_session_set_error(session, errcode);
assert(!PyErr_Occurred());
}
else {
assert(errcode == _PyXI_ERR_NO_ERROR);
assert(session->error.override == NULL);
}
// Capture the preserved namespace.
_PyXI_namespace *preserved = NULL;
PyObject *preservedobj = NULL;
if (result != NULL) {
errcode = _PyXI_ERR_NO_ERROR;
if (_pop_preserved(session, &preserved, &preservedobj, &errcode) < 0) {
if (session->error.info != NULL) {
// XXX Chain the exception (i.e. set __context__)?
PyErr_FormatUnraisable(
"Exception ignored while capturing preserved objects");
}
else {
_session_set_error(session, errcode);
}
}
}
// Exit the session.
struct xi_session_error err;
(void)_session_pop_error(session, &err);
_exit_session(session);
// Restore the preserved namespace.
assert(preserved == NULL || preservedobj == NULL);
if (_finish_preserved(preserved, &preservedobj) < 0) {
assert(preservedobj == NULL);
if (err.info != NULL) {
// XXX Chain the exception (i.e. set __context__)?
PyErr_FormatUnraisable(
"Exception ignored while capturing preserved objects");
}
else {
errcode = _PyXI_ERR_PRESERVE_FAILURE;
_propagate_not_shareable_error(&errcode);
}
}
if (result != NULL) {
result->preserved = preservedobj;
result->errcode = errcode;
}
// Apply the error from the other interpreter, if any.
if (err.info != NULL) {
res = -1;
assert(!PyErr_Occurred());
PyObject *excinfo = _PyXI_ApplyError(err.info);
_PyXI_excinfo_clear(&err.info->uncaught);
if (excinfo == NULL) {
assert(PyErr_Occurred());
if (result != NULL) {
_PyXI_ClearResult(result);
*result = (_PyXI_session_result){
.errcode = _PyXI_ERR_EXC_PROPAGATION_FAILURE,
};
}
}
else if (result != NULL) {
result->excinfo = excinfo;
}
else {
#ifdef Py_DEBUG
fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded");
#endif
}
}
return res;
}
@ -2483,15 +2631,15 @@ _PyXI_Exit(_PyXI_session *session)
static void
_capture_current_exception(_PyXI_session *session)
{
assert(session->error == NULL);
assert(session->error.info == NULL);
if (!PyErr_Occurred()) {
assert(session->error_override == NULL);
assert(session->error.override == NULL);
return;
}
// Handle the exception override.
_PyXI_errcode *override = session->error_override;
session->error_override = NULL;
_PyXI_errcode *override = session->error.override;
session->error.override = NULL;
_PyXI_errcode errcode = override != NULL
? *override
: _PyXI_ERR_UNCAUGHT_EXCEPTION;
@ -2514,7 +2662,7 @@ _capture_current_exception(_PyXI_session *session)
}
// Capture the exception.
_PyXI_error *err = &session->_error;
_PyXI_error *err = &session->error._info;
*err = (_PyXI_error){
.interp = session->init_tstate->interp,
};
@ -2541,7 +2689,7 @@ _capture_current_exception(_PyXI_session *session)
// Finished!
assert(!PyErr_Occurred());
session->error = err;
session->error.info = err;
}
static inline void
@ -2549,15 +2697,19 @@ _session_set_error(_PyXI_session *session, _PyXI_errcode errcode)
{
assert(_session_is_active(session));
assert(PyErr_Occurred());
if (errcode == _PyXI_ERR_NO_ERROR) {
// We're a bit forgiving here.
errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION;
}
if (errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION) {
session->_error_override = errcode;
session->error_override = &session->_error_override;
session->error._override = errcode;
session->error.override = &session->error._override;
}
_capture_current_exception(session);
}
static int
_ensure_main_ns(_PyXI_session *session)
_ensure_main_ns(_PyXI_session *session, _PyXI_errcode *p_errcode)
{
assert(_session_is_active(session));
if (session->main_ns != NULL) {
@ -2566,11 +2718,17 @@ _ensure_main_ns(_PyXI_session *session)
// Cache __main__.__dict__.
PyObject *main_mod = _Py_GetMainModule(session->init_tstate);
if (_Py_CheckMainModule(main_mod) < 0) {
if (p_errcode != NULL) {
*p_errcode = _PyXI_ERR_MAIN_NS_FAILURE;
}
return -1;
}
PyObject *ns = PyModule_GetDict(main_mod); // borrowed
Py_DECREF(main_mod);
if (ns == NULL) {
if (p_errcode != NULL) {
*p_errcode = _PyXI_ERR_MAIN_NS_FAILURE;
}
return -1;
}
session->main_ns = Py_NewRef(ns);
@ -2578,21 +2736,150 @@ _ensure_main_ns(_PyXI_session *session)
}
PyObject *
_PyXI_GetMainNamespace(_PyXI_session *session)
_PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_errcode *p_errcode)
{
if (!_session_is_active(session)) {
PyErr_SetString(PyExc_RuntimeError, "session not active");
return NULL;
}
if (_ensure_main_ns(session) < 0) {
_session_set_error(session, _PyXI_ERR_MAIN_NS_FAILURE);
_capture_current_exception(session);
if (_ensure_main_ns(session, p_errcode) < 0) {
return NULL;
}
return session->main_ns;
}
static int
_pop_preserved(_PyXI_session *session,
_PyXI_namespace **p_xidata, PyObject **p_obj,
_PyXI_errcode *p_errcode)
{
assert(_PyThreadState_GET() == session->init_tstate); // active session
if (session->_preserved == NULL) {
*p_xidata = NULL;
*p_obj = NULL;
return 0;
}
if (session->init_tstate == session->prev_tstate) {
// We did not switch interpreters.
*p_xidata = NULL;
*p_obj = session->_preserved;
session->_preserved = NULL;
return 0;
}
*p_obj = NULL;
// We did switch interpreters.
Py_ssize_t len = PyDict_Size(session->_preserved);
if (len < 0) {
if (p_errcode != NULL) {
*p_errcode = _PyXI_ERR_PRESERVE_FAILURE;
}
return -1;
}
else if (len == 0) {
*p_xidata = NULL;
}
else {
_PyXI_namespace *xidata = _create_sharedns(session->_preserved);
if (xidata == NULL) {
if (p_errcode != NULL) {
*p_errcode = _PyXI_ERR_PRESERVE_FAILURE;
}
return -1;
}
_PyXI_errcode errcode = _PyXI_ERR_NO_ERROR;
if (_fill_sharedns(xidata, session->_preserved,
_PyXIDATA_FULL_FALLBACK, &errcode) < 0)
{
assert(session->error.info == NULL);
if (errcode != _PyXI_ERR_NOT_SHAREABLE) {
errcode = _PyXI_ERR_PRESERVE_FAILURE;
}
if (p_errcode != NULL) {
*p_errcode = errcode;
}
_destroy_sharedns(xidata);
return -1;
}
*p_xidata = xidata;
}
Py_CLEAR(session->_preserved);
return 0;
}
static int
_finish_preserved(_PyXI_namespace *xidata, PyObject **p_preserved)
{
if (xidata == NULL) {
return 0;
}
int res = -1;
if (p_preserved != NULL) {
PyObject *ns = PyDict_New();
if (ns == NULL) {
goto finally;
}
if (_apply_sharedns(xidata, ns, NULL) < 0) {
Py_CLEAR(ns);
goto finally;
}
*p_preserved = ns;
}
res = 0;
finally:
_destroy_sharedns(xidata);
return res;
}
int
_PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value,
_PyXI_errcode *p_errcode)
{
if (!_session_is_active(session)) {
PyErr_SetString(PyExc_RuntimeError, "session not active");
return -1;
}
if (session->_preserved == NULL) {
session->_preserved = PyDict_New();
if (session->_preserved == NULL) {
set_exc_with_cause(PyExc_RuntimeError,
"failed to initialize preserved objects");
if (p_errcode != NULL) {
*p_errcode = _PyXI_ERR_PRESERVE_FAILURE;
}
return -1;
}
}
if (PyDict_SetItemString(session->_preserved, name, value) < 0) {
set_exc_with_cause(PyExc_RuntimeError, "failed to preserve object");
if (p_errcode != NULL) {
*p_errcode = _PyXI_ERR_PRESERVE_FAILURE;
}
return -1;
}
return 0;
}
PyObject *
_PyXI_GetPreserved(_PyXI_session_result *result, const char *name)
{
PyObject *value = NULL;
if (result->preserved != NULL) {
(void)PyDict_GetItemStringRef(result->preserved, name, &value);
}
return value;
}
void
_PyXI_ClearResult(_PyXI_session_result *result)
{
Py_CLEAR(result->preserved);
Py_CLEAR(result->excinfo);
}
/*********************/
/* runtime lifecycle */
/*********************/

4
Python/import.c
View file

@ -3964,8 +3964,10 @@ PyImport_Import(PyObject *module_name)
if (globals != NULL) {
Py_INCREF(globals);
builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__));
if (builtins == NULL)
if (builtins == NULL) {
// XXX Fall back to interp->builtins or sys.modules['builtins']?
goto err;
}
}
else {
/* No globals -- use standard builtins, and fake globals */