mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-09 16:34:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 17

gh-76785: Consolidate Some Interpreter-related Testing Helpers (gh-117485)

Browse source 
This eliminates the duplication of functionally identical helpers in the _testinternalcapi and _xxsubinterpreters modules.
This commit is contained in:
Eric Snow 2024-04-02 17:16:50 -06:00 • committed by GitHub
parent f341d6017d
commit 857d3151c9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 527 additions and 313 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Lib/test/support/interpreters/__init__.py
View file

@ -73,7 +73,7 @@ class ExecutionFailed(RuntimeError):
def create():
"""Return a new (idle) Python interpreter."""
id = _interpreters.create(isolated=True)
id = _interpreters.create(reqrefs=True)
return Interpreter(id)
@ -109,13 +109,13 @@ class Interpreter:
assert hasattr(self, '_ownsref')
except KeyError:
# This may raise InterpreterNotFoundError:
_interpreters._incref(id)
_interpreters.incref(id)
try:
self = super().__new__(cls)
self._id = id
self._ownsref = True
except BaseException:
_interpreters._deccref(id)
_interpreters.decref(id)
raise
_known[id] = self
return self
@ -142,7 +142,7 @@ class Interpreter:
return
self._ownsref = False
try:
_interpreters._decref(self.id)
_interpreters.decref(self.id)
except InterpreterNotFoundError:
pass

4
Lib/test/test__xxsubinterpreters.py
View file

@ -584,7 +584,7 @@ class RunStringTests(TestBase):
def test_create_daemon_thread(self):
with self.subTest('isolated'):
expected = 'spam spam spam spam spam'
subinterp = interpreters.create(isolated=True)
subinterp = interpreters.create('isolated')
script, file = _captured_script(f"""
import threading
def f():
@ -604,7 +604,7 @@ class RunStringTests(TestBase):
self.assertEqual(out, expected)
with self.subTest('not isolated'):
subinterp = interpreters.create(isolated=False)
subinterp = interpreters.create('legacy')
script, file = _captured_script("""
import threading
def f():

83
Lib/test/test_capi/test_misc.py
View file

@ -2204,6 +2204,7 @@ class SubinterpreterTest(unittest.TestCase):
self.assertEqual(main_attr_id, subinterp_attr_id)
@requires_subinterpreters
class InterpreterConfigTests(unittest.TestCase):
supported = {
@ -2277,11 +2278,11 @@ class InterpreterConfigTests(unittest.TestCase):
expected = self.supported[expected]
args = (name,) if name else ()
config1 = _testinternalcapi.new_interp_config(*args)
config1 = _interpreters.new_config(*args)
self.assert_ns_equal(config1, expected)
self.assertIsNot(config1, expected)
config2 = _testinternalcapi.new_interp_config(*args)
config2 = _interpreters.new_config(*args)
self.assert_ns_equal(config2, expected)
self.assertIsNot(config2, expected)
self.assertIsNot(config2, config1)
@ -2298,7 +2299,7 @@ class InterpreterConfigTests(unittest.TestCase):
with self.subTest(f'noop ({name})'):
expected = vanilla
overrides = vars(vanilla)
config = _testinternalcapi.new_interp_config(name, **overrides)
config = _interpreters.new_config(name, **overrides)
self.assert_ns_equal(config, expected)
with self.subTest(f'change all ({name})'):
@ -2308,7 +2309,7 @@ class InterpreterConfigTests(unittest.TestCase):
continue
overrides['gil'] = gil
expected = types.SimpleNamespace(**overrides)
config = _testinternalcapi.new_interp_config(
config = _interpreters.new_config(
name, **overrides)
self.assert_ns_equal(config, expected)
@ -2324,14 +2325,14 @@ class InterpreterConfigTests(unittest.TestCase):
expected = types.SimpleNamespace(
**dict(vars(vanilla), **overrides),
)
config = _testinternalcapi.new_interp_config(
config = _interpreters.new_config(
name, **overrides)
self.assert_ns_equal(config, expected)
with self.subTest('unsupported field'):
for name in self.supported:
with self.assertRaises(ValueError):
_testinternalcapi.new_interp_config(name, spam=True)
_interpreters.new_config(name, spam=True)
# Bad values for bool fields.
for field, value in vars(self.supported['empty']).items():
@ -2341,19 +2342,18 @@ class InterpreterConfigTests(unittest.TestCase):
for value in [1, '', 'spam', 1.0, None, object()]:
with self.subTest(f'unsupported value ({field}={value!r})'):
with self.assertRaises(TypeError):
_testinternalcapi.new_interp_config(**{field: value})
_interpreters.new_config(**{field: value})
# Bad values for .gil.
for value in [True, 1, 1.0, None, object()]:
with self.subTest(f'unsupported value(gil={value!r})'):
with self.assertRaises(TypeError):
_testinternalcapi.new_interp_config(gil=value)
_interpreters.new_config(gil=value)
for value in ['', 'spam']:
with self.subTest(f'unsupported value (gil={value!r})'):
with self.assertRaises(ValueError):
_testinternalcapi.new_interp_config(gil=value)
_interpreters.new_config(gil=value)
@requires_subinterpreters
def test_interp_init(self):
questionable = [
# strange
@ -2412,11 +2412,10 @@ class InterpreterConfigTests(unittest.TestCase):
with self.subTest(f'valid: {config}'):
check(config)
@requires_subinterpreters
def test_get_config(self):
@contextlib.contextmanager
def new_interp(config):
interpid = _testinternalcapi.new_interpreter(config)
interpid = _interpreters.create(config, reqrefs=False)
try:
yield interpid
finally:
@ -2426,32 +2425,32 @@ class InterpreterConfigTests(unittest.TestCase):
pass
with self.subTest('main'):
expected = _testinternalcapi.new_interp_config('legacy')
expected = _interpreters.new_config('legacy')
expected.gil = 'own'
interpid = _interpreters.get_main()
config = _testinternalcapi.get_interp_config(interpid)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('isolated'):
expected = _testinternalcapi.new_interp_config('isolated')
expected = _interpreters.new_config('isolated')
with new_interp('isolated') as interpid:
config = _testinternalcapi.get_interp_config(interpid)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('legacy'):
expected = _testinternalcapi.new_interp_config('legacy')
expected = _interpreters.new_config('legacy')
with new_interp('legacy') as interpid:
config = _testinternalcapi.get_interp_config(interpid)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('custom'):
orig = _testinternalcapi.new_interp_config(
orig = _interpreters.new_config(
'empty',
use_main_obmalloc=True,
gil='shared',
)
with new_interp(orig) as interpid:
config = _testinternalcapi.get_interp_config(interpid)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, orig)
@ -2529,14 +2528,19 @@ class InterpreterIDTests(unittest.TestCase):
self.assertFalse(
_testinternalcapi.interpreter_exists(interpid))
def get_refcount_helpers(self):
return (
_testinternalcapi.get_interpreter_refcount,
(lambda id: _interpreters.incref(id, implieslink=False)),
_interpreters.decref,
)
def test_linked_lifecycle_does_not_exist(self):
exists = _testinternalcapi.interpreter_exists
is_linked = _testinternalcapi.interpreter_refcount_linked
link = _testinternalcapi.link_interpreter_refcount
unlink = _testinternalcapi.unlink_interpreter_refcount
get_refcount = _testinternalcapi.get_interpreter_refcount
incref = _testinternalcapi.interpreter_incref
decref = _testinternalcapi.interpreter_decref
get_refcount, incref, decref = self.get_refcount_helpers()
with self.subTest('never existed'):
interpid = _testinternalcapi.unused_interpreter_id()
@ -2578,8 +2582,7 @@ class InterpreterIDTests(unittest.TestCase):
get_refcount = _testinternalcapi.get_interpreter_refcount
# A new interpreter will start out not linked, with a refcount of 0.
interpid = _testinternalcapi.new_interpreter()
self.add_interp_cleanup(interpid)
interpid = self.new_interpreter()
linked = is_linked(interpid)
refcount = get_refcount(interpid)
@ -2589,12 +2592,9 @@ class InterpreterIDTests(unittest.TestCase):
def test_linked_lifecycle_never_linked(self):
exists = _testinternalcapi.interpreter_exists
is_linked = _testinternalcapi.interpreter_refcount_linked
get_refcount = _testinternalcapi.get_interpreter_refcount
incref = _testinternalcapi.interpreter_incref
decref = _testinternalcapi.interpreter_decref
get_refcount, incref, decref = self.get_refcount_helpers()
interpid = _testinternalcapi.new_interpreter()
self.add_interp_cleanup(interpid)
interpid = self.new_interpreter()
# Incref will not automatically link it.
incref(interpid)
@ -2618,8 +2618,7 @@ class InterpreterIDTests(unittest.TestCase):
link = _testinternalcapi.link_interpreter_refcount
unlink = _testinternalcapi.unlink_interpreter_refcount
interpid = _testinternalcapi.new_interpreter()
self.add_interp_cleanup(interpid)
interpid = self.new_interpreter()
# Linking at refcount 0 does not destroy the interpreter.
link(interpid)
@ -2639,12 +2638,9 @@ class InterpreterIDTests(unittest.TestCase):
exists = _testinternalcapi.interpreter_exists
is_linked = _testinternalcapi.interpreter_refcount_linked
link = _testinternalcapi.link_interpreter_refcount
get_refcount = _testinternalcapi.get_interpreter_refcount
incref = _testinternalcapi.interpreter_incref
decref = _testinternalcapi.interpreter_decref
get_refcount, incref, decref = self.get_refcount_helpers()
interpid = _testinternalcapi.new_interpreter()
self.add_interp_cleanup(interpid)
interpid = self.new_interpreter()
# Linking it will not change the refcount.
link(interpid)
@ -2666,11 +2662,9 @@ class InterpreterIDTests(unittest.TestCase):
def test_linked_lifecycle_incref_link(self):
is_linked = _testinternalcapi.interpreter_refcount_linked
link = _testinternalcapi.link_interpreter_refcount
get_refcount = _testinternalcapi.get_interpreter_refcount
incref = _testinternalcapi.interpreter_incref
get_refcount, incref, _ = self.get_refcount_helpers()
interpid = _testinternalcapi.new_interpreter()
self.add_interp_cleanup(interpid)
interpid = self.new_interpreter()
incref(interpid)
self.assertEqual(
@ -2688,12 +2682,9 @@ class InterpreterIDTests(unittest.TestCase):
is_linked = _testinternalcapi.interpreter_refcount_linked
link = _testinternalcapi.link_interpreter_refcount
unlink = _testinternalcapi.unlink_interpreter_refcount
get_refcount = _testinternalcapi.get_interpreter_refcount
incref = _testinternalcapi.interpreter_incref
decref = _testinternalcapi.interpreter_decref
get_refcount, incref, decref = self.get_refcount_helpers()
interpid = _testinternalcapi.new_interpreter()
self.add_interp_cleanup(interpid)
interpid = self.new_interpreter()
link(interpid)
self.assertTrue(

2
Lib/test/test_import/__init__.py
View file

@ -2163,7 +2163,7 @@ class SinglephaseInitTests(unittest.TestCase):
# subinterpreters
def add_subinterpreter(self):
interpid = _interpreters.create(isolated=False)
interpid = _interpreters.create('legacy')
def ensure_destroyed():
try:
_interpreters.destroy(interpid)

4
Lib/test/test_importlib/test_util.py
View file

@ -656,7 +656,7 @@ class MagicNumberTests(unittest.TestCase):
class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase):
def run_with_own_gil(self, script):
interpid = _interpreters.create(isolated=True)
interpid = _interpreters.create('isolated')
def ensure_destroyed():
try:
_interpreters.destroy(interpid)
@ -669,7 +669,7 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase):
raise ImportError(excsnap.msg)
def run_with_shared_gil(self, script):
interpid = _interpreters.create(isolated=False)
interpid = _interpreters.create('legacy')
def ensure_destroyed():
try:
_interpreters.destroy(interpid)

211
Lib/test/test_interpreters/test_api.py
View file

@ -1,13 +1,14 @@
import os
import pickle
import threading
from textwrap import dedent
import threading
import types
import unittest
from test import support
from test.support import import_helper
# Raise SkipTest if subinterpreters not supported.
import_helper.import_module('_xxsubinterpreters')
_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters
from test.support.interpreters import InterpreterNotFoundError
from .utils import _captured_script, _run_output, _running, TestBase
@ -932,6 +933,212 @@ class TestIsShareable(TestBase):
interpreters.is_shareable(obj))
class LowLevelTests(TestBase):
# The behaviors in the low-level module are important in as much
# as they are exercised by the high-level module. Therefore the
# most important testing happens in the high-level tests.
# These low-level tests cover corner cases that are not
# encountered by the high-level module, thus they
# mostly shouldn't matter as much.
def test_new_config(self):
# This test overlaps with
# test.test_capi.test_misc.InterpreterConfigTests.
default = _interpreters.new_config('isolated')
with self.subTest('no arg'):
config = _interpreters.new_config()
self.assert_ns_equal(config, default)
self.assertIsNot(config, default)
with self.subTest('default'):
config1 = _interpreters.new_config('default')
self.assert_ns_equal(config1, default)
self.assertIsNot(config1, default)
config2 = _interpreters.new_config('default')
self.assert_ns_equal(config2, config1)
self.assertIsNot(config2, config1)
for arg in ['', 'default']:
with self.subTest(f'default ({arg!r})'):
config = _interpreters.new_config(arg)
self.assert_ns_equal(config, default)
self.assertIsNot(config, default)
supported = {
'isolated': types.SimpleNamespace(
use_main_obmalloc=False,
allow_fork=False,
allow_exec=False,
allow_threads=True,
allow_daemon_threads=False,
check_multi_interp_extensions=True,
gil='own',
),
'legacy': types.SimpleNamespace(
use_main_obmalloc=True,
allow_fork=True,
allow_exec=True,
allow_threads=True,
allow_daemon_threads=True,
check_multi_interp_extensions=False,
gil='shared',
),
'empty': types.SimpleNamespace(
use_main_obmalloc=False,
allow_fork=False,
allow_exec=False,
allow_threads=False,
allow_daemon_threads=False,
check_multi_interp_extensions=False,
gil='default',
),
}
gil_supported = ['default', 'shared', 'own']
for name, vanilla in supported.items():
with self.subTest(f'supported ({name})'):
expected = vanilla
config1 = _interpreters.new_config(name)
self.assert_ns_equal(config1, expected)
self.assertIsNot(config1, expected)
config2 = _interpreters.new_config(name)
self.assert_ns_equal(config2, config1)
self.assertIsNot(config2, config1)
with self.subTest(f'noop override ({name})'):
expected = vanilla
overrides = vars(vanilla)
config = _interpreters.new_config(name, **overrides)
self.assert_ns_equal(config, expected)
with self.subTest(f'override all ({name})'):
overrides = {k: not v for k, v in vars(vanilla).items()}
for gil in gil_supported:
if vanilla.gil == gil:
continue
overrides['gil'] = gil
expected = types.SimpleNamespace(**overrides)
config = _interpreters.new_config(name, **overrides)
self.assert_ns_equal(config, expected)
# Override individual fields.
for field, old in vars(vanilla).items():
if field == 'gil':
values = [v for v in gil_supported if v != old]
else:
values = [not old]
for val in values:
with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'):
overrides = {field: val}
expected = types.SimpleNamespace(
**dict(vars(vanilla), **overrides),
)
config = _interpreters.new_config(name, **overrides)
self.assert_ns_equal(config, expected)
with self.subTest('extra override'):
with self.assertRaises(ValueError):
_interpreters.new_config(spam=True)
# Bad values for bool fields.
for field, value in vars(supported['empty']).items():
if field == 'gil':
continue
assert isinstance(value, bool)
for value in [1, '', 'spam', 1.0, None, object()]:
with self.subTest(f'bad override ({field}={value!r})'):
with self.assertRaises(TypeError):
_interpreters.new_config(**{field: value})
# Bad values for .gil.
for value in [True, 1, 1.0, None, object()]:
with self.subTest(f'bad override (gil={value!r})'):
with self.assertRaises(TypeError):
_interpreters.new_config(gil=value)
for value in ['', 'spam']:
with self.subTest(f'bad override (gil={value!r})'):
with self.assertRaises(ValueError):
_interpreters.new_config(gil=value)
def test_get_config(self):
# This test overlaps with
# test.test_capi.test_misc.InterpreterConfigTests.
with self.subTest('main'):
expected = _interpreters.new_config('legacy')
expected.gil = 'own'
interpid = _interpreters.get_main()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('isolated'):
expected = _interpreters.new_config('isolated')
interpid = _interpreters.create('isolated')
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('legacy'):
expected = _interpreters.new_config('legacy')
interpid = _interpreters.create('legacy')
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
def test_create(self):
isolated = _interpreters.new_config('isolated')
legacy = _interpreters.new_config('legacy')
default = isolated
with self.subTest('no arg'):
interpid = _interpreters.create()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, default)
with self.subTest('arg: None'):
interpid = _interpreters.create(None)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, default)
with self.subTest('arg: \'empty\''):
with self.assertRaises(RuntimeError):
# The "empty" config isn't viable on its own.
_interpreters.create('empty')
for arg, expected in {
'': default,
'default': default,
'isolated': isolated,
'legacy': legacy,
}.items():
with self.subTest(f'str arg: {arg!r}'):
interpid = _interpreters.create(arg)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('custom'):
orig = _interpreters.new_config('empty')
orig.use_main_obmalloc = True
orig.gil = 'shared'
interpid = _interpreters.create(orig)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, orig)
with self.subTest('missing fields'):
orig = _interpreters.new_config()
del orig.gil
with self.assertRaises(ValueError):
_interpreters.create(orig)
with self.subTest('extra fields'):
orig = _interpreters.new_config()
orig.spam = True
with self.assertRaises(ValueError):
_interpreters.create(orig)
if __name__ == '__main__':
# Test needs to be a package, so we can do relative imports.
unittest.main()

6
Lib/test/test_interpreters/test_queues.py
View file

@ -28,9 +28,9 @@ class TestBase(_TestBase):
class LowLevelTests(TestBase):
# The behaviors in the low-level module is important in as much
# as it is exercised by the high-level module. Therefore the
# most # important testing happens in the high-level tests.
# The behaviors in the low-level module are important in as much
# as they are exercised by the high-level module. Therefore the
# most important testing happens in the high-level tests.
# These low-level tests cover corner cases that are not
# encountered by the high-level module, thus they
# mostly shouldn't matter as much.

21
Lib/test/test_interpreters/utils.py
View file

@ -68,6 +68,9 @@ def _running(interp):
class TestBase(unittest.TestCase):
def tearDown(self):
clean_up_interpreters()
def pipe(self):
def ensure_closed(fd):
try:
@ -156,5 +159,19 @@ class TestBase(unittest.TestCase):
self.assertNotEqual(exitcode, 0)
return stdout, stderr
def tearDown(self):
clean_up_interpreters()
def assert_ns_equal(self, ns1, ns2, msg=None):
# This is mostly copied from TestCase.assertDictEqual.
self.assertEqual(type(ns1), type(ns2))
if ns1 == ns2:
return
import difflib
import pprint
from unittest.util import _common_shorten_repr
standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2)
diff = ('\n' + '\n'.join(difflib.ndiff(
pprint.pformat(vars(ns1)).splitlines(),
pprint.pformat(vars(ns2)).splitlines())))
diff = f'namespace({diff})'
standardMsg = self._truncateMessage(standardMsg, diff)
self.fail(self._formatMessage(msg, standardMsg))

215
Modules/_testinternalcapi.c
View file

@ -23,7 +23,6 @@
#include "pycore_initconfig.h" // _Py_GetConfigsAsDict()
#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy()
#include "pycore_long.h" // _PyLong_Sign()
#include "pycore_namespace.h" // _PyNamespace_New()
#include "pycore_object.h" // _PyObject_IsFreed()
#include "pycore_optimizer.h" // _Py_UopsSymbol, etc.
#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal()
@ -831,6 +830,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module,
}
// Maybe this could be replaced by get_interpreter_config()?
static PyObject *
get_interp_settings(PyObject *self, PyObject *args)
{
@ -1357,129 +1357,6 @@ dict_getitem_knownhash(PyObject *self, PyObject *args)
}
static int
init_named_interp_config(PyInterpreterConfig *config, const char *name)
{
if (name == NULL) {
name = "isolated";
}
if (strcmp(name, "isolated") == 0) {
*config = (PyInterpreterConfig)_PyInterpreterConfig_INIT;
}
else if (strcmp(name, "legacy") == 0) {
*config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT;
}
else if (strcmp(name, "empty") == 0) {
*config = (PyInterpreterConfig){0};
}
else {
PyErr_Format(PyExc_ValueError,
"unsupported config name '%s'", name);
return -1;
}
return 0;
}
static PyObject *
new_interp_config(PyObject *self, PyObject *args, PyObject *kwds)
{
const char *name = NULL;
if (!PyArg_ParseTuple(args, "|s:new_config", &name)) {
return NULL;
}
PyObject *overrides = kwds;
if (name == NULL) {
name = "isolated";
}
PyInterpreterConfig config;
if (init_named_interp_config(&config, name) < 0) {
return NULL;
}
if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) {
if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) {
return NULL;
}
}
PyObject *dict = _PyInterpreterConfig_AsDict(&config);
if (dict == NULL) {
return NULL;
}
PyObject *configobj = _PyNamespace_New(dict);
Py_DECREF(dict);
return configobj;
}
static PyObject *
get_interp_config(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *idobj = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:get_config", kwlist, &idobj))
{
return NULL;
}
PyInterpreterState *interp;
if (idobj == NULL) {
interp = PyInterpreterState_Get();
}
else {
interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
}
PyInterpreterConfig config;
if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) {
return NULL;
}
PyObject *dict = _PyInterpreterConfig_AsDict(&config);
if (dict == NULL) {
return NULL;
}
PyObject *configobj = _PyNamespace_New(dict);
Py_DECREF(dict);
return configobj;
}
static int
interp_config_from_object(PyObject *configobj, PyInterpreterConfig *config)
{
if (configobj == NULL || configobj == Py_None) {
if (init_named_interp_config(config, NULL) < 0) {
return -1;
}
}
else if (PyUnicode_Check(configobj)) {
if (init_named_interp_config(config, PyUnicode_AsUTF8(configobj)) < 0) {
return -1;
}
}
else {
PyObject *dict = PyObject_GetAttrString(configobj, "__dict__");
if (dict == NULL) {
PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
return -1;
}
int res = _PyInterpreterConfig_InitFromDict(config, dict);
Py_DECREF(dict);
if (res < 0) {
return -1;
}
}
return 0;
}
/* To run some code in a sub-interpreter. */
static PyObject *
run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
@ -1495,7 +1372,14 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
}
PyInterpreterConfig config;
if (interp_config_from_object(configobj, &config) < 0) {
PyObject *dict = PyObject_GetAttrString(configobj, "__dict__");
if (dict == NULL) {
PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
return NULL;
}
int res = _PyInterpreterConfig_InitFromDict(&config, dict);
Py_DECREF(dict);
if (res < 0) {
return NULL;
}
@ -1546,58 +1430,6 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored))
return PyLong_FromLongLong(interpid);
}
static PyObject *
new_interpreter(PyObject *self, PyObject *args)
{
PyObject *configobj = NULL;
if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) {
return NULL;
}
PyInterpreterConfig config;
if (interp_config_from_object(configobj, &config) < 0) {
return NULL;
}
// Unlike _interpreters.create(), we do not automatically link
// the interpreter to its refcount.
PyThreadState *save_tstate = PyThreadState_Get();
PyThreadState *tstate = NULL;
PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config);
PyThreadState_Swap(save_tstate);
if (PyStatus_Exception(status)) {
_PyErr_SetFromPyStatus(status);
return NULL;
}
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
if (_PyInterpreterState_IDInitref(interp) < 0) {
goto error;
}
int64_t interpid = PyInterpreterState_GetID(interp);
if (interpid < 0) {
goto error;
}
PyObject *idobj = PyLong_FromLongLong(interpid);
if (idobj == NULL) {
goto error;
}
PyThreadState_Swap(tstate);
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
return idobj;
error:
save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
return NULL;
}
static PyObject *
interpreter_exists(PyObject *self, PyObject *idobj)
{
@ -1660,28 +1492,6 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj)
Py_RETURN_FALSE;
}
static PyObject *
interpreter_incref(PyObject *self, PyObject *idobj)
{
PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
_PyInterpreterState_IDIncref(interp);
Py_RETURN_NONE;
}
static PyObject *
interpreter_decref(PyObject *self, PyObject *idobj)
{
PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
_PyInterpreterState_IDDecref(interp);
Py_RETURN_NONE;
}
static void
_xid_capsule_destructor(PyObject *capsule)
@ -1928,23 +1738,16 @@ static PyMethodDef module_functions[] = {
{"get_object_dict_values", get_object_dict_values, METH_O},
{"hamt", new_hamt, METH_NOARGS},
{"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS},
{"new_interp_config", _PyCFunction_CAST(new_interp_config),
METH_VARARGS | METH_KEYWORDS},
{"get_interp_config", _PyCFunction_CAST(get_interp_config),
METH_VARARGS | METH_KEYWORDS},
{"run_in_subinterp_with_config",
_PyCFunction_CAST(run_in_subinterp_with_config),
METH_VARARGS | METH_KEYWORDS},
{"normalize_interp_id", normalize_interp_id, METH_O},
{"unused_interpreter_id", unused_interpreter_id, METH_NOARGS},
{"new_interpreter", new_interpreter, METH_VARARGS},
{"interpreter_exists", interpreter_exists, METH_O},
{"get_interpreter_refcount", get_interpreter_refcount, METH_O},
{"link_interpreter_refcount", link_interpreter_refcount, METH_O},
{"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O},
{"interpreter_refcount_linked", interpreter_refcount_linked, METH_O},
{"interpreter_incref", interpreter_incref, METH_O},
{"interpreter_decref", interpreter_decref, METH_O},
{"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS},
{"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS},
{"get_crossinterp_data", get_crossinterp_data, METH_VARARGS},

286
Modules/_xxsubinterpretersmodule.c
View file

@ -12,8 +12,10 @@
#include "pycore_initconfig.h" // _PyErr_SetFromPyStatus()
#include "pycore_long.h" // _PyLong_IsNegative()
#include "pycore_modsupport.h" // _PyArg_BadArgument()
#include "pycore_namespace.h" // _PyNamespace_New()
#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree()
#include "pycore_pyerrors.h" // _Py_excinfo
#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict()
#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain()
#include "marshal.h" // PyMarshal_ReadObjectFromString()
@ -367,6 +369,115 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p)
/* interpreter-specific code ************************************************/
static int
init_named_config(PyInterpreterConfig *config, const char *name)
{
if (name == NULL
|| strcmp(name, "") == 0
|| strcmp(name, "default") == 0)
{
name = "isolated";
}
if (strcmp(name, "isolated") == 0) {
*config = (PyInterpreterConfig)_PyInterpreterConfig_INIT;
}
else if (strcmp(name, "legacy") == 0) {
*config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT;
}
else if (strcmp(name, "empty") == 0) {
*config = (PyInterpreterConfig){0};
}
else {
PyErr_Format(PyExc_ValueError,
"unsupported config name '%s'", name);
return -1;
}
return 0;
}
static int
config_from_object(PyObject *configobj, PyInterpreterConfig *config)
{
if (configobj == NULL || configobj == Py_None) {
if (init_named_config(config, NULL) < 0) {
return -1;
}
}
else if (PyUnicode_Check(configobj)) {
if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) {
return -1;
}
}
else {
PyObject *dict = PyObject_GetAttrString(configobj, "__dict__");
if (dict == NULL) {
PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
return -1;
}
int res = _PyInterpreterConfig_InitFromDict(config, dict);
Py_DECREF(dict);
if (res < 0) {
return -1;
}
}
return 0;
}
static PyInterpreterState *
new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate)
{
PyThreadState *save_tstate = PyThreadState_Get();
assert(save_tstate != NULL);
PyThreadState *tstate = NULL;
// XXX Possible GILState issues?
PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
PyThreadState_Swap(save_tstate);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
_PyErr_SetFromPyStatus(status);
return NULL;
}
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
if (_PyInterpreterState_IDInitref(interp) < 0) {
goto error;
}
if (p_idobj != NULL) {
// We create the object using the original interpreter.
PyObject *idobj = get_interpid_obj(interp);
if (idobj == NULL) {
goto error;
}
*p_idobj = idobj;
}
if (p_tstate != NULL) {
*p_tstate = tstate;
}
else {
PyThreadState_Swap(tstate);
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
}
return interp;
error:
// XXX Possible GILState issues?
save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
return NULL;
}
static int
_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags)
{
@ -436,64 +547,98 @@ _run_in_interpreter(PyInterpreterState *interp,
/* module level code ********************************************************/
static PyObject *
interp_create(PyObject *self, PyObject *args, PyObject *kwds)
interp_new_config(PyObject *self, PyObject *args, PyObject *kwds)
{
const char *name = NULL;
if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config",
&name))
{
return NULL;
}
PyObject *overrides = kwds;
static char *kwlist[] = {"isolated", NULL};
int isolated = 1;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist,
&isolated)) {
PyInterpreterConfig config;
if (init_named_config(&config, name) < 0) {
return NULL;
}
// Create and initialize the new interpreter.
PyThreadState *save_tstate = PyThreadState_Get();
assert(save_tstate != NULL);
const PyInterpreterConfig config = isolated
? (PyInterpreterConfig)_PyInterpreterConfig_INIT
: (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT;
if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) {
if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) {
return NULL;
}
}
// XXX Possible GILState issues?
PyThreadState *tstate = NULL;
PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config);
PyThreadState_Swap(save_tstate);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
_PyErr_SetFromPyStatus(status);
PyObject *dict = _PyInterpreterConfig_AsDict(&config);
if (dict == NULL) {
return NULL;
}
PyObject *configobj = _PyNamespace_New(dict);
Py_DECREF(dict);
return configobj;
}
PyDoc_STRVAR(new_config_doc,
"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\
\n\
Return a representation of a new PyInterpreterConfig.\n\
\n\
The name determines the initial values of the config. Supported named\n\
configs are: default, isolated, legacy, and empty.\n\
\n\
Any keyword arguments are set on the corresponding config fields,\n\
overriding the initial values.");
static PyObject *
interp_create(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"config", "reqrefs", NULL};
PyObject *configobj = NULL;
int reqrefs = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist,
&configobj, &reqrefs)) {
return NULL;
}
PyInterpreterConfig config;
if (config_from_object(configobj, &config) < 0) {
return NULL;
}
PyObject *idobj = NULL;
PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL);
if (interp == NULL) {
// XXX Move the chained exception to interpreters.create()?
PyObject *exc = PyErr_GetRaisedException();
assert(exc != NULL);
PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed");
_PyErr_ChainExceptions1(exc);
return NULL;
}
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
PyObject *idobj = get_interpid_obj(interp);
if (idobj == NULL) {
// XXX Possible GILState issues?
save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
return NULL;
if (reqrefs) {
// Decref to 0 will destroy the interpreter.
_PyInterpreterState_RequireIDRef(interp, 1);
}
PyThreadState_Swap(tstate);
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
_PyInterpreterState_RequireIDRef(interp, 1);
return idobj;
}
PyDoc_STRVAR(create_doc,
"create() -> ID\n\
"create([config], *, reqrefs=False) -> ID\n\
\n\
Create a new interpreter and return a unique generated ID.\n\
\n\
The caller is responsible for destroying the interpreter before exiting.");
The caller is responsible for destroying the interpreter before exiting,\n\
typically by using _interpreters.destroy(). This can be managed \n\
automatically by passing \"reqrefs=True\" and then using _incref() and\n\
_decref()` appropriately.\n\
\n\
\"config\" must be a valid interpreter config or the name of a\n\
predefined config (\"isolated\" or \"legacy\"). The default\n\
is \"isolated\".");
static PyObject *
@ -1008,12 +1153,57 @@ Return whether or not the identified interpreter is running.");
static PyObject *
interp_incref(PyObject *self, PyObject *args, PyObject *kwds)
interp_get_config(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *id;
PyObject *idobj = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:_incref", kwlist, &id)) {
"O:get_config", kwlist, &idobj))
{
return NULL;
}
PyInterpreterState *interp;
if (idobj == NULL) {
interp = PyInterpreterState_Get();
}
else {
interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
}
PyInterpreterConfig config;
if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) {
return NULL;
}
PyObject *dict = _PyInterpreterConfig_AsDict(&config);
if (dict == NULL) {
return NULL;
}
PyObject *configobj = _PyNamespace_New(dict);
Py_DECREF(dict);
return configobj;
}
PyDoc_STRVAR(get_config_doc,
"get_config(id) -> types.SimpleNamespace\n\
\n\
Return a representation of the config used to initialize the interpreter.");
static PyObject *
interp_incref(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", "implieslink", NULL};
PyObject *id;
int implieslink = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O|$p:incref", kwlist,
&id, &implieslink))
{
return NULL;
}
@ -1021,8 +1211,10 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds)
if (interp == NULL) {
return NULL;
}
if (_PyInterpreterState_IDInitref(interp) < 0) {
return NULL;
if (implieslink) {
// Decref to 0 will destroy the interpreter.
_PyInterpreterState_RequireIDRef(interp, 1);
}
_PyInterpreterState_IDIncref(interp);
@ -1036,7 +1228,7 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds)
static char *kwlist[] = {"id", NULL};
PyObject *id;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:_incref", kwlist, &id)) {
"O:decref", kwlist, &id)) {
return NULL;
}
@ -1051,6 +1243,8 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds)
static PyMethodDef module_functions[] = {
{"new_config", _PyCFunction_CAST(interp_new_config),
METH_VARARGS | METH_KEYWORDS, new_config_doc},
{"create", _PyCFunction_CAST(interp_create),
METH_VARARGS | METH_KEYWORDS, create_doc},
{"destroy", _PyCFunction_CAST(interp_destroy),
@ -1064,6 +1258,8 @@ static PyMethodDef module_functions[] = {
{"is_running", _PyCFunction_CAST(interp_is_running),
METH_VARARGS | METH_KEYWORDS, is_running_doc},
{"get_config", _PyCFunction_CAST(interp_get_config),
METH_VARARGS | METH_KEYWORDS, get_config_doc},
{"exec", _PyCFunction_CAST(interp_exec),
METH_VARARGS | METH_KEYWORDS, exec_doc},
{"call", _PyCFunction_CAST(interp_call),
@ -1078,9 +1274,9 @@ static PyMethodDef module_functions[] = {
{"is_shareable", _PyCFunction_CAST(object_is_shareable),
METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"_incref", _PyCFunction_CAST(interp_incref),
{"incref", _PyCFunction_CAST(interp_incref),
METH_VARARGS | METH_KEYWORDS, NULL},
{"_decref", _PyCFunction_CAST(interp_decref),
{"decref", _PyCFunction_CAST(interp_decref),
METH_VARARGS | METH_KEYWORDS, NULL},
{NULL, NULL} /* sentinel */