mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-26 11:45:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Revert "GH-128914: Remove conditional stack effects from bytecodes.c and the code generators (GH-128918)" (GH-129202)

Browse source 
The commit introduced a ~2.5-3% regression in the free threading build.

This reverts commit ab61d3f430.
This commit is contained in:
Sam Gross 2025-01-23 04:26:25 -05:00 • committed by GitHub
parent d7d066c3ab
commit a10f99375e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 1679 additions and 1460 deletions
Download patch file Download diff file Expand all files Collapse all files

217
Python/bytecodes.c
View file

@ -348,8 +348,8 @@ dummy_func(
DECREF_INPUTS();
}
pure inst(PUSH_NULL, (-- null)) {
null = PyStackRef_NULL;
pure inst(PUSH_NULL, (-- res)) {
res = PyStackRef_NULL;
}
no_save_ip inst(END_FOR, (value -- )) {
@ -1654,7 +1654,7 @@ dummy_func(
specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) {
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
DISPATCH_SAME_OPARG();
@ -1665,10 +1665,11 @@ dummy_func(
}
// res[1] because we need a pointer to res to pass it to _PyEval_LoadGlobalStackRef
op(_LOAD_GLOBAL, ( -- res[1])) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
op(_LOAD_GLOBAL, ( -- res[1], null if (oparg & 1))) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
_PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res);
ERROR_IF(PyStackRef_IsNull(*res), error);
null = PyStackRef_NULL;
}
macro(LOAD_GLOBAL) =
@ -1706,7 +1707,7 @@ dummy_func(
assert(DK_IS_UNICODE(builtins_keys));
}
op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res)) {
op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res, null if (oparg & 1))) {
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
DEAD(globals_keys);
@ -1720,9 +1721,10 @@ dummy_func(
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
}
op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res)) {
op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res, null if (oparg & 1))) {
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
DEAD(builtins_keys);
@ -1736,6 +1738,7 @@ dummy_func(
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
}
macro(LOAD_GLOBAL_MODULE) =
@ -2009,27 +2012,17 @@ dummy_func(
GO_TO_INSTRUCTION(LOAD_SUPER_ATTR);
}
inst(INSTRUMENTED_LOAD_SUPER_METHOD, (unused/1 -- )) {
// cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
// don't want to specialize instrumented instructions
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(LOAD_SUPER_METHOD);
}
family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = {
LOAD_SUPER_ATTR_ATTR,
};
family(LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = {
LOAD_SUPER_METHOD_METHOD,
LOAD_SUPER_ATTR_METHOD,
};
specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) {
#if ENABLE_SPECIALIZATION_FT
int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0);
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR);
@ -2037,24 +2030,12 @@ dummy_func(
#endif /* ENABLE_SPECIALIZATION_FT */
}
specializing op(_SPECIALIZE_LOAD_SUPER_METHOD, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) {
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr)) {
tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) {
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
if (opcode >= MIN_INSTRUMENTED_OPCODE) {
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
@ -2068,7 +2049,7 @@ dummy_func(
// handle any case whose performance we care about
PyObject *stack[] = {class, self};
PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL);
if (opcode >= MIN_INSTRUMENTED_OPCODE) {
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
if (super == NULL) {
_Py_call_instrumentation_exc2(
@ -2091,13 +2072,12 @@ dummy_func(
Py_DECREF(super);
ERROR_IF(attr_o == NULL, error);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
}
macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR;
macro(LOAD_SUPER_METHOD) = _SPECIALIZE_LOAD_SUPER_METHOD + _LOAD_SUPER_ATTR + PUSH_NULL;
inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st)) {
inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st, unused if (0))) {
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
@ -2113,7 +2093,7 @@ dummy_func(
attr_st = PyStackRef_FromPyObjectSteal(attr);
}
inst(LOAD_SUPER_METHOD_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) {
inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) {
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
@ -2152,20 +2132,17 @@ dummy_func(
LOAD_ATTR_CLASS_WITH_METACLASS_CHECK,
LOAD_ATTR_PROPERTY,
LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN,
LOAD_ATTR_METHOD_WITH_VALUES,
LOAD_ATTR_METHOD_NO_DICT,
LOAD_ATTR_METHOD_LAZY_DICT,
LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES,
LOAD_ATTR_NONDESCRIPTOR_NO_DICT,
};
family(LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR) = {
LOAD_METHOD_WITH_VALUES,
LOAD_METHOD_NO_DICT,
LOAD_METHOD_LAZY_DICT,
};
specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) {
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_Py_Specialize_LoadAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG();
@ -2175,67 +2152,50 @@ dummy_func(
#endif /* ENABLE_SPECIALIZATION_FT */
}
specializing op(_SPECIALIZE_LOAD_METHOD, (counter/1, owner -- owner)) {
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
_Py_Specialize_LoadMethod(owner, next_instr, name);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_METHOD);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
op(_LOAD_METHOD, (owner -- attr, self_or_null)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
PyObject *attr_o;
/* Designed to work in tandem with CALL, pushes two values. */
attr_o = NULL;
int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
if (is_meth) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
attr_o = NULL;
int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
if (is_meth) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
DEAD(owner);
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
DEAD(owner);
DECREF_INPUTS();
ERROR_IF(attr_o == NULL, error);
self_or_null = PyStackRef_NULL;
}
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
/* Classic, pushes one value. */
attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
DECREF_INPUTS();
ERROR_IF(attr_o == NULL, error);
/* We need to define self_or_null on all paths */
self_or_null = PyStackRef_NULL;
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
op(_LOAD_ATTR, (owner -- attr)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
DECREF_INPUTS();
ERROR_IF(attr_o == NULL, error);
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
macro(LOAD_ATTR) =
_SPECIALIZE_LOAD_ATTR +
unused/8 +
_LOAD_ATTR;
macro(LOAD_METHOD) =
_SPECIALIZE_LOAD_METHOD +
unused/8 +
_LOAD_METHOD;
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
@ -2260,7 +2220,7 @@ dummy_func(
DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid));
}
op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) {
split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
@ -2273,6 +2233,7 @@ dummy_func(
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
@ -2293,7 +2254,7 @@ dummy_func(
mod_keys = keys;
}
op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr)) {
op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr, null if (oparg & 1))) {
assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
@ -2311,6 +2272,7 @@ dummy_func(
attr = PyStackRef_FromPyObjectSteal(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
@ -2330,7 +2292,7 @@ dummy_func(
dict = dict_o;
}
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr)) {
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) {
PyObject *attr_o;
if (!LOCK_OBJECT(dict)) {
POP_INPUT(dict);
@ -2342,7 +2304,7 @@ dummy_func(
POP_INPUT(dict);
DEOPT_IF(true);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
UNLOCK_OBJECT(dict);
POP_INPUT(dict);
@ -2364,6 +2326,7 @@ dummy_func(
attr = PyStackRef_FromPyObjectNew(attr_o);
UNLOCK_OBJECT(dict);
DEAD(dict);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
@ -2374,7 +2337,7 @@ dummy_func(
_LOAD_ATTR_WITH_HINT +
unused/5;
op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) {
split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **addr = (PyObject **)((char *)owner_o + index);
@ -2387,6 +2350,7 @@ dummy_func(
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
@ -2404,10 +2368,11 @@ dummy_func(
EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version);
}
op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) {
split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) {
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
@ -2424,6 +2389,7 @@ dummy_func(
_LOAD_ATTR_CLASS;
op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _PyInterpreterFrame *)) {
assert((oparg & 1) == 0);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
PyCodeObject *code = (PyCodeObject *)f->func_code;
@ -2446,8 +2412,10 @@ dummy_func(
_SAVE_RETURN_OFFSET +
_PUSH_FRAME;
inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused)) {
inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
@ -2461,7 +2429,7 @@ dummy_func(
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(
tstate, PyStackRef_FromPyObjectNew(f), 2, frame);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
@ -3380,7 +3348,8 @@ dummy_func(
DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version);
}
op(_LOAD_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) {
split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) {
assert(oparg & 1);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -3390,14 +3359,15 @@ dummy_func(
DEAD(owner);
}
macro(LOAD_METHOD_WITH_VALUES) =
macro(LOAD_ATTR_METHOD_WITH_VALUES) =
unused/1 +
_GUARD_TYPE_VERSION +
_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT +
_GUARD_KEYS_VERSION +
_LOAD_METHOD_WITH_VALUES;
_LOAD_ATTR_METHOD_WITH_VALUES;
op(_LOAD_METHOD_NO_DICT, (descr/4, owner -- attr, self)) {
op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) {
assert(oparg & 1);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -3407,13 +3377,14 @@ dummy_func(
DEAD(owner);
}
macro(LOAD_METHOD_NO_DICT) =
macro(LOAD_ATTR_METHOD_NO_DICT) =
unused/1 +
_GUARD_TYPE_VERSION +
unused/2 +
_LOAD_METHOD_NO_DICT;
_LOAD_ATTR_METHOD_NO_DICT;
op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) {
op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr, unused if (0))) {
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
DECREF_INPUTS();
@ -3427,7 +3398,8 @@ dummy_func(
_GUARD_KEYS_VERSION +
_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) {
op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) {
assert((oparg & 1) == 0);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -3448,7 +3420,8 @@ dummy_func(
DEOPT_IF(dict != NULL);
}
op(_LOAD_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) {
op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) {
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
@ -3457,12 +3430,12 @@ dummy_func(
DEAD(owner);
}
macro(LOAD_METHOD_LAZY_DICT) =
macro(LOAD_ATTR_METHOD_LAZY_DICT) =
unused/1 +
_GUARD_TYPE_VERSION +
_CHECK_ATTR_METHOD_LAZY_DICT +
unused/1 +
_LOAD_METHOD_LAZY_DICT;
_LOAD_ATTR_METHOD_LAZY_DICT;
// Cache layout: counter/1, func_version/2
// CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members!
@ -4564,7 +4537,7 @@ dummy_func(
GO_TO_INSTRUCTION(CALL_FUNCTION_EX);
}
op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in -- func, unused, tuple, kwargs_out)) {
op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) {
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (PyTuple_CheckExact(callargs_o)) {
tuple = callargs;
@ -4588,7 +4561,7 @@ dummy_func(
}
}
op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st -- result)) {
op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
// DICT_MERGE is called before this opcode if there are kwargs.
@ -4722,16 +4695,11 @@ dummy_func(
LLTRACE_RESUME_FRAME();
}
inst(BUILD_SLICE, (args[oparg] -- slice)) {
assert(oparg == 2 || oparg == 3);
_PyStackRef start = args[0];
_PyStackRef stop = args[1];
inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) {
PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
PyObject * step_o = NULL;
if (oparg == 3) {
step_o = PyStackRef_AsPyObjectBorrow(args[2]);
}
PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
DECREF_INPUTS();
ERROR_IF(slice_o == NULL, error);
@ -5074,25 +5042,27 @@ dummy_func(
DEOPT_IF(func->func_version != func_version);
}
tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res)) {
tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) {
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
PyObject *res_o = entries[index].me_value;
DEOPT_IF(res_o == NULL);
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
null = PyStackRef_NULL;
}
tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res)) {
tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) {
PyDictObject *dict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
PyObject *res_o = entries[index].me_value;
DEOPT_IF(res_o == NULL);
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
null = PyStackRef_NULL;
}
tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr)) {
tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
@ -5103,6 +5073,7 @@ dummy_func(
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
DECREF_INPUTS();
}

27
Python/codegen.c
View file

@ -348,6 +348,8 @@ codegen_addop_o(compiler *c, location loc,
RETURN_IF_ERROR_IN_SCOPE((C), ret); \
} while (0)
#define LOAD_METHOD -1
#define LOAD_SUPER_METHOD -2
#define LOAD_ZERO_SUPER_ATTR -3
#define LOAD_ZERO_SUPER_METHOD -4
@ -364,11 +366,20 @@ codegen_addop_name(compiler *c, location loc,
if (arg < 0) {
return ERROR;
}
if (opcode == LOAD_ATTR) {
arg <<= 1;
}
if (opcode == LOAD_METHOD) {
opcode = LOAD_ATTR;
arg <<= 1;
arg |= 1;
}
if (opcode == LOAD_SUPER_ATTR) {
arg <<= 2;
arg |= 2;
}
if (opcode == LOAD_SUPER_METHOD) {
opcode = LOAD_SUPER_ATTR;
arg <<= 2;
arg |= 3;
}
@ -377,7 +388,7 @@ codegen_addop_name(compiler *c, location loc,
arg <<= 2;
}
if (opcode == LOAD_ZERO_SUPER_METHOD) {
opcode = LOAD_SUPER_METHOD;
opcode = LOAD_SUPER_ATTR;
arg <<= 2;
arg |= 1;
}
@ -3154,6 +3165,9 @@ codegen_nameop(compiler *c, location loc,
assert(op);
Py_DECREF(mangled);
if (op == LOAD_GLOBAL) {
arg <<= 1;
}
ADDOP_I(c, loc, op, arg);
return SUCCESS;
@ -4094,10 +4108,7 @@ ex_call:
}
assert(have_dict);
}
if (nkwelts == 0) {
ADDOP(c, loc, PUSH_NULL);
}
ADDOP(c, loc, CALL_FUNCTION_EX);
ADDOP_I(c, loc, CALL_FUNCTION_EX, nkwelts > 0);
return SUCCESS;
}
@ -4830,10 +4841,8 @@ codegen_async_with(compiler *c, stmt_ty s, int pos)
SETUP_WITH E
<code to store to VAR> or POP_TOP
<code for BLOCK>
LOAD_CONST None
LOAD_CONST None
LOAD_CONST None
CALL 3
LOAD_CONST (None, None, None)
CALL_FUNCTION_EX 0
JUMP EXIT
E: WITH_EXCEPT_START (calls EXPR.__exit__)
POP_JUMP_IF_TRUE T:

306
Python/executor_cases.c.h generated
View file

@ -406,9 +406,9 @@
}
case _PUSH_NULL: {
_PyStackRef null;
null = PyStackRef_NULL;
stack_pointer[0] = null;
_PyStackRef res;
res = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
@ -1953,14 +1953,17 @@
case _LOAD_GLOBAL: {
_PyStackRef *res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
res = &stack_pointer[0];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(*res)) JUMP_TO_ERROR();
stack_pointer += 1;
null = PyStackRef_NULL;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2026,6 +2029,8 @@
case _LOAD_GLOBAL_MODULE_FROM_KEYS: {
PyDictKeysObject *globals_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
@ -2047,8 +2052,10 @@
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2056,6 +2063,8 @@
case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: {
PyDictKeysObject *builtins_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
@ -2077,8 +2086,10 @@
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2508,8 +2519,6 @@
/* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_LOAD_SUPER_METHOD is not a viable micro-op for tier 2 because it is instrumented */
case _LOAD_SUPER_ATTR_ATTR: {
_PyStackRef self_st;
_PyStackRef class_st;
@ -2547,7 +2556,7 @@
break;
}
case _LOAD_SUPER_METHOD_METHOD: {
case _LOAD_SUPER_ATTR_METHOD: {
_PyStackRef self_st;
_PyStackRef class_st;
_PyStackRef global_super_st;
@ -2596,62 +2605,58 @@
break;
}
case _LOAD_METHOD: {
case _LOAD_ATTR: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self_or_null;
_PyStackRef self_or_null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
PyObject *attr_o;
/* Designed to work in tandem with CALL, pushes two values. */
attr_o = NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (is_meth) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
attr_o = NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (is_meth) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
PyStackRef_CLOSE(owner);
if (attr_o == NULL) JUMP_TO_ERROR();
self_or_null = PyStackRef_NULL;
}
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
/* Classic, pushes one value. */
_PyFrame_SetStackPointer(frame, stack_pointer);
attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) JUMP_TO_ERROR();
/* We need to define self_or_null on all paths */
self_or_null = PyStackRef_NULL;
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer[-1] = attr;
stack_pointer[0] = self_or_null;
stack_pointer += 1;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR: {
_PyStackRef owner;
_PyStackRef attr;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) JUMP_TO_ERROR();
attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer[-1] = attr;
break;
}
case _GUARD_TYPE_VERSION: {
_PyStackRef owner;
owner = stack_pointer[-1];
@ -2699,9 +2704,11 @@
break;
}
case _LOAD_ATTR_INSTANCE_VALUE: {
case _LOAD_ATTR_INSTANCE_VALUE_0: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
@ -2722,11 +2729,48 @@
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_INSTANCE_VALUE_1: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
/* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */
case _CHECK_ATTR_MODULE_PUSH_KEYS: {
_PyStackRef owner;
PyDictKeysObject *mod_keys;
@ -2755,6 +2799,8 @@
PyDictKeysObject *mod_keys;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
owner = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
@ -2782,8 +2828,12 @@
attr = PyStackRef_FromPyObjectSteal(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2810,6 +2860,7 @@
PyDictObject *dict;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
dict = (PyDictObject *)stack_pointer[-1].bits;
owner = stack_pointer[-2];
@ -2832,7 +2883,7 @@
JUMP_TO_JUMP_TARGET();
}
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
UNLOCK_OBJECT(dict);
stack_pointer += -1;
@ -2865,16 +2916,20 @@
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-2] = attr;
stack_pointer += -1;
if (oparg & 1) stack_pointer[-1] = null;
stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_SLOT: {
case _LOAD_ATTR_SLOT_0: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
@ -2894,11 +2949,47 @@
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_SLOT_1: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **addr = (PyObject **)((char *)owner_o + index);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
if (!increfed) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
/* _LOAD_ATTR_SLOT is split on (oparg & 1) */
case _CHECK_ATTR_CLASS: {
_PyStackRef owner;
owner = stack_pointer[-1];
@ -2916,24 +3007,50 @@
break;
}
case _LOAD_ATTR_CLASS: {
case _LOAD_ATTR_CLASS_0: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_CLASS_1: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
/* _LOAD_ATTR_CLASS is split on (oparg & 1) */
case _LOAD_ATTR_PROPERTY_FRAME: {
_PyStackRef owner;
_PyInterpreterFrame *new_frame;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *fget = (PyObject *)CURRENT_OPERAND0();
assert((oparg & 1) == 0);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
PyCodeObject *code = (PyCodeObject *)f->func_code;
@ -3939,12 +4056,14 @@
break;
}
case _LOAD_METHOD_WITH_VALUES: {
case _LOAD_ATTR_METHOD_WITH_VALUES: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self;
_PyStackRef self = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
assert(oparg & 1);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -3958,12 +4077,14 @@
break;
}
case _LOAD_METHOD_NO_DICT: {
case _LOAD_ATTR_METHOD_NO_DICT: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self;
_PyStackRef self = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
assert(oparg & 1);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -3980,8 +4101,10 @@
case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: {
_PyStackRef owner;
_PyStackRef attr;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
PyStackRef_CLOSE(owner);
@ -3993,8 +4116,10 @@
case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: {
_PyStackRef owner;
_PyStackRef attr;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
assert((oparg & 1) == 0);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -4018,12 +4143,14 @@
break;
}
case _LOAD_METHOD_LAZY_DICT: {
case _LOAD_ATTR_METHOD_LAZY_DICT: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self;
_PyStackRef self = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
@ -5517,14 +5644,15 @@
/* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */
case _MAKE_CALLARGS_A_TUPLE: {
_PyStackRef kwargs_in;
_PyStackRef kwargs_in = PyStackRef_NULL;
_PyStackRef callargs;
_PyStackRef func;
_PyStackRef tuple;
_PyStackRef kwargs_out;
kwargs_in = stack_pointer[-1];
callargs = stack_pointer[-2];
func = stack_pointer[-4];
_PyStackRef kwargs_out = PyStackRef_NULL;
oparg = CURRENT_OPARG();
if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; }
callargs = stack_pointer[-1 - (oparg & 1)];
func = stack_pointer[-3 - (oparg & 1)];
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (PyTuple_CheckExact(callargs_o)) {
tuple = callargs;
@ -5547,8 +5675,8 @@
PyStackRef_CLOSE(callargs);
tuple = PyStackRef_FromPyObjectSteal(tuple_o);
}
stack_pointer[-2] = tuple;
stack_pointer[-1] = kwargs_out;
stack_pointer[-1 - (oparg & 1)] = tuple;
if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out;
break;
}
@ -5625,27 +5753,25 @@
}
case _BUILD_SLICE: {
_PyStackRef *args;
_PyStackRef step = PyStackRef_NULL;
_PyStackRef stop;
_PyStackRef start;
_PyStackRef slice;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
assert(oparg == 2 || oparg == 3);
_PyStackRef start = args[0];
_PyStackRef stop = args[1];
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
PyObject * step_o = NULL;
if (oparg == 3) {
step_o = PyStackRef_AsPyObjectBorrow(args[2]);
}
PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
PyStackRef_CLOSE(start);
PyStackRef_CLOSE(stop);
PyStackRef_XCLOSE(step);
if (slice_o == NULL) JUMP_TO_ERROR();
slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-oparg] = slice;
stack_pointer += 1 - oparg;
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -6012,6 +6138,8 @@
case _LOAD_GLOBAL_MODULE: {
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
@ -6022,14 +6150,18 @@
}
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
null = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_GLOBAL_BUILTINS: {
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictObject *dict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
@ -6040,8 +6172,10 @@
}
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
null = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -6049,6 +6183,8 @@
case _LOAD_ATTR_MODULE: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
@ -6064,8 +6200,12 @@
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}

6
Python/flowgraph.c
View file

@ -1836,6 +1836,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts)
INSTR_SET_OP0(inst, NOP);
}
break;
case LOAD_GLOBAL:
if (nextop == PUSH_NULL && (oparg & 1) == 0) {
INSTR_SET_OP1(inst, LOAD_GLOBAL, oparg | 1);
INSTR_SET_OP0(&bb->b_instr[i + 1], NOP);
}
break;
case COMPARE_OP:
if (nextop == TO_BOOL) {
INSTR_SET_OP0(inst, NOP);

633
Python/generated_cases.c.h generated
View file

@ -833,30 +833,28 @@
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SLICE);
_PyStackRef *args;
_PyStackRef start;
_PyStackRef stop;
_PyStackRef step = PyStackRef_NULL;
_PyStackRef slice;
args = &stack_pointer[-oparg];
assert(oparg == 2 || oparg == 3);
_PyStackRef start = args[0];
_PyStackRef stop = args[1];
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
PyObject * step_o = NULL;
if (oparg == 3) {
step_o = PyStackRef_AsPyObjectBorrow(args[2]);
}
PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
PyStackRef_CLOSE(start);
PyStackRef_CLOSE(stop);
PyStackRef_XCLOSE(step);
if (slice_o == NULL) {
stack_pointer += -oparg;
stack_pointer += -2 - ((oparg == 3) ? 1 : 0);
assert(WITHIN_STACK_BOUNDS());
goto error;
}
slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-oparg] = slice;
stack_pointer += 1 - oparg;
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -1708,18 +1706,18 @@
(void)this_instr;
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef kwargs_in;
_PyStackRef kwargs_in = PyStackRef_NULL;
_PyStackRef tuple;
_PyStackRef kwargs_out;
_PyStackRef kwargs_out = PyStackRef_NULL;
_PyStackRef func_st;
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef kwargs_st = PyStackRef_NULL;
_PyStackRef result;
// _MAKE_CALLARGS_A_TUPLE
{
kwargs_in = stack_pointer[-1];
callargs = stack_pointer[-2];
func = stack_pointer[-4];
if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; }
callargs = stack_pointer[-1 - (oparg & 1)];
func = stack_pointer[-3 - (oparg & 1)];
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (PyTuple_CheckExact(callargs_o)) {
tuple = callargs;
@ -1761,8 +1759,8 @@
assert(PyTuple_CheckExact(callargs));
PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING;
stack_pointer[-2] = callargs_st;
stack_pointer[-1] = kwargs_st;
stack_pointer[-1 - (oparg & 1)] = callargs_st;
if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st;
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
@ -1805,7 +1803,7 @@
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
stack_pointer += -3;
stack_pointer += -2 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(
@ -1826,8 +1824,8 @@
assert(PyTuple_CheckExact(callargs));
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
stack_pointer[-2] = callargs_st;
stack_pointer[-1] = kwargs_st;
stack_pointer[-1 - (oparg & 1)] = callargs_st;
if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st;
_PyFrame_SetStackPointer(frame, stack_pointer);
result_o = PyObject_Call(func, callargs, kwargs);
stack_pointer = _PyFrame_GetStackPointer(frame);
@ -1837,7 +1835,11 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(callargs_st);
PyStackRef_CLOSE(func_st);
if (result_o == NULL) goto pop_4_error;
if (result_o == NULL) {
stack_pointer += -3 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
goto error;
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
// _CHECK_PERIODIC
@ -1845,19 +1847,19 @@
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[-4] = result;
stack_pointer += -3;
stack_pointer[-3 - (oparg & 1)] = result;
stack_pointer += -2 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) goto error;
stack_pointer += 3;
stack_pointer += 2 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
}
}
stack_pointer[-4] = result;
stack_pointer += -3;
stack_pointer[-3 - (oparg & 1)] = result;
stack_pointer += -2 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -4846,18 +4848,6 @@
goto PREDICTED_LOAD_SUPER_ATTR;
}
TARGET(INSTRUMENTED_LOAD_SUPER_METHOD) {
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_METHOD);
/* Skip 1 cache entry */
// cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
// don't want to specialize instrumented instructions
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
goto PREDICTED_LOAD_SUPER_METHOD;
}
TARGET(INSTRUMENTED_NOT_TAKEN) {
_Py_CODEUNIT* const prev_instr = frame->instr_ptr;
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
@ -5311,6 +5301,7 @@
(void)this_instr;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self_or_null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_ATTR
{
owner = stack_pointer[-1];
@ -5318,7 +5309,7 @@
(void)counter;
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadAttr(owner, next_instr, name);
@ -5332,15 +5323,50 @@
/* Skip 8 cache entries */
// _LOAD_ATTR
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) goto pop_1_error;
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
PyObject *attr_o;
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
attr_o = NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (is_meth) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
PyStackRef_CLOSE(owner);
if (attr_o == NULL) goto pop_1_error;
self_or_null = PyStackRef_NULL;
}
}
else {
/* Classic, pushes one value. */
_PyFrame_SetStackPointer(frame, stack_pointer);
attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) goto pop_1_error;
/* We need to define self_or_null on all paths */
self_or_null = PyStackRef_NULL;
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -5351,6 +5377,7 @@
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_CLASS
{
@ -5368,9 +5395,13 @@
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -5381,6 +5412,7 @@
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_CLASS
{
@ -5404,9 +5436,13 @@
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -5422,6 +5458,7 @@
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *getattribute = read_obj(&this_instr[6].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
@ -5434,7 +5471,7 @@
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(
tstate, PyStackRef_FromPyObjectNew(f), 2, frame);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
@ -5452,6 +5489,7 @@
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
@ -5483,10 +5521,143 @@
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
/* Skip 1 cache entry */
// _LOAD_ATTR_METHOD_LAZY_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_NO_DICT) {
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyDictValues *ivs = _PyObject_InlineValues(owner_o);
DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -5498,6 +5669,7 @@
_PyStackRef owner;
PyDictKeysObject *mod_keys;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_MODULE_PUSH_KEYS
{
@ -5530,10 +5702,14 @@
attr = PyStackRef_FromPyObjectSteal(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -5557,6 +5733,7 @@
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -5601,6 +5778,7 @@
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
PyStackRef_CLOSE(owner);
@ -5634,6 +5812,7 @@
// _LOAD_ATTR_PROPERTY_FRAME
{
PyObject *fget = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
PyCodeObject *code = (PyCodeObject *)f->func_code;
@ -5681,6 +5860,7 @@
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
@ -5704,10 +5884,14 @@
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -5719,6 +5903,7 @@
_PyStackRef owner;
PyDictObject *dict;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
@ -5748,7 +5933,7 @@
UNLOCK_OBJECT(dict);
DEOPT_IF(true, LOAD_ATTR);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
UNLOCK_OBJECT(dict);
DEOPT_IF(true, LOAD_ATTR);
@ -5766,10 +5951,14 @@
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -6070,13 +6259,14 @@
_Py_CODEUNIT* const this_instr = next_instr - 5;
(void)this_instr;
_PyStackRef *res;
_PyStackRef null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_GLOBAL
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
@ -6093,13 +6283,15 @@
// _LOAD_GLOBAL
{
res = &stack_pointer[0];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(*res)) goto error;
null = PyStackRef_NULL;
}
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -6111,6 +6303,7 @@
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
PyDictKeysObject *builtins_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
@ -6145,9 +6338,11 @@
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
}
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -6159,6 +6354,7 @@
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
PyDictKeysObject *globals_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION_PUSH_KEYS
{
@ -6185,9 +6381,11 @@
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
}
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -6212,198 +6410,6 @@
DISPATCH();
}
TARGET(LOAD_METHOD) {
frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_METHOD);
PREDICTED_LOAD_METHOD:;
_Py_CODEUNIT* const this_instr = next_instr - 10;
(void)this_instr;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self_or_null;
// _SPECIALIZE_LOAD_METHOD
{
owner = stack_pointer[-1];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadMethod(owner, next_instr, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_METHOD);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
/* Skip 8 cache entries */
// _LOAD_METHOD
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *attr_o;
/* Designed to work in tandem with CALL, pushes two values. */
attr_o = NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (is_meth) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
PyStackRef_CLOSE(owner);
if (attr_o == NULL) goto pop_1_error;
self_or_null = PyStackRef_NULL;
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
stack_pointer[-1] = attr;
stack_pointer[0] = self_or_null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_METHOD_LAZY_DICT) {
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_METHOD_LAZY_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD);
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_METHOD);
}
/* Skip 1 cache entry */
// _LOAD_METHOD_LAZY_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_METHOD_NO_DICT) {
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_METHOD_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD);
}
/* Skip 2 cache entries */
// _LOAD_METHOD_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_METHOD_WITH_VALUES) {
_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_METHOD_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyDictValues *ivs = _PyObject_InlineValues(owner_o);
DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD);
}
// _LOAD_METHOD_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
@ -6483,6 +6489,7 @@
_PyStackRef class_st;
_PyStackRef self_st;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_SUPER_ATTR
{
class_st = stack_pointer[-2];
@ -6490,10 +6497,11 @@
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION_FT
int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0);
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH_SAME_OPARG();
}
@ -6507,7 +6515,7 @@
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
if (opcode >= MIN_INSTRUMENTED_OPCODE) {
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_2args(
@ -6527,7 +6535,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (opcode >= MIN_INSTRUMENTED_OPCODE) {
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
if (super == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -6560,9 +6568,11 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) goto error;
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
}
stack_pointer[0] = attr;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -6602,111 +6612,10 @@
DISPATCH();
}
TARGET(LOAD_SUPER_METHOD) {
TARGET(LOAD_SUPER_ATTR_METHOD) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_METHOD);
PREDICTED_LOAD_SUPER_METHOD:;
_Py_CODEUNIT* const this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef global_super_st;
_PyStackRef class_st;
_PyStackRef self_st;
_PyStackRef attr;
_PyStackRef null;
// _SPECIALIZE_LOAD_SUPER_METHOD
{
class_st = stack_pointer[-2];
global_super_st = stack_pointer[-3];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
// _LOAD_SUPER_ATTR
{
self_st = stack_pointer[-1];
PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
if (opcode >= MIN_INSTRUMENTED_OPCODE) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, global_super, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
goto pop_3_error;
}
}
// we make no attempt to optimize here; specializations should
// handle any case whose performance we care about
PyObject *stack[] = {class, self};
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (opcode >= MIN_INSTRUMENTED_OPCODE) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
if (super == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, global_super, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, global_super, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
Py_CLEAR(super);
}
}
}
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (super == NULL) goto pop_3_error;
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *attr_o = PyObject_GetAttr(super, name);
Py_DECREF(super);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) goto error;
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
// _PUSH_NULL
{
null = PyStackRef_NULL;
}
stack_pointer[0] = attr;
stack_pointer[1] = null;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
TARGET(LOAD_SUPER_METHOD_METHOD) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_METHOD_METHOD);
INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
_PyStackRef global_super_st;
_PyStackRef class_st;
@ -6721,8 +6630,8 @@
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(oparg & 1);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_METHOD);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
@ -7087,9 +6996,9 @@
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_NULL);
_PyStackRef null;
null = PyStackRef_NULL;
stack_pointer[0] = null;
_PyStackRef res;
res = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();

5
Python/instrumentation.c
View file

@ -81,8 +81,6 @@ static const int8_t EVENT_FOR_OPCODE[256] = {
[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
[LOAD_SUPER_METHOD] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_LOAD_SUPER_METHOD] = PY_MONITORING_EVENT_CALL,
[RESUME] = -1,
[YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
[INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
@ -128,7 +126,6 @@ static const uint8_t DE_INSTRUMENT[256] = {
[INSTRUMENTED_END_FOR] = END_FOR,
[INSTRUMENTED_END_SEND] = END_SEND,
[INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,
[INSTRUMENTED_LOAD_SUPER_METHOD] = LOAD_SUPER_METHOD,
[INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN,
};
@ -167,8 +164,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = {
[INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER,
[LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
[INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
[LOAD_SUPER_METHOD] = INSTRUMENTED_LOAD_SUPER_METHOD,
[INSTRUMENTED_LOAD_SUPER_METHOD] = INSTRUMENTED_LOAD_SUPER_METHOD,
[NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,
[INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,

18
Python/opcode_targets.h generated
View file

@ -3,7 +3,6 @@ static void *opcode_targets[256] = {
&&TARGET_BINARY_SLICE,
&&TARGET_BINARY_SUBSCR,
&&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
&&TARGET_CALL_FUNCTION_EX,
&&TARGET_CHECK_EG_MATCH,
&&TARGET_CHECK_EXC_MATCH,
&&TARGET_CLEANUP_THROW,
@ -16,8 +15,8 @@ static void *opcode_targets[256] = {
&&TARGET_FORMAT_WITH_SPEC,
&&TARGET_GET_AITER,
&&TARGET_GET_ANEXT,
&&TARGET_RESERVED,
&&TARGET_GET_ITER,
&&TARGET_RESERVED,
&&TARGET_GET_LEN,
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_INTERPRETER_EXIT,
@ -52,6 +51,7 @@ static void *opcode_targets[256] = {
&&TARGET_BUILD_STRING,
&&TARGET_BUILD_TUPLE,
&&TARGET_CALL,
&&TARGET_CALL_FUNCTION_EX,
&&TARGET_CALL_INTRINSIC_1,
&&TARGET_CALL_INTRINSIC_2,
&&TARGET_CALL_KW,
@ -89,12 +89,10 @@ static void *opcode_targets[256] = {
&&TARGET_LOAD_FROM_DICT_OR_DEREF,
&&TARGET_LOAD_FROM_DICT_OR_GLOBALS,
&&TARGET_LOAD_GLOBAL,
&&TARGET_LOAD_METHOD,
&&TARGET_LOAD_NAME,
&&TARGET_LOAD_SMALL_INT,
&&TARGET_LOAD_SPECIAL,
&&TARGET_LOAD_SUPER_ATTR,
&&TARGET_LOAD_SUPER_METHOD,
&&TARGET_MAKE_CELL,
&&TARGET_MAP_ADD,
&&TARGET_MATCH_CLASS,
@ -148,6 +146,8 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_RESUME,
&&TARGET_BINARY_OP_ADD_FLOAT,
&&TARGET_BINARY_OP_ADD_INT,
@ -198,6 +198,9 @@ static void *opcode_targets[256] = {
&&TARGET_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK,
&&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
&&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
&&TARGET_LOAD_ATTR_METHOD_NO_DICT,
&&TARGET_LOAD_ATTR_METHOD_WITH_VALUES,
&&TARGET_LOAD_ATTR_MODULE,
&&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT,
&&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES,
@ -208,11 +211,8 @@ static void *opcode_targets[256] = {
&&TARGET_LOAD_CONST_MORTAL,
&&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_METHOD_LAZY_DICT,
&&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_LOAD_METHOD_WITH_VALUES,
&&TARGET_LOAD_SUPER_ATTR_ATTR,
&&TARGET_LOAD_SUPER_METHOD_METHOD,
&&TARGET_LOAD_SUPER_ATTR_METHOD,
&&TARGET_RESUME_CHECK,
&&TARGET_SEND_GEN,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
@ -233,11 +233,11 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_INSTRUMENTED_END_FOR,
&&TARGET_INSTRUMENTED_POP_ITER,
&&TARGET_INSTRUMENTED_END_SEND,
&&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR,
&&TARGET_INSTRUMENTED_LOAD_SUPER_METHOD,
&&TARGET_INSTRUMENTED_FOR_ITER,
&&TARGET_INSTRUMENTED_CALL_KW,
&&TARGET_INSTRUMENTED_CALL_FUNCTION_EX,

5
Python/optimizer.c
View file

@ -1274,7 +1274,10 @@ uop_optimize(
for (int pc = 0; pc < length; pc++) {
int opcode = buffer[pc].opcode;
int oparg = buffer[pc].oparg;
if (oparg < _PyUop_Replication[opcode]) {
if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) {
buffer[pc].opcode = opcode + 1 + (oparg & 1);
}
else if (oparg < _PyUop_Replication[opcode]) {
buffer[pc].opcode = opcode + oparg + 1;
}
else if (is_terminator(&buffer[pc])) {

4
Python/optimizer_analysis.c
View file

@ -109,10 +109,10 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj)
return NULL;
}
if (_Py_IsImmortal(res)) {
inst->opcode = _LOAD_CONST_INLINE_BORROW;
inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW;
}
else {
inst->opcode = _LOAD_CONST_INLINE;
inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE;
}
inst->operand0 = (uint64_t)res;
return res;

30
Python/optimizer_bytecodes.c
View file

@ -528,8 +528,9 @@ dummy_func(void) {
top_out = top_in;
}
op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) {
op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) {
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)offset;
(void)owner;
}
@ -552,19 +553,15 @@ dummy_func(void) {
}
}
op(_LOAD_ATTR, (owner -- attr)) {
(void)owner;
attr = sym_new_not_null(ctx);
}
op(_LOAD_METHOD, (owner -- attr, self_or_null)) {
op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) {
(void)owner;
attr = sym_new_not_null(ctx);
self_or_null = sym_new_unknown(ctx);
}
op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr)) {
op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr, null if (oparg & 1))) {
(void)index;
null = sym_new_null(ctx);
attr = NULL;
if (this_instr[-1].opcode == _NOP) {
// Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched.
@ -592,38 +589,41 @@ dummy_func(void) {
(void)owner;
}
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr)) {
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) {
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
(void)dict;
}
op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) {
op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)index;
(void)owner;
}
op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) {
op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) {
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)descr;
(void)owner;
}
op(_LOAD_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) {
op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) {
(void)descr;
attr = sym_new_not_null(ctx);
self = owner;
}
op(_LOAD_METHOD_NO_DICT, (descr/4, owner -- attr, self)) {
op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) {
(void)descr;
attr = sym_new_not_null(ctx);
self = owner;
}
op(_LOAD_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) {
op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) {
(void)descr;
attr = sym_new_not_null(ctx);
self = owner;
@ -819,7 +819,7 @@ dummy_func(void) {
Py_UNREACHABLE();
}
op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- )) {
op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) {
SYNC_SP();
ctx->frame->stack_pointer = stack_pointer;
ctx->frame = new_frame;

104
Python/optimizer_cases.c.h generated
View file

@ -928,9 +928,12 @@
case _LOAD_GLOBAL: {
JitOptSymbol **res;
JitOptSymbol *null = NULL;
res = &stack_pointer[0];
res[0] = sym_new_not_null(ctx);
stack_pointer += 1;
null = sym_new_null(ctx);
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -963,15 +966,25 @@
case _LOAD_GLOBAL_MODULE_FROM_KEYS: {
JitOptSymbol *res;
JitOptSymbol *null = NULL;
res = sym_new_not_null(ctx);
null = sym_new_null(ctx);
stack_pointer[-1] = res;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: {
JitOptSymbol *res;
JitOptSymbol *null = NULL;
res = sym_new_not_null(ctx);
null = sym_new_null(ctx);
stack_pointer[-1] = res;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -1096,8 +1109,6 @@
/* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */
/* _INSTRUMENTED_LOAD_SUPER_METHOD is not a viable micro-op for tier 2 */
case _LOAD_SUPER_ATTR_ATTR: {
JitOptSymbol *attr_st;
attr_st = sym_new_not_null(ctx);
@ -1107,7 +1118,7 @@
break;
}
case _LOAD_SUPER_METHOD_METHOD: {
case _LOAD_SUPER_ATTR_METHOD: {
JitOptSymbol *attr;
JitOptSymbol *self_or_null;
attr = sym_new_not_null(ctx);
@ -1119,31 +1130,21 @@
break;
}
case _LOAD_METHOD: {
case _LOAD_ATTR: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *self_or_null;
JitOptSymbol *self_or_null = NULL;
owner = stack_pointer[-1];
(void)owner;
attr = sym_new_not_null(ctx);
self_or_null = sym_new_unknown(ctx);
stack_pointer[-1] = attr;
stack_pointer[0] = self_or_null;
stack_pointer += 1;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR: {
JitOptSymbol *owner;
JitOptSymbol *attr;
owner = stack_pointer[-1];
(void)owner;
attr = sym_new_not_null(ctx);
stack_pointer[-1] = attr;
break;
}
case _GUARD_TYPE_VERSION: {
JitOptSymbol *owner;
owner = stack_pointer[-1];
@ -1181,12 +1182,17 @@
case _LOAD_ATTR_INSTANCE_VALUE: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *null = NULL;
owner = stack_pointer[-1];
uint16_t offset = (uint16_t)this_instr->operand0;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)offset;
(void)owner;
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -1224,9 +1230,11 @@
case _LOAD_ATTR_MODULE_FROM_KEYS: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *null = NULL;
owner = stack_pointer[-2];
uint16_t index = (uint16_t)this_instr->operand0;
(void)index;
null = sym_new_null(ctx);
attr = NULL;
if (this_instr[-1].opcode == _NOP) {
// Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched.
@ -1235,7 +1243,8 @@
assert(PyModule_CheckExact(mod));
PyObject *dict = mod->md_dict;
stack_pointer[-2] = attr;
stack_pointer += -1;
if (oparg & 1) stack_pointer[-1] = null;
stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
PyObject *res = convert_global_to_const(this_instr, dict);
if (res != NULL) {
@ -1245,7 +1254,7 @@
else {
this_instr->opcode = _LOAD_ATTR_MODULE;
}
stack_pointer += 1;
stack_pointer += 1 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
}
if (attr == NULL) {
@ -1253,7 +1262,8 @@
attr = sym_new_not_null(ctx);
}
stack_pointer[-2] = attr;
stack_pointer += -1;
if (oparg & 1) stack_pointer[-1] = null;
stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -1274,15 +1284,18 @@
JitOptSymbol *dict;
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *null = NULL;
dict = stack_pointer[-1];
owner = stack_pointer[-2];
uint16_t hint = (uint16_t)this_instr->operand0;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)hint;
(void)owner;
(void)dict;
stack_pointer[-2] = attr;
stack_pointer += -1;
if (oparg & 1) stack_pointer[-1] = null;
stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -1290,12 +1303,17 @@
case _LOAD_ATTR_SLOT: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *null = NULL;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)this_instr->operand0;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)index;
(void)owner;
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -1306,12 +1324,17 @@
case _LOAD_ATTR_CLASS: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *null = NULL;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)this_instr->operand0;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
(void)descr;
(void)owner;
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -1697,10 +1720,10 @@
break;
}
case _LOAD_METHOD_WITH_VALUES: {
case _LOAD_ATTR_METHOD_WITH_VALUES: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *self;
JitOptSymbol *self = NULL;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)this_instr->operand0;
(void)descr;
@ -1713,10 +1736,10 @@
break;
}
case _LOAD_METHOD_NO_DICT: {
case _LOAD_ATTR_METHOD_NO_DICT: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *self;
JitOptSymbol *self = NULL;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)this_instr->operand0;
(void)descr;
@ -1747,10 +1770,10 @@
break;
}
case _LOAD_METHOD_LAZY_DICT: {
case _LOAD_ATTR_METHOD_LAZY_DICT: {
JitOptSymbol *owner;
JitOptSymbol *attr;
JitOptSymbol *self;
JitOptSymbol *self = NULL;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)this_instr->operand0;
(void)descr;
@ -2239,11 +2262,11 @@
case _MAKE_CALLARGS_A_TUPLE: {
JitOptSymbol *tuple;
JitOptSymbol *kwargs_out;
JitOptSymbol *kwargs_out = NULL;
tuple = sym_new_not_null(ctx);
kwargs_out = sym_new_not_null(ctx);
stack_pointer[-2] = tuple;
stack_pointer[-1] = kwargs_out;
stack_pointer[-1 - (oparg & 1)] = tuple;
if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out;
break;
}
@ -2292,8 +2315,8 @@
case _BUILD_SLICE: {
JitOptSymbol *slice;
slice = sym_new_not_null(ctx);
stack_pointer[-oparg] = slice;
stack_pointer += 1 - oparg;
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2616,26 +2639,37 @@
case _LOAD_GLOBAL_MODULE: {
JitOptSymbol *res;
JitOptSymbol *null = NULL;
res = sym_new_not_null(ctx);
null = sym_new_null(ctx);
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_GLOBAL_BUILTINS: {
JitOptSymbol *res;
JitOptSymbol *null = NULL;
res = sym_new_not_null(ctx);
null = sym_new_null(ctx);
stack_pointer[0] = res;
stack_pointer += 1;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_MODULE: {
JitOptSymbol *attr;
JitOptSymbol *null = NULL;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}

110
Python/specialize.c
View file

@ -804,7 +804,7 @@ _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _P
SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS);
goto fail;
}
uint8_t load_code = load_method ? LOAD_SUPER_METHOD_METHOD : LOAD_SUPER_ATTR_ATTR;
uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR;
specialize(instr, load_code);
return;
fail:
@ -1109,7 +1109,7 @@ instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version)
static int
do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
bool shadow, uint32_t shared_keys_version,
DescriptorClassification kind, PyObject *descr, unsigned int tp_version, bool load_method)
DescriptorClassification kind, PyObject *descr, unsigned int tp_version)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
@ -1117,16 +1117,17 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg);
switch(kind) {
case OVERRIDING:
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
return -1;
case METHOD:
{
if (shadow) {
goto try_instance;
}
if (load_method) {
if (oparg & 1) {
if (specialize_attr_loadclassattr(owner, instr, name, descr,
tp_version, kind, true,
shared_keys_version)) {
@ -1136,7 +1137,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
return -1;
}
}
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
case PROPERTY:
@ -1145,28 +1146,28 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
assert(Py_TYPE(descr) == &PyProperty_Type);
PyObject *fget = ((_PyPropertyObject *)descr)->prop_get;
if (fget == NULL) {
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
return -1;
}
if (!Py_IS_TYPE(fget, &PyFunction_Type)) {
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION);
return -1;
}
if (load_method) {
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_ATTR_METHOD);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION);
return -1;
}
if (!function_check_args(fget, 1, LOAD_ATTR)) {
return -1;
}
if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
/* Don't specialize if PEP 523 is active */
if (_PyInterpreterState_GET()->eval_frame) {
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OTHER);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_HasDeferredRefcount(fget)) {
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
return -1;
}
#endif
@ -1179,10 +1180,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
}
case OBJECT_SLOT:
{
if (load_method) {
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER);
return -1;
}
PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
struct PyMemberDef *dmem = member->d_member;
Py_ssize_t offset = dmem->offset;
@ -1207,10 +1204,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
}
case DUNDER_CLASS:
{
if (load_method) {
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER);
return -1;
}
Py_ssize_t offset = offsetof(PyObject, ob_type);
assert(offset == (uint16_t)offset);
cache->index = (uint16_t)offset;
@ -1219,20 +1212,16 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
return 0;
}
case OTHER_SLOT:
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
return -1;
case MUTABLE:
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
return -1;
case GETSET_OVERRIDDEN:
SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
return -1;
case GETATTRIBUTE_IS_PYTHON_FUNCTION:
{
if (load_method) {
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER);
return -1;
}
#ifndef Py_GIL_DISABLED
// In free-threaded builds it's possible for tp_getattro to change
// after the call to analyze_descriptor. That is fine: the version
@ -1244,6 +1233,10 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
if (!function_check_args(descr, 2, LOAD_ATTR)) {
return -1;
}
if (oparg & 1) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
return -1;
}
uint32_t version = function_get_version(descr, LOAD_ATTR);
if (version == 0) {
return -1;
@ -1277,7 +1270,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
if (shadow) {
goto try_instance;
}
if (!load_method) {
if ((oparg & 1) == 0) {
if (specialize_attr_loadclassattr(owner, instr, name, descr,
tp_version, kind, false,
shared_keys_version)) {
@ -1294,10 +1287,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
}
Py_UNREACHABLE();
try_instance:
if (load_method) {
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER);
return -1;
}
if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
{
@ -1307,7 +1296,7 @@ try_instance:
}
static int
specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, bool load_method)
specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name)
{
// 0 is not a valid version
uint32_t shared_keys_version = 0;
@ -1316,7 +1305,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
unsigned int tp_version = 0;
PyTypeObject *type = Py_TYPE(owner);
DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version);
int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version, load_method);
int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version);
Py_XDECREF(descr);
return result;
}
@ -1344,40 +1333,7 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam
fail = specialize_class_load_attr(owner, instr, name);
}
else {
fail = specialize_instance_load_attr(owner, instr, name, false);
}
if (fail) {
unspecialize(instr);
}
}
void
_Py_Specialize_LoadMethod(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
{
PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
PyTypeObject *type = Py_TYPE(owner);
bool fail;
if (!_PyType_IsReady(type)) {
// We *might* not really need this check, but we inherited it from
// PyObject_GenericGetAttr and friends... and this way we still do the
// right thing if someone forgets to call PyType_Ready(type):
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER);
fail = true;
}
else if (Py_TYPE(owner)->tp_getattro == PyModule_Type.tp_getattro) {
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER);
fail = true;
}
else if (PyType_Check(owner)) {
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER);
fail = true;
}
else {
fail = specialize_instance_load_attr(owner, instr, name, true);
fail = specialize_instance_load_attr(owner, instr, name);
}
if (fail) {
@ -1619,7 +1575,7 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
#ifdef Py_GIL_DISABLED
if (!_PyObject_HasDeferredRefcount(descr)) {
SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED);
return 0;
}
#endif
@ -1631,11 +1587,11 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0);
#endif
if (shared_keys_version == 0) {
SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return 0;
}
write_u32(cache->keys_version, shared_keys_version);
specialize(instr, is_method ? LOAD_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
}
else {
Py_ssize_t dictoffset;
@ -1645,17 +1601,17 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
else {
dictoffset = owner_cls->tp_dictoffset;
if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) {
SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
}
if (dictoffset == 0) {
specialize(instr, is_method ? LOAD_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
specialize(instr, is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
}
else if (is_method) {
PyObject *dict = *(PyObject **) ((char *)owner + dictoffset);
if (dict) {
SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
/* Cache entries must be unsigned values, so we offset the
@ -1664,10 +1620,10 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
dictoffset -= MANAGED_DICT_OFFSET;
assert(((uint16_t)dictoffset) == dictoffset);
cache->dict_offset = (uint16_t)dictoffset;
specialize(instr, LOAD_METHOD_LAZY_DICT);
specialize(instr, LOAD_ATTR_METHOD_LAZY_DICT);
}
else {
SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
return 0;
}
}