gh-100239: specialize long tail of binary operations (#128722)

This commit is contained in:
Irit Katriel 2025-01-16 15:22:13 +00:00 committed by GitHub
parent e81fe940c9
commit 3893a92d95
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 796 additions and 492 deletions

View file

@ -1174,7 +1174,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
assert(tp_version != 0);
write_u32(lm_cache->type_version, tp_version);
/* borrowed */
write_obj(lm_cache->descr, fget);
write_ptr(lm_cache->descr, fget);
specialize(instr, LOAD_ATTR_PROPERTY);
return 0;
}
@ -1254,7 +1254,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject*
#endif
write_u32(lm_cache->keys_version, version);
/* borrowed */
write_obj(lm_cache->descr, descr);
write_ptr(lm_cache->descr, descr);
write_u32(lm_cache->type_version, tp_version);
specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
return 0;
@ -1534,7 +1534,7 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
}
#endif
write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
write_ptr(cache->descr, descr);
if (metaclass_check) {
write_u32(cache->keys_version, meta_version);
specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
@ -1642,7 +1642,7 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr,
* working since Python 2.6 and it's battle-tested.
*/
write_u32(cache->type_version, tp_version);
write_obj(cache->descr, descr);
write_ptr(cache->descr, descr);
return 1;
}
@ -2412,6 +2412,92 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
}
#endif
/** Binary Op Specialization Extensions */
/* float-long */
static int
float_compactlong_guard(PyObject *lhs, PyObject *rhs)
{
return (
PyFloat_CheckExact(lhs) &&
PyLong_CheckExact(rhs) &&
_PyLong_IsCompact((PyLongObject *)rhs)
);
}
#define FLOAT_LONG_ACTION(NAME, OP) \
static PyObject * \
(NAME)(PyObject *lhs, PyObject *rhs) \
{ \
double lhs_val = PyFloat_AsDouble(lhs); \
Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \
return PyFloat_FromDouble(lhs_val OP rhs_val); \
}
FLOAT_LONG_ACTION(float_compactlong_add, +)
FLOAT_LONG_ACTION(float_compactlong_subtract, -)
FLOAT_LONG_ACTION(float_compactlong_multiply, *)
FLOAT_LONG_ACTION(float_compactlong_true_div, /)
#undef FLOAT_LONG_ACTION
/* long-float */
static int
compactlong_float_guard(PyObject *lhs, PyObject *rhs)
{
return (
PyFloat_CheckExact(rhs) &&
PyLong_CheckExact(lhs) &&
_PyLong_IsCompact((PyLongObject *)lhs)
);
}
#define LONG_FLOAT_ACTION(NAME, OP) \
static PyObject * \
(NAME)(PyObject *lhs, PyObject *rhs) \
{ \
double rhs_val = PyFloat_AsDouble(rhs); \
Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \
return PyFloat_FromDouble(lhs_val OP rhs_val); \
}
LONG_FLOAT_ACTION(compactlong_float_add, +)
LONG_FLOAT_ACTION(compactlong_float_subtract, -)
LONG_FLOAT_ACTION(compactlong_float_multiply, *)
LONG_FLOAT_ACTION(compactlong_float_true_div, /)
#undef LONG_FLOAT_ACTION
static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = {
[NB_ADD] = {float_compactlong_guard, float_compactlong_add},
[NB_SUBTRACT] = {float_compactlong_guard, float_compactlong_subtract},
[NB_TRUE_DIVIDE] = {float_compactlong_guard, float_compactlong_true_div},
[NB_MULTIPLY] = {float_compactlong_guard, float_compactlong_multiply},
};
static _PyBinaryOpSpecializationDescr compactlong_float_specs[NB_OPARG_LAST+1] = {
[NB_ADD] = {compactlong_float_guard, compactlong_float_add},
[NB_SUBTRACT] = {compactlong_float_guard, compactlong_float_subtract},
[NB_TRUE_DIVIDE] = {compactlong_float_guard, compactlong_float_true_div},
[NB_MULTIPLY] = {compactlong_float_guard, compactlong_float_multiply},
};
static int
binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg,
_PyBinaryOpSpecializationDescr **descr)
{
#define LOOKUP_SPEC(TABLE, OPARG) \
if ((TABLE)[(OPARG)].action) { \
if ((TABLE)[(OPARG)].guard(lhs, rhs)) { \
*descr = &((TABLE)[OPARG]); \
return 1; \
} \
}
LOOKUP_SPEC(compactlong_float_specs, oparg);
LOOKUP_SPEC(float_compactlong_specs, oparg);
#undef LOOKUP_SPEC
return 0;
}
void
_Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr,
int oparg, _PyStackRef *locals)
@ -2420,6 +2506,12 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in
PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
if (instr->op.code == BINARY_OP_EXTEND) {
write_ptr(cache->external_cache, NULL);
}
switch (oparg) {
case NB_ADD:
case NB_INPLACE_ADD:
@ -2474,8 +2566,17 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in
}
break;
}
_PyBinaryOpSpecializationDescr *descr;
if (binary_op_extended_specialization(lhs, rhs, oparg, &descr)) {
specialize(instr, BINARY_OP_EXTEND);
write_ptr(cache->external_cache, (void*)descr);
return;
}
SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
unspecialize(instr);
return;
}