mirror of
https://github.com/python/cpython.git
synced 2025-11-02 03:01:58 +00:00
GH-115776: Embed the values array into the object, for "normal" Python objects. (GH-116115)
This commit is contained in:
Mark Shannon
GitHub
parent
c97d3af239
commit
c32dc47aca
35 changed files with 787 additions and 537 deletions
|
|
@ -77,12 +77,11 @@ typedef struct _object_stats {
|
|||
uint64_t frees;
|
||||
uint64_t to_freelist;
|
||||
uint64_t from_freelist;
|
||||
uint64_t new_values;
|
||||
uint64_t inline_values;
|
||||
uint64_t dict_materialized_on_request;
|
||||
uint64_t dict_materialized_new_key;
|
||||
uint64_t dict_materialized_too_big;
|
||||
uint64_t dict_materialized_str_subclass;
|
||||
uint64_t dict_dematerialized;
|
||||
uint64_t type_cache_hits;
|
||||
uint64_t type_cache_misses;
|
||||
uint64_t type_cache_dunder_hits;
|
||||
|
|
|
|||
|
|
@ -79,7 +79,10 @@ typedef struct {
|
|||
typedef struct {
|
||||
uint16_t counter;
|
||||
uint16_t type_version[2];
|
||||
uint16_t keys_version[2];
|
||||
union {
|
||||
uint16_t keys_version[2];
|
||||
uint16_t dict_offset;
|
||||
};
|
||||
uint16_t descr[4];
|
||||
} _PyLoadMethodCache;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ extern "C" {
|
|||
|
||||
#include "pycore_freelist.h" // _PyFreeListState
|
||||
#include "pycore_identifier.h" // _Py_Identifier
|
||||
#include "pycore_object.h" // PyDictOrValues
|
||||
#include "pycore_object.h" // PyManagedDictPointer
|
||||
|
||||
// Unsafe flavor of PyDict_GetItemWithError(): no error checking
|
||||
extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
|
||||
|
|
@ -181,6 +181,10 @@ struct _dictkeysobject {
|
|||
* [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
|
||||
*/
|
||||
struct _dictvalues {
|
||||
uint8_t capacity;
|
||||
uint8_t size;
|
||||
uint8_t embedded;
|
||||
uint8_t valid;
|
||||
PyObject *values[1];
|
||||
};
|
||||
|
||||
|
|
@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
|
|||
size_t index = (size_t)1 << dk->dk_log2_index_bytes;
|
||||
return (&indices[index]);
|
||||
}
|
||||
|
||||
static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
|
||||
assert(dk->dk_kind == DICT_KEYS_GENERAL);
|
||||
return (PyDictKeyEntry*)_DK_ENTRIES(dk);
|
||||
|
|
@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
|
|||
#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
|
||||
#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
|
||||
|
||||
#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
|
||||
#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
|
||||
|
||||
#ifdef Py_GIL_DISABLED
|
||||
#define DICT_NEXT_VERSION(INTERP) \
|
||||
(_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
|
||||
|
|
@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
|
|||
return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
|
||||
}
|
||||
|
||||
extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
|
||||
PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
|
||||
extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
|
||||
|
||||
PyAPI_FUNC(PyObject *)_PyDict_FromItems(
|
||||
PyObject *const *keys, Py_ssize_t keys_offset,
|
||||
PyObject *const *values, Py_ssize_t values_offset,
|
||||
Py_ssize_t length);
|
||||
|
||||
static inline uint8_t *
|
||||
get_insertion_order_array(PyDictValues *values)
|
||||
{
|
||||
return (uint8_t *)&values->values[values->capacity];
|
||||
}
|
||||
|
||||
static inline void
|
||||
_PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
|
||||
{
|
||||
assert(ix < SHARED_KEYS_MAX_SIZE);
|
||||
uint8_t *size_ptr = ((uint8_t *)values)-2;
|
||||
int size = *size_ptr;
|
||||
assert(size+2 < DICT_VALUES_SIZE(values));
|
||||
size++;
|
||||
size_ptr[-size] = (uint8_t)ix;
|
||||
*size_ptr = size;
|
||||
int size = values->size;
|
||||
uint8_t *array = get_insertion_order_array(values);
|
||||
assert(size < values->capacity);
|
||||
assert(((uint8_t)ix) == ix);
|
||||
array[size] = (uint8_t)ix;
|
||||
values->size = size+1;
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
shared_keys_usable_size(PyDictKeysObject *keys)
|
||||
{
|
||||
#ifdef Py_GIL_DISABLED
|
||||
// dk_usable will decrease for each instance that is created and each
|
||||
// value that is added. dk_nentries will increase for each value that
|
||||
// is added. We want to always return the right value or larger.
|
||||
// We therefore increase dk_nentries first and we decrease dk_usable
|
||||
// second, and conversely here we read dk_usable first and dk_entries
|
||||
// second (to avoid the case where we read entries before the increment
|
||||
// and read usable after the decrement)
|
||||
return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
|
||||
_Py_atomic_load_ssize_acquire(&keys->dk_nentries));
|
||||
#else
|
||||
return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
_PyInlineValuesSize(PyTypeObject *tp)
|
||||
{
|
||||
PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
|
||||
assert(keys != NULL);
|
||||
size_t size = shared_keys_usable_size(keys);
|
||||
size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
|
||||
assert(prefix_size < 256);
|
||||
return prefix_size + (size + 1) * sizeof(PyObject *);
|
||||
}
|
||||
|
||||
int
|
||||
_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
|
|||
{
|
||||
assert(op != NULL);
|
||||
Py_SET_TYPE(op, typeobj);
|
||||
if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) {
|
||||
Py_INCREF(typeobj);
|
||||
}
|
||||
assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
|
||||
Py_INCREF(typeobj);
|
||||
_Py_NewReference(op);
|
||||
}
|
||||
|
||||
|
|
@ -611,8 +610,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
|
|||
extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
|
||||
extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
|
||||
|
||||
extern int _PyObject_InitializeDict(PyObject *obj);
|
||||
int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
|
||||
void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
|
||||
extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
|
||||
PyObject *name, PyObject *value);
|
||||
PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
|
||||
|
|
@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
|
|||
#endif
|
||||
|
||||
typedef union {
|
||||
PyObject *dict;
|
||||
/* Use a char* to generate a warning if directly assigning a PyDictValues */
|
||||
char *values;
|
||||
} PyDictOrValues;
|
||||
PyDictObject *dict;
|
||||
} PyManagedDictPointer;
|
||||
|
||||
static inline PyDictOrValues *
|
||||
_PyObject_DictOrValuesPointer(PyObject *obj)
|
||||
static inline PyManagedDictPointer *
|
||||
_PyObject_ManagedDictPointer(PyObject *obj)
|
||||
{
|
||||
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
|
||||
return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET);
|
||||
}
|
||||
|
||||
static inline int
|
||||
_PyDictOrValues_IsValues(PyDictOrValues dorv)
|
||||
{
|
||||
return ((uintptr_t)dorv.values) & 1;
|
||||
return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
|
||||
}
|
||||
|
||||
static inline PyDictValues *
|
||||
_PyDictOrValues_GetValues(PyDictOrValues dorv)
|
||||
_PyObject_InlineValues(PyObject *obj)
|
||||
{
|
||||
assert(_PyDictOrValues_IsValues(dorv));
|
||||
return (PyDictValues *)(dorv.values + 1);
|
||||
}
|
||||
|
||||
static inline PyObject *
|
||||
_PyDictOrValues_GetDict(PyDictOrValues dorv)
|
||||
{
|
||||
assert(!_PyDictOrValues_IsValues(dorv));
|
||||
return dorv.dict;
|
||||
}
|
||||
|
||||
static inline void
|
||||
_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values)
|
||||
{
|
||||
ptr->values = ((char *)values) - 1;
|
||||
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
|
||||
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
|
||||
assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject));
|
||||
return (PyDictValues *)((char *)obj + sizeof(PyObject));
|
||||
}
|
||||
|
||||
extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
|
||||
extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
|
||||
extern int _PyObject_IsInstanceDictEmpty(PyObject *);
|
||||
|
||||
// Export for 'math' shared extension
|
||||
|
|
|
|||
6
Include/internal/pycore_opcode_metadata.h
generated
6
Include/internal/pycore_opcode_metadata.h
generated
|
|
@ -1085,7 +1085,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
|
|||
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
|
||||
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
|
||||
[LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
|
||||
[LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
|
||||
[LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
|
||||
[LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
|
||||
[LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG },
|
||||
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
|
||||
|
|
@ -1269,7 +1269,7 @@ _PyOpcode_macro_expansion[256] = {
|
|||
[LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } },
|
||||
[LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } },
|
||||
[LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } },
|
||||
[LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
|
||||
[LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
|
||||
[LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } },
|
||||
[LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } },
|
||||
[LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } },
|
||||
|
|
@ -1316,7 +1316,7 @@ _PyOpcode_macro_expansion[256] = {
|
|||
[SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } },
|
||||
[SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } },
|
||||
[STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } },
|
||||
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
|
||||
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
|
||||
[STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } },
|
||||
[STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } },
|
||||
[STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } },
|
||||
|
|
|
|||
2
Include/internal/pycore_uop_ids.h
generated
2
Include/internal/pycore_uop_ids.h
generated
|
|
@ -109,7 +109,7 @@ extern "C" {
|
|||
#define _GUARD_BOTH_INT 347
|
||||
#define _GUARD_BOTH_UNICODE 348
|
||||
#define _GUARD_BUILTINS_VERSION 349
|
||||
#define _GUARD_DORV_VALUES 350
|
||||
#define _GUARD_DORV_NO_DICT 350
|
||||
#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351
|
||||
#define _GUARD_GLOBALS_VERSION 352
|
||||
#define _GUARD_IS_FALSE_POP 353
|
||||
|
|
|
|||
10
Include/internal/pycore_uop_metadata.h
generated
10
Include/internal/pycore_uop_metadata.h
generated
|
|
@ -136,8 +136,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
|
|||
[_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG,
|
||||
[_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
|
||||
[_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
|
||||
[_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG,
|
||||
[_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
|
||||
[_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
|
||||
[_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG,
|
||||
[_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG,
|
||||
[_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG,
|
||||
[_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG,
|
||||
|
|
@ -145,7 +145,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
|
|||
[_LOAD_ATTR_CLASS_0] = 0,
|
||||
[_LOAD_ATTR_CLASS_1] = 0,
|
||||
[_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG,
|
||||
[_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
|
||||
[_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
|
||||
[_STORE_ATTR_INSTANCE_VALUE] = 0,
|
||||
[_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG,
|
||||
[_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
|
||||
|
|
@ -342,7 +342,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
|
|||
[_GUARD_BOTH_INT] = "_GUARD_BOTH_INT",
|
||||
[_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE",
|
||||
[_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION",
|
||||
[_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES",
|
||||
[_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT",
|
||||
[_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT",
|
||||
[_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION",
|
||||
[_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP",
|
||||
|
|
@ -736,7 +736,7 @@ int _PyUop_num_popped(int opcode, int oparg)
|
|||
return 1;
|
||||
case _LOAD_ATTR_CLASS:
|
||||
return 1;
|
||||
case _GUARD_DORV_VALUES:
|
||||
case _GUARD_DORV_NO_DICT:
|
||||
return 1;
|
||||
case _STORE_ATTR_INSTANCE_VALUE:
|
||||
return 2;
|
||||
|
|
|
|||
|
|
@ -629,13 +629,18 @@ given type object has a specified feature.
|
|||
/* Track types initialized using _PyStaticType_InitBuiltin(). */
|
||||
#define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1)
|
||||
|
||||
/* The values array is placed inline directly after the rest of
|
||||
* the object. Implies Py_TPFLAGS_HAVE_GC.
|
||||
*/
|
||||
#define Py_TPFLAGS_INLINE_VALUES (1 << 2)
|
||||
|
||||
/* Placement of weakref pointers are managed by the VM, not by the type.
|
||||
* The VM will automatically set tp_weaklistoffset.
|
||||
*/
|
||||
#define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3)
|
||||
|
||||
/* Placement of dict (and values) pointers are managed by the VM, not by the type.
|
||||
* The VM will automatically set tp_dictoffset.
|
||||
* The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC.
|
||||
*/
|
||||
#define Py_TPFLAGS_MANAGED_DICT (1 << 4)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue