bpo-44337: Port LOAD_ATTR to PEP 659 adaptive interpreter (GH-26595)

* Specialize LOAD_ATTR with  LOAD_ATTR_SLOT and LOAD_ATTR_SPLIT_KEYS

* Move dict-common.h to internal/pycore_dict.h

* Add LOAD_ATTR_WITH_HINT specialized opcode.

* Quicken in function if loopy

* Specialize LOAD_ATTR for module attributes.

* Add specialization stats

Include/internal/pycore_code.h

@@ -42,6 +42,12 @@ typedef struct {
     uint16_t index;
 } _PyAdaptiveEntry;
 
+
+typedef struct {
+    uint32_t tp_version;
+    uint32_t dk_version_or_hint;
+} _PyLoadAttrCache;
+
 /* Add specialized versions of entries to this union.
  *
  * Do not break the invariant: sizeof(SpecializedCacheEntry) == 8
@@ -55,6 +61,7 @@ typedef struct {
 typedef union {
     _PyEntryZero zero;
     _PyAdaptiveEntry adaptive;
+    _PyLoadAttrCache load_attr;
 } SpecializedCacheEntry;
 
 #define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT))
@@ -255,6 +262,83 @@ PyAPI_FUNC(PyObject *) _PyCode_GetCellvars(PyCodeObject *);
 PyAPI_FUNC(PyObject *) _PyCode_GetFreevars(PyCodeObject *);
 
 
+/* Cache hits and misses */
+
+static inline uint8_t
+saturating_increment(uint8_t c)
+{
+    return c<<1;
+}
+
+static inline uint8_t
+saturating_decrement(uint8_t c)
+{
+    return (c>>1) + 128;
+}
+
+static inline uint8_t
+saturating_zero(void)
+{
+    return 255;
+}
+
+/* Starting value for saturating counter.
+ * Technically this should be 1, but that is likely to
+ * cause a bit of thrashing when we optimize then get an immediate miss.
+ * We want to give the counter a change to stabilize, so we start at 3.
+ */
+static inline uint8_t
+saturating_start(void)
+{
+    return saturating_zero()<<3;
+}
+
+static inline void
+record_cache_hit(_PyAdaptiveEntry *entry) {
+    entry->counter = saturating_increment(entry->counter);
+}
+
+static inline void
+record_cache_miss(_PyAdaptiveEntry *entry) {
+    entry->counter = saturating_decrement(entry->counter);
+}
+
+static inline int
+too_many_cache_misses(_PyAdaptiveEntry *entry) {
+    return entry->counter == saturating_zero();
+}
+
+#define BACKOFF 64
+
+static inline void
+cache_backoff(_PyAdaptiveEntry *entry) {
+    entry->counter = BACKOFF;
+}
+
+/* Specialization functions */
+
+int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
+
+#define SPECIALIZATION_STATS 0
+#if SPECIALIZATION_STATS
+
+typedef struct _specialization_stats {
+    uint64_t specialization_success;
+    uint64_t specialization_failure;
+    uint64_t loadattr_hit;
+    uint64_t loadattr_deferred;
+    uint64_t loadattr_miss;
+    uint64_t loadattr_deopt;
+} SpecializationStats;
+
+extern SpecializationStats _specialization_stats;
+#define STAT_INC(name) _specialization_stats.name++
+void _Py_PrintSpecializationStats(void);
+#else
+#define STAT_INC(name) ((void)0)
+#endif
+
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_dict.h

@@ -0,0 +1,96 @@
+
+#ifndef Py_INTERNAL_DICT_H
+#define Py_INTERNAL_DICT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+
+typedef struct {
+    /* Cached hash code of me_key. */
+    Py_hash_t me_hash;
+    PyObject *me_key;
+    PyObject *me_value; /* This field is only meaningful for combined tables */
+} PyDictKeyEntry;
+
+/* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index].
+ * -1 when no entry found, -3 when compare raises error.
+ */
+Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr);
+
+
+#define DKIX_EMPTY (-1)
+#define DKIX_DUMMY (-2)  /* Used internally */
+#define DKIX_ERROR (-3)
+
+typedef enum {
+    DICT_KEYS_GENERAL = 0,
+    DICT_KEYS_UNICODE = 1,
+    DICT_KEYS_SPLIT = 2
+} DictKeysKind;
+
+/* See dictobject.c for actual layout of DictKeysObject */
+struct _dictkeysobject {
+    Py_ssize_t dk_refcnt;
+
+    /* Size of the hash table (dk_indices). It must be a power of 2. */
+    uint8_t dk_log2_size;
+
+    /* Kind of keys */
+    uint8_t dk_kind;
+
+    /* Version number -- Reset to 0 by any modification to keys */
+    uint32_t dk_version;
+
+    /* Number of usable entries in dk_entries. */
+    Py_ssize_t dk_usable;
+
+    /* Number of used entries in dk_entries. */
+    Py_ssize_t dk_nentries;
+
+    /* Actual hash table of dk_size entries. It holds indices in dk_entries,
+       or DKIX_EMPTY(-1) or DKIX_DUMMY(-2).
+
+       Indices must be: 0 <= indice < USABLE_FRACTION(dk_size).
+
+       The size in bytes of an indice depends on dk_size:
+
+       - 1 byte if dk_size <= 0xff (char*)
+       - 2 bytes if dk_size <= 0xffff (int16_t*)
+       - 4 bytes if dk_size <= 0xffffffff (int32_t*)
+       - 8 bytes otherwise (int64_t*)
+
+       Dynamically sized, SIZEOF_VOID_P is minimum. */
+    char dk_indices[];  /* char is required to avoid strict aliasing. */
+
+    /* "PyDictKeyEntry dk_entries[dk_usable];" array follows:
+       see the DK_ENTRIES() macro */
+};
+
+#define DK_LOG_SIZE(dk)  ((dk)->dk_log2_size)
+#if SIZEOF_VOID_P > 4
+#define DK_SIZE(dk)      (((int64_t)1)<<DK_LOG_SIZE(dk))
+#define DK_IXSIZE(dk)                     \
+    (DK_LOG_SIZE(dk) <= 7 ?               \
+        1 : DK_LOG_SIZE(dk) <= 15 ?       \
+            2 : DK_LOG_SIZE(dk) <= 31 ?   \
+                4 : sizeof(int64_t))
+#else
+#define DK_SIZE(dk)      (1<<DK_LOG_SIZE(dk))
+#define DK_IXSIZE(dk)                     \
+    (DK_LOG_SIZE(dk) <= 7 ?               \
+        1 : DK_LOG_SIZE(dk) <= 15 ?       \
+            2 : sizeof(int32_t))
+#endif
+#define DK_ENTRIES(dk) \
+    ((PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)]))
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif   /* !Py_INTERNAL_DICT_H */

Include/opcode.h

@@ -136,6 +136,12 @@ extern "C" {
 #define DICT_MERGE              164
 #define DICT_UPDATE             165
 #define CALL_METHOD_KW          166
+#define JUMP_ABSOLUTE_QUICK       7
+#define LOAD_ATTR_ADAPTIVE        8
+#define LOAD_ATTR_SPLIT_KEYS     13
+#define LOAD_ATTR_WITH_HINT      14
+#define LOAD_ATTR_SLOT           18
+#define LOAD_ATTR_MODULE         21
 #ifdef NEED_OPCODE_JUMP_TABLES
 static uint32_t _PyOpcode_RelativeJump[8] = {
     0U,