gh-132261: Store annotations at hidden internal keys in the class dict (#132345)

Doc/library/annotationlib.rst

@@ -303,12 +303,12 @@ Functions
 .. function:: get_annotate_function(obj)
 
    Retrieve the :term:`annotate function` for *obj*. Return :const:`!None`
-   if *obj* does not have an annotate function.
+   if *obj* does not have an annotate function. *obj* may be a class, function,
+   module, or a namespace dictionary for a class. The last case is useful during
+   class creation, e.g. in the ``__new__`` method of a metaclass.
 
    This is usually equivalent to accessing the :attr:`~object.__annotate__`
-   attribute of *obj*, but direct access to the attribute may return the wrong
+   attribute of *obj*, but access through this public function is preferred.
-   object in certain situations involving metaclasses. This function should be
-   used instead of accessing the attribute directly.
 
    .. versionadded:: 3.14
 

Include/internal/pycore_global_objects_fini_generated.h

@@ -587,7 +587,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__and__));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate_func__));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations_cache__));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__));

Include/internal/pycore_global_strings.h

@@ -78,7 +78,9 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(__and__)
         STRUCT_FOR_ID(__anext__)
         STRUCT_FOR_ID(__annotate__)
+        STRUCT_FOR_ID(__annotate_func__)
         STRUCT_FOR_ID(__annotations__)
+        STRUCT_FOR_ID(__annotations_cache__)
         STRUCT_FOR_ID(__args__)
         STRUCT_FOR_ID(__await__)
         STRUCT_FOR_ID(__bases__)

Include/internal/pycore_magic_number.h

@@ -274,6 +274,7 @@ Known values:
     Python 3.14a6 3619 (Renumber RESUME opcode from 149 to 128)
     Python 3.14a6 3620 (Optimize bytecode for all/any/tuple called on a genexp)
     Python 3.14a7 3621 (Optimize LOAD_FAST opcodes into LOAD_FAST_BORROW)
+    Python 3.14a7 3622 (Store annotations in different class dict keys)
 
     Python 3.15 will start with 3650
 
@@ -286,7 +287,7 @@ PC/launcher.c must also be updated.
 
 */
 
-#define PYC_MAGIC_NUMBER 3621
+#define PYC_MAGIC_NUMBER 3622
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \

Include/internal/pycore_runtime_init_generated.h

@@ -585,7 +585,9 @@ extern "C" {
     INIT_ID(__and__), \
     INIT_ID(__anext__), \
     INIT_ID(__annotate__), \
+    INIT_ID(__annotate_func__), \
     INIT_ID(__annotations__), \
+    INIT_ID(__annotations_cache__), \
     INIT_ID(__args__), \
     INIT_ID(__await__), \
     INIT_ID(__bases__), \

Include/internal/pycore_unicodeobject_generated.h

@@ -100,10 +100,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     _PyUnicode_InternStatic(interp, &string);
     assert(_PyUnicode_CheckConsistency(string, 1));
     assert(PyUnicode_GET_LENGTH(string) != 1);
+    string = &_Py_ID(__annotate_func__);
+    _PyUnicode_InternStatic(interp, &string);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    assert(PyUnicode_GET_LENGTH(string) != 1);
     string = &_Py_ID(__annotations__);
     _PyUnicode_InternStatic(interp, &string);
     assert(_PyUnicode_CheckConsistency(string, 1));
     assert(PyUnicode_GET_LENGTH(string) != 1);
+    string = &_Py_ID(__annotations_cache__);
+    _PyUnicode_InternStatic(interp, &string);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    assert(PyUnicode_GET_LENGTH(string) != 1);
     string = &_Py_ID(__args__);
     _PyUnicode_InternStatic(interp, &string);
     assert(_PyUnicode_CheckConsistency(string, 1));

Lib/annotationlib.py

@@ -619,14 +619,6 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False):
         raise ValueError(f"Invalid format: {format!r}")
 
 
-# We use the descriptors from builtins.type instead of accessing
-# .__annotations__ and .__annotate__ directly on class objects, because
-# otherwise we could get wrong results in some cases involving metaclasses.
-# See PEP 749.
-_BASE_GET_ANNOTATE = type.__dict__["__annotate__"].__get__
-_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__
-
-
 def get_annotate_function(obj):
     """Get the __annotate__ function for an object.
 
@@ -635,12 +627,11 @@ def get_annotate_function(obj):
 
     Returns the __annotate__ function or None.
     """
-    if isinstance(obj, type):
+    if isinstance(obj, dict):
         try:
-            return _BASE_GET_ANNOTATE(obj)
+            return obj["__annotate__"]
-        except AttributeError:
+        except KeyError:
-            # AttributeError is raised for static types.
+            return obj.get("__annotate_func__", None)
-            return None
     return getattr(obj, "__annotate__", None)
 
 
@@ -833,7 +824,7 @@ def _get_and_call_annotate(obj, format):
 def _get_dunder_annotations(obj):
     if isinstance(obj, type):
         try:
-            ann = _BASE_GET_ANNOTATIONS(obj)
+            ann = obj.__annotations__
         except AttributeError:
             # For static types, the descriptor raises AttributeError.
             return {}

Lib/pydoc.py

@@ -330,7 +330,8 @@ def visiblename(name, all=None, obj=None):
                 '__date__', '__doc__', '__file__', '__spec__',
                 '__loader__', '__module__', '__name__', '__package__',
                 '__path__', '__qualname__', '__slots__', '__version__',
-                '__static_attributes__', '__firstlineno__'}:
+                '__static_attributes__', '__firstlineno__',
+                '__annotate_func__', '__annotations_cache__'}:
         return 0
     # Private names are hidden, but special names are displayed.
     if name.startswith('__') and name.endswith('__'): return 1

Lib/test/test_ast/test_ast.py

@@ -298,7 +298,7 @@ class AST_Tests(unittest.TestCase):
         x = ast.arguments()
         self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs',
                                      'kw_defaults', 'kwarg', 'defaults'))
-        self.assertEqual(x.__annotations__, {
+        self.assertEqual(ast.arguments.__annotations__, {
             'posonlyargs': list[ast.arg],
             'args': list[ast.arg],
             'vararg': ast.arg | None,

Lib/test/test_pydoc/test_pydoc.py

@@ -78,11 +78,6 @@ CLASSES
      |  __weakref__%s
 
     class B(builtins.object)
-     |  Methods defined here:
-     |
-     |  __annotate__(format, /)
-     |
-     |  ----------------------------------------------------------------------
      |  Data descriptors defined here:
      |
      |  __dict__%s
@@ -180,9 +175,6 @@ class A(builtins.object)
             list of weak references to the object
 
 class B(builtins.object)
-    Methods defined here:
-        __annotate__(format, /)
-    ----------------------------------------------------------------------
     Data descriptors defined here:
         __dict__
             dictionary for instance variables

Lib/test/test_type_annotations.py

@@ -4,32 +4,33 @@ import textwrap
 import types
 import unittest
 from test.support import run_code, check_syntax_error, cpython_only
+from test.test_inspect import inspect_stringized_annotations
 
 
 class TypeAnnotationTests(unittest.TestCase):
 
     def test_lazy_create_annotations(self):
         # type objects lazy create their __annotations__ dict on demand.
-        # the annotations dict is stored in type.__dict__.
+        # the annotations dict is stored in type.__dict__ (as __annotations_cache__).
         # a freshly created type shouldn't have an annotations dict yet.
         foo = type("Foo", (), {})
         for i in range(3):
-            self.assertFalse("__annotations__" in foo.__dict__)
+            self.assertFalse("__annotations_cache__" in foo.__dict__)
             d = foo.__annotations__
-            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertTrue("__annotations_cache__" in foo.__dict__)
             self.assertEqual(foo.__annotations__, d)
-            self.assertEqual(foo.__dict__['__annotations__'], d)
+            self.assertEqual(foo.__dict__['__annotations_cache__'], d)
             del foo.__annotations__
 
     def test_setting_annotations(self):
         foo = type("Foo", (), {})
         for i in range(3):
-            self.assertFalse("__annotations__" in foo.__dict__)
+            self.assertFalse("__annotations_cache__" in foo.__dict__)
             d = {'a': int}
             foo.__annotations__ = d
-            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertTrue("__annotations_cache__" in foo.__dict__)
             self.assertEqual(foo.__annotations__, d)
-            self.assertEqual(foo.__dict__['__annotations__'], d)
+            self.assertEqual(foo.__dict__['__annotations_cache__'], d)
             del foo.__annotations__
 
     def test_annotations_getset_raises(self):
@@ -53,9 +54,30 @@ class TypeAnnotationTests(unittest.TestCase):
             a:int=3
             b:str=4
         self.assertEqual(C.__annotations__, {"a": int, "b": str})
-        self.assertTrue("__annotations__" in C.__dict__)
+        self.assertTrue("__annotations_cache__" in C.__dict__)
         del C.__annotations__
-        self.assertFalse("__annotations__" in C.__dict__)
+        self.assertFalse("__annotations_cache__" in C.__dict__)
+
+    def test_pep563_annotations(self):
+        isa = inspect_stringized_annotations
+        self.assertEqual(
+            isa.__annotations__, {"a": "int", "b": "str"},
+        )
+        self.assertEqual(
+            isa.MyClass.__annotations__, {"a": "int", "b": "str"},
+        )
+
+    def test_explicitly_set_annotations(self):
+        class C:
+            __annotations__ = {"what": int}
+        self.assertEqual(C.__annotations__, {"what": int})
+
+    def test_explicitly_set_annotate(self):
+        class C:
+            __annotate__ = lambda format: {"what": int}
+        self.assertEqual(C.__annotations__, {"what": int})
+        self.assertIsInstance(C.__annotate__, types.FunctionType)
+        self.assertEqual(C.__annotate__(annotationlib.Format.VALUE), {"what": int})
 
     def test_del_annotations_and_annotate(self):
         # gh-132285

Lib/test/test_typing.py

@@ -3825,6 +3825,7 @@ class ProtocolTests(BaseTestCase):
         acceptable_extra_attrs = {
             '_is_protocol', '_is_runtime_protocol', '__parameters__',
             '__init__', '__annotations__', '__subclasshook__', '__annotate__',
+            '__annotations_cache__', '__annotate_func__',
         }
         self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs)
         self.assertLessEqual(

Lib/typing.py

@@ -1784,7 +1784,7 @@ _SPECIAL_NAMES = frozenset({
     '__init__', '__module__', '__new__', '__slots__',
     '__subclasshook__', '__weakref__', '__class_getitem__',
     '__match_args__', '__static_attributes__', '__firstlineno__',
-    '__annotate__',
+    '__annotate__', '__annotate_func__', '__annotations_cache__',
 })
 
 # These special attributes will be not collected as protocol members.
@@ -2875,7 +2875,8 @@ _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__',
                          '_fields', '_field_defaults',
                          '_make', '_replace', '_asdict', '_source'})
 
-_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__'})
+_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__',
+                      '__annotate_func__', '__annotations_cache__'})
 
 
 class NamedTupleMeta(type):
@@ -2893,8 +2894,7 @@ class NamedTupleMeta(type):
             types = ns["__annotations__"]
             field_names = list(types)
             annotate = _make_eager_annotate(types)
-        elif "__annotate__" in ns:
+        elif (original_annotate := _lazy_annotationlib.get_annotate_function(ns)) is not None:
-            original_annotate = ns["__annotate__"]
             types = _lazy_annotationlib.call_annotate_function(
                 original_annotate, _lazy_annotationlib.Format.FORWARDREF)
             field_names = list(types)
@@ -3080,8 +3080,7 @@ class _TypedDictMeta(type):
         if "__annotations__" in ns:
             own_annotate = None
             own_annotations = ns["__annotations__"]
-        elif "__annotate__" in ns:
+        elif (own_annotate := _lazy_annotationlib.get_annotate_function(ns)) is not None:
-            own_annotate = ns["__annotate__"]
             own_annotations = _lazy_annotationlib.call_annotate_function(
                 own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict
             )

Misc/NEWS.d/next/Core_and_Builtins/2025-04-09-21-51-37.gh-issue-132261.gL8thm.rst

@@ -0,0 +1,4 @@
+The internal storage for annotations and annotate functions on classes now
+uses different keys in the class dictionary. This eliminates various edge
+cases where access to the ``__annotate__`` and ``__annotations__``
+attributes would behave unpredictably.

Objects/typeobject.c

@@ -1915,10 +1915,17 @@ type_get_annotate(PyObject *tp, void *Py_UNUSED(closure))
 
     PyObject *annotate;
     PyObject *dict = PyType_GetDict(type);
+    // First try __annotate__, in case that's been set explicitly
     if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) {
         Py_DECREF(dict);
         return NULL;
     }
+    if (!annotate) {
+        if (PyDict_GetItemRef(dict, &_Py_ID(__annotate_func__), &annotate) < 0) {
+            Py_DECREF(dict);
+            return NULL;
+        }
+    }
     if (annotate) {
         descrgetfunc get = Py_TYPE(annotate)->tp_descr_get;
         if (get) {
@@ -1927,7 +1934,7 @@ type_get_annotate(PyObject *tp, void *Py_UNUSED(closure))
     }
     else {
         annotate = Py_None;
-        int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate);
+        int result = PyDict_SetItem(dict, &_Py_ID(__annotate_func__), annotate);
         if (result < 0) {
             Py_DECREF(dict);
             return NULL;
@@ -1959,13 +1966,13 @@ type_set_annotate(PyObject *tp, PyObject *value, void *Py_UNUSED(closure))
 
     PyObject *dict = PyType_GetDict(type);
     assert(PyDict_Check(dict));
-    int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value);
+    int result = PyDict_SetItem(dict, &_Py_ID(__annotate_func__), value);
     if (result < 0) {
         Py_DECREF(dict);
         return -1;
     }
     if (!Py_IsNone(value)) {
-        if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) {
+        if (PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL) == -1) {
             Py_DECREF(dict);
             PyType_Modified(type);
             return -1;
@@ -1987,10 +1994,18 @@ type_get_annotations(PyObject *tp, void *Py_UNUSED(closure))
 
     PyObject *annotations;
     PyObject *dict = PyType_GetDict(type);
+    // First try __annotations__ (e.g. for "from __future__ import annotations")
     if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) {
         Py_DECREF(dict);
         return NULL;
     }
+    if (!annotations) {
+        if (PyDict_GetItemRef(dict, &_Py_ID(__annotations_cache__), &annotations) < 0) {
+            Py_DECREF(dict);
+            return NULL;
+        }
+    }
+
     if (annotations) {
         descrgetfunc get = Py_TYPE(annotations)->tp_descr_get;
         if (get) {
@@ -1998,7 +2013,7 @@ type_get_annotations(PyObject *tp, void *Py_UNUSED(closure))
         }
     }
     else {
-        PyObject *annotate = type_get_annotate(tp, NULL);
+        PyObject *annotate = PyObject_GetAttrString((PyObject *)type, "__annotate__");
         if (annotate == NULL) {
             Py_DECREF(dict);
             return NULL;
@@ -2026,7 +2041,7 @@ type_get_annotations(PyObject *tp, void *Py_UNUSED(closure))
         Py_DECREF(annotate);
         if (annotations) {
             int result = PyDict_SetItem(
-                    dict, &_Py_ID(__annotations__), annotations);
+                    dict, &_Py_ID(__annotations_cache__), annotations);
             if (result) {
                 Py_CLEAR(annotations);
             } else {
@@ -2053,10 +2068,10 @@ type_set_annotations(PyObject *tp, PyObject *value, void *Py_UNUSED(closure))
     PyObject *dict = PyType_GetDict(type);
     if (value != NULL) {
         /* set */
-        result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value);
+        result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value);
     } else {
         /* delete */
-        result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL);
+        result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL);
         if (result == 0) {
             PyErr_SetString(PyExc_AttributeError, "__annotations__");
             Py_DECREF(dict);
@@ -2067,6 +2082,11 @@ type_set_annotations(PyObject *tp, PyObject *value, void *Py_UNUSED(closure))
         Py_DECREF(dict);
         return -1;
     } else {  // result can be 0 or 1
+        if (PyDict_Pop(dict, &_Py_ID(__annotate_func__), NULL) < 0) {
+            PyType_Modified(type);
+            Py_DECREF(dict);
+            return -1;
+        }
         if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) {
             PyType_Modified(type);
             Py_DECREF(dict);

Python/codegen.c

@@ -815,7 +815,10 @@ codegen_process_deferred_annotations(compiler *c, location loc)
     Py_DECREF(conditional_annotation_indices);
 
     RETURN_IF_ERROR(codegen_leave_annotations_scope(c, loc));
-    RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_ID(__annotate__), Store));
+    RETURN_IF_ERROR(codegen_nameop(
+        c, loc,
+        ste->ste_type == ClassBlock ? &_Py_ID(__annotate_func__) : &_Py_ID(__annotate__),
+        Store));
 
     return SUCCESS;
 error: