bpo-41559: Change PEP 612 implementation to pure Python (#25449)

Lib/_collections_abc.py

@@ -443,6 +443,18 @@ class _CallableGenericAlias(GenericAlias):
             ga_args = args
         return super().__new__(cls, origin, ga_args)
 
+    @property
+    def __parameters__(self):
+        params = []
+        for arg in self.__args__:
+            # Looks like a genericalias
+            if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple):
+                params.extend(arg.__parameters__)
+            else:
+                if _is_typevarlike(arg):
+                    params.append(arg)
+        return tuple(dict.fromkeys(params))
+
     def __repr__(self):
         if _has_special_args(self.__args__):
             return super().__repr__()
@@ -458,16 +470,50 @@ class _CallableGenericAlias(GenericAlias):
 
     def __getitem__(self, item):
         # Called during TypeVar substitution, returns the custom subclass
-        # rather than the default types.GenericAlias object.
+        # rather than the default types.GenericAlias object.  Most of the
-        ga = super().__getitem__(item)
+        # code is copied from typing's _GenericAlias and the builtin
-        args = ga.__args__
+        # types.GenericAlias.
-        # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
-        if not isinstance(ga.__args__[0], tuple):
-            t_result = ga.__args__[-1]
-            t_args = ga.__args__[:-1]
-            args = (t_args, t_result)
-        return _CallableGenericAlias(Callable, args)
 
+        # A special case in PEP 612 where if X = Callable[P, int],
+        # then X[int, str] == X[[int, str]].
+        param_len = len(self.__parameters__)
+        if param_len == 0:
+            raise TypeError(f'There are no type or parameter specification'
+                            f'variables left in {self}')
+        if (param_len == 1
+                and isinstance(item, (tuple, list))
+                and len(item) > 1) or not isinstance(item, tuple):
+            item = (item,)
+        item_len = len(item)
+        if item_len != param_len:
+            raise TypeError(f'Too {"many" if item_len > param_len else "few"}'
+                            f' arguments for {self};'
+                            f' actual {item_len}, expected {param_len}')
+        subst = dict(zip(self.__parameters__, item))
+        new_args = []
+        for arg in self.__args__:
+            if _is_typevarlike(arg):
+                arg = subst[arg]
+            # Looks like a GenericAlias
+            elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple):
+                subparams = arg.__parameters__
+                if subparams:
+                    subargs = tuple(subst[x] for x in subparams)
+                    arg = arg[subargs]
+            new_args.append(arg)
+
+        # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
+        if not isinstance(new_args[0], (tuple, list)):
+            t_result = new_args[-1]
+            t_args = new_args[:-1]
+            new_args = (t_args, t_result)
+        return _CallableGenericAlias(Callable, tuple(new_args))
+
+def _is_typevarlike(arg):
+    obj = type(arg)
+    # looks like a TypeVar/ParamSpec
+    return (obj.__module__ == 'typing'
+            and obj.__name__ in {'ParamSpec', 'TypeVar'})
 
 def _has_special_args(args):
     """Checks if args[0] matches either ``...``, ``ParamSpec`` or

Lib/test/test_genericalias.py

@@ -353,6 +353,12 @@ class BaseTest(unittest.TestCase):
             self.assertEqual(repr(C4[dict]).split(".")[-1], "Callable[[int, dict], str]")
             self.assertEqual(C4[dict], Callable[[int, dict], str])
 
+            # substitute a nested GenericAlias (both typing and the builtin
+            # version)
+            C5 = Callable[[typing.List[T], tuple[K, T], V], int]
+            self.assertEqual(C5[int, str, float],
+                             Callable[[typing.List[int], tuple[str, int], float], int])
+
         with self.subTest("Testing type erasure"):
             class C1(Callable):
                 def __call__(self):
@@ -391,5 +397,16 @@ class BaseTest(unittest.TestCase):
             self.assertEqual(repr(C1), "collections.abc.Callable"
                                        "[typing.Concatenate[int, ~P], int]")
 
+        with self.subTest("Testing TypeErrors"):
+            with self.assertRaisesRegex(TypeError, "variables left in"):
+                alias[int]
+            P = typing.ParamSpec('P')
+            C1 = Callable[P, T]
+            with self.assertRaisesRegex(TypeError, "many arguments for"):
+                C1[int, str, str]
+            with self.assertRaisesRegex(TypeError, "few arguments for"):
+                C1[int]
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Library/2021-04-17-10-49-57.bpo-41559.caIwt9.rst

@@ -0,0 +1,6 @@
+:pep:`612` is now implemented purely in Python; builtin ``types.GenericAlias``
+objects no longer include ``typing.ParamSpec`` in ``__parameters__``
+(with the exception of ``collections.abc.Callable``\ 's ``GenericAlias``).
+This means previously invalid uses of ``ParamSpec`` (such as
+``list[P]``) which worked in earlier versions of Python 3.10 alpha,
+will now raise ``TypeError`` during substitution.

Objects/genericaliasobject.c

@@ -156,25 +156,13 @@ error:
     return NULL;
 }
 
-/* Checks if a variable number of names are from typing.py.
+// isinstance(obj, TypeVar) without importing typing.py.
-*  If any one of the names are found, return 1, else 0.
+// Returns -1 for errors.
-**/
+static int
-static inline int
+is_typevar(PyObject *obj)
-is_typing_name(PyObject *obj, int num, ...)
 {
-    va_list names;
-    va_start(names, num);
-
     PyTypeObject *type = Py_TYPE(obj);
-    int hit = 0;
+    if (strcmp(type->tp_name, "TypeVar") != 0) {
-    for (int i = 0; i < num; ++i) {
-        if (!strcmp(type->tp_name, va_arg(names, const char *))) {
-            hit = 1;
-            break;
-        }
-    }
-    va_end(names);
-    if (!hit) {
         return 0;
     }
     PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__");
@@ -184,24 +172,9 @@ is_typing_name(PyObject *obj, int num, ...)
     int res = PyUnicode_Check(module)
         && _PyUnicode_EqualToASCIIString(module, "typing");
     Py_DECREF(module);
-    
     return res;
 }
 
-// isinstance(obj, (TypeVar, ParamSpec)) without importing typing.py.
-// Returns -1 for errors.
-static inline int
-is_typevarlike(PyObject *obj)
-{
-    return is_typing_name(obj, 2, "TypeVar", "ParamSpec");
-}
-
-static inline int
-is_paramspec(PyObject *obj)
-{
-    return is_typing_name(obj, 1, "ParamSpec");
-}
-
 // Index of item in self[:len], or -1 if not found (self is a tuple)
 static Py_ssize_t
 tuple_index(PyObject *self, Py_ssize_t len, PyObject *item)
@@ -236,7 +209,7 @@ make_parameters(PyObject *args)
     Py_ssize_t iparam = 0;
     for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) {
         PyObject *t = PyTuple_GET_ITEM(args, iarg);
-        int typevar = is_typevarlike(t);
+        int typevar = is_typevar(t);
         if (typevar < 0) {
             Py_DECREF(parameters);
             return NULL;
@@ -306,14 +279,7 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems)
             if (iparam >= 0) {
                 arg = argitems[iparam];
             }
-            // convert all the lists inside args to tuples to help
+            Py_INCREF(arg);
-            // with caching in other libaries
-            if (PyList_CheckExact(arg)) {
-                arg = PyList_AsTuple(arg);
-            }
-            else {
-                Py_INCREF(arg);
-            }
             PyTuple_SET_ITEM(subargs, i, arg);
         }
 
@@ -348,19 +314,11 @@ ga_getitem(PyObject *self, PyObject *item)
     int is_tuple = PyTuple_Check(item);
     Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
     PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
-    // A special case in PEP 612 where if X = Callable[P, int], 
+    if (nitems != nparams) {
-    // then X[int, str] == X[[int, str]].
+        return PyErr_Format(PyExc_TypeError,
-    if (nparams == 1 && nitems > 1 && is_tuple &&
+                            "Too %s arguments for %R",
-        is_paramspec(PyTuple_GET_ITEM(alias->parameters, 0))) {
+                            nitems > nparams ? "many" : "few",
-        argitems = &item;
+                            self);
-    }
-    else {
-        if (nitems != nparams) {
-            return PyErr_Format(PyExc_TypeError,
-                "Too %s arguments for %R",
-                nitems > nparams ? "many" : "few",
-                self);
-        }
     }
     /* Replace all type variables (specified by alias->parameters)
        with corresponding values specified by argitems.
@@ -375,7 +333,7 @@ ga_getitem(PyObject *self, PyObject *item)
     }
     for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) {
         PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg);
-        int typevar = is_typevarlike(arg);
+        int typevar = is_typevar(arg);
         if (typevar < 0) {
             Py_DECREF(newargs);
             return NULL;
@@ -384,13 +342,7 @@ ga_getitem(PyObject *self, PyObject *item)
             Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg);
             assert(iparam >= 0);
             arg = argitems[iparam];
-            // convert lists to tuples to help with caching in other libaries.
+            Py_INCREF(arg);
-            if (PyList_CheckExact(arg)) {
-                arg = PyList_AsTuple(arg);
-            }
-            else {
-                Py_INCREF(arg);
-            }
         }
         else {
             arg = subs_tvars(arg, alias->parameters, argitems);