bpo-39481: Implementation for PEP 585 (#18239)

This implements things like `list[int]`,
which returns an object of type `types.GenericAlias`.
This object mostly acts as a proxy for `list`,
but has attributes `__origin__` and `__args__`
that allow recovering the parts (with values `list` and `(int,)`.

There is also an approximate notion of type variables;
e.g. `list[T]` has a `__parameters__` attribute equal to `(T,)`.
Type variables are objects of type `typing.TypeVar`.

Lib/_collections_abc.py

@@ -9,6 +9,8 @@ Unit tests are in test_collections.
 from abc import ABCMeta, abstractmethod
 import sys
 
+GenericAlias = type(list[int])
+
 __all__ = ["Awaitable", "Coroutine",
            "AsyncIterable", "AsyncIterator", "AsyncGenerator",
            "Hashable", "Iterable", "Iterator", "Generator", "Reversible",
@@ -110,6 +112,8 @@ class Awaitable(metaclass=ABCMeta):
             return _check_methods(C, "__await__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class Coroutine(Awaitable):
 
@@ -169,6 +173,8 @@ class AsyncIterable(metaclass=ABCMeta):
             return _check_methods(C, "__aiter__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class AsyncIterator(AsyncIterable):
 
@@ -255,6 +261,8 @@ class Iterable(metaclass=ABCMeta):
             return _check_methods(C, "__iter__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class Iterator(Iterable):
 
@@ -274,6 +282,7 @@ class Iterator(Iterable):
             return _check_methods(C, '__iter__', '__next__')
         return NotImplemented
 
+
 Iterator.register(bytes_iterator)
 Iterator.register(bytearray_iterator)
 #Iterator.register(callable_iterator)
@@ -353,6 +362,7 @@ class Generator(Iterator):
                                   'send', 'throw', 'close')
         return NotImplemented
 
+
 Generator.register(generator)
 
 
@@ -385,6 +395,9 @@ class Container(metaclass=ABCMeta):
             return _check_methods(C, "__contains__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
+
 class Collection(Sized, Iterable, Container):
 
     __slots__ = ()
@@ -395,6 +408,7 @@ class Collection(Sized, Iterable, Container):
             return _check_methods(C,  "__len__", "__iter__", "__contains__")
         return NotImplemented
 
+
 class Callable(metaclass=ABCMeta):
 
     __slots__ = ()
@@ -409,6 +423,8 @@ class Callable(metaclass=ABCMeta):
             return _check_methods(C, "__call__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 ### SETS ###
 
@@ -550,6 +566,7 @@ class Set(Collection):
             h = 590923713
         return h
 
+
 Set.register(frozenset)
 
 
@@ -632,6 +649,7 @@ class MutableSet(Set):
                 self.discard(value)
         return self
 
+
 MutableSet.register(set)
 
 
@@ -688,6 +706,7 @@ class Mapping(Collection):
 
     __reversed__ = None
 
+
 Mapping.register(mappingproxy)
 
 
@@ -704,6 +723,8 @@ class MappingView(Sized):
     def __repr__(self):
         return '{0.__class__.__name__}({0._mapping!r})'.format(self)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class KeysView(MappingView, Set):
 
@@ -719,6 +740,7 @@ class KeysView(MappingView, Set):
     def __iter__(self):
         yield from self._mapping
 
+
 KeysView.register(dict_keys)
 
 
@@ -743,6 +765,7 @@ class ItemsView(MappingView, Set):
         for key in self._mapping:
             yield (key, self._mapping[key])
 
+
 ItemsView.register(dict_items)
 
 
@@ -761,6 +784,7 @@ class ValuesView(MappingView, Collection):
         for key in self._mapping:
             yield self._mapping[key]
 
+
 ValuesView.register(dict_values)
 
 
@@ -847,6 +871,7 @@ class MutableMapping(Mapping):
             self[key] = default
         return default
 
+
 MutableMapping.register(dict)
 
 
@@ -914,6 +939,7 @@ class Sequence(Reversible, Collection):
         'S.count(value) -> integer -- return number of occurrences of value'
         return sum(1 for v in self if v is value or v == value)
 
+
 Sequence.register(tuple)
 Sequence.register(str)
 Sequence.register(range)
@@ -1000,5 +1026,6 @@ class MutableSequence(Sequence):
         self.extend(values)
         return self
 
+
 MutableSequence.register(list)
 MutableSequence.register(bytearray)  # Multiply inheriting, see ByteString

Lib/contextlib.py

@@ -4,7 +4,7 @@ import sys
 import _collections_abc
 from collections import deque
 from functools import wraps
-from types import MethodType
+from types import MethodType, GenericAlias
 
 __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
            "AbstractContextManager", "AbstractAsyncContextManager",
@@ -16,6 +16,8 @@ class AbstractContextManager(abc.ABC):
 
     """An abstract base class for context managers."""
 
+    __class_getitem__ = classmethod(GenericAlias)
+
     def __enter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -36,6 +38,8 @@ class AbstractAsyncContextManager(abc.ABC):
 
     """An abstract base class for asynchronous context managers."""
 
+    __class_getitem__ = classmethod(GenericAlias)
+
     async def __aenter__(self):
         """Return `self` upon entering the runtime context."""
         return self

Lib/os.py

@@ -28,6 +28,8 @@ import stat as st
 
 from _collections_abc import _check_methods
 
+GenericAlias = type(list[int])
+
 _names = sys.builtin_module_names
 
 # Note:  more names are added to __all__ later.
@@ -1074,8 +1076,7 @@ class PathLike(abc.ABC):
             return _check_methods(subclass, '__fspath__')
         return NotImplemented
 
-    def __class_getitem__(cls, type):
-        return cls
+    __class_getitem__ = classmethod(GenericAlias)
 
 
 if name == 'nt':

Lib/subprocess.py

@@ -52,6 +52,7 @@ import threading
 import warnings
 import contextlib
 from time import monotonic as _time
+import types
 
 try:
     import pwd
@@ -446,17 +447,7 @@ class CompletedProcess(object):
             args.append('stderr={!r}'.format(self.stderr))
         return "{}({})".format(type(self).__name__, ', '.join(args))
 
-    def __class_getitem__(cls, type):
-        """Provide minimal support for using this class as generic
-        (for example in type annotations).
-
-        See PEP 484 and PEP 560 for more details. For example,
-        `CompletedProcess[bytes]` is a valid expression at runtime
-        (type argument `bytes` indicates the type used for stdout).
-        Note, no type checking happens at runtime, but a static type
-        checker can be used.
-        """
-        return cls
+    __class_getitem__ = classmethod(types.GenericAlias)
 
 
     def check_returncode(self):
@@ -1000,16 +991,7 @@ class Popen(object):
             obj_repr = obj_repr[:76] + "...>"
         return obj_repr
 
-    def __class_getitem__(cls, type):
-        """Provide minimal support for using this class as generic
-        (for example in type annotations).
-
-        See PEP 484 and PEP 560 for more details. For example, `Popen[bytes]`
-        is a valid expression at runtime (type argument `bytes` indicates the
-        type used for stdout). Note, no type checking happens at runtime, but
-        a static type checker can be used.
-        """
-        return cls
+    __class_getitem__ = classmethod(types.GenericAlias)
 
     @property
     def universal_newlines(self):

Lib/tempfile.py

@@ -44,6 +44,7 @@ import shutil as _shutil
 import errno as _errno
 from random import Random as _Random
 import sys as _sys
+import types as _types
 import weakref as _weakref
 import _thread
 _allocate_lock = _thread.allocate_lock
@@ -643,17 +644,7 @@ class SpooledTemporaryFile:
                                    'encoding': encoding, 'newline': newline,
                                    'dir': dir, 'errors': errors}
 
-    def __class_getitem__(cls, type):
-        """Provide minimal support for using this class as generic
-        (for example in type annotations).
-
-        See PEP 484 and PEP 560 for more details. For example,
-        `SpooledTemporaryFile[str]` is a valid expression at runtime (type
-        argument `str` indicates whether the file is open in bytes or text
-        mode). Note, no type checking happens at runtime, but a static type
-        checker can be used.
-        """
-        return cls
+    __class_getitem__ = classmethod(_types.GenericAlias)
 
     def _check(self, file):
         if self._rolled: return

Lib/test/test_descrtut.py

@@ -167,6 +167,7 @@ You can get the information from the list type:
     >>> pprint.pprint(dir(list))    # like list.__dict__.keys(), but sorted
     ['__add__',
      '__class__',
+     '__class_getitem__',
      '__contains__',
      '__delattr__',
      '__delitem__',

Lib/test/test_doctest.py

@@ -665,7 +665,7 @@ plain ol' Python and is guaranteed to be available.
 
     >>> import builtins
     >>> tests = doctest.DocTestFinder().find(builtins)
-    >>> 800 < len(tests) < 820 # approximate number of objects with docstrings
+    >>> 810 < len(tests) < 830 # approximate number of objects with docstrings
     True
     >>> real_tests = [t for t in tests if len(t.examples) > 0]
     >>> len(real_tests) # objects that actually have doctests

Lib/test/test_genericalias.py

@@ -0,0 +1,214 @@
+"""Tests for C-implemented GenericAlias."""
+
+import unittest
+import pickle
+from collections import (
+    defaultdict, deque, OrderedDict, Counter, UserDict, UserList
+)
+from collections.abc import *
+from contextlib import AbstractContextManager, AbstractAsyncContextManager
+from re import Pattern, Match
+from types import GenericAlias, MappingProxyType
+import typing
+
+from typing import TypeVar
+T = TypeVar('T')
+
+class BaseTest(unittest.TestCase):
+    """Test basics."""
+
+    def test_subscriptable(self):
+        for t in (type, tuple, list, dict, set, frozenset,
+                  defaultdict, deque,
+                  OrderedDict, Counter, UserDict, UserList,
+                  Pattern, Match,
+                  AbstractContextManager, AbstractAsyncContextManager,
+                  Awaitable, Coroutine,
+                  AsyncIterable, AsyncIterator,
+                  AsyncGenerator, Generator,
+                  Iterable, Iterator,
+                  Reversible,
+                  Container, Collection,
+                  Callable,
+                  Set, MutableSet,
+                  Mapping, MutableMapping, MappingView,
+                  KeysView, ItemsView, ValuesView,
+                  Sequence, MutableSequence,
+                  MappingProxyType,
+                  ):
+            tname = t.__name__
+            with self.subTest(f"Testing {tname}"):
+                alias = t[int]
+                self.assertIs(alias.__origin__, t)
+                self.assertEqual(alias.__args__, (int,))
+                self.assertEqual(alias.__parameters__, ())
+
+    def test_unsubscriptable(self):
+        for t in int, str, float, Sized, Hashable:
+            tname = t.__name__
+            with self.subTest(f"Testing {tname}"):
+                with self.assertRaises(TypeError):
+                    t[int]
+
+    def test_instantiate(self):
+        for t in tuple, list, dict, set, frozenset, defaultdict, deque:
+            tname = t.__name__
+            with self.subTest(f"Testing {tname}"):
+                alias = t[int]
+                self.assertEqual(alias(), t())
+                if t is dict:
+                    self.assertEqual(alias(iter([('a', 1), ('b', 2)])), dict(a=1, b=2))
+                    self.assertEqual(alias(a=1, b=2), dict(a=1, b=2))
+                elif t is defaultdict:
+                    def default():
+                        return 'value'
+                    a = alias(default)
+                    d = defaultdict(default)
+                    self.assertEqual(a['test'], d['test'])
+                else:
+                    self.assertEqual(alias(iter((1, 2, 3))), t((1, 2, 3)))
+
+    def test_unbound_methods(self):
+        t = list[int]
+        a = t()
+        t.append(a, 'foo')
+        self.assertEqual(a, ['foo'])
+        x = t.__getitem__(a, 0)
+        self.assertEqual(x, 'foo')
+        self.assertEqual(t.__len__(a), 1)
+
+    def test_subclassing(self):
+        class C(list[int]):
+            pass
+        self.assertEqual(C.__bases__, (list,))
+        self.assertEqual(C.__class__, type)
+
+    def test_class_methods(self):
+        t = dict[int, None]
+        self.assertEqual(dict.fromkeys(range(2)), {0: None, 1: None})  # This works
+        self.assertEqual(t.fromkeys(range(2)), {0: None, 1: None})  # Should be equivalent
+
+    def test_no_chaining(self):
+        t = list[int]
+        with self.assertRaises(TypeError):
+            t[int]
+
+    def test_generic_subclass(self):
+        class MyList(list):
+            pass
+        t = MyList[int]
+        self.assertIs(t.__origin__, MyList)
+        self.assertEqual(t.__args__, (int,))
+        self.assertEqual(t.__parameters__, ())
+
+    def test_repr(self):
+        class MyList(list):
+            pass
+        self.assertEqual(repr(list[str]), 'list[str]')
+        self.assertEqual(repr(list[()]), 'list[()]')
+        self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]')
+        self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr.<locals>.MyList[int]'))
+        self.assertEqual(repr(list[str]()), '[]')  # instances should keep their normal repr
+
+    def test_exposed_type(self):
+        import types
+        a = types.GenericAlias(list, int)
+        self.assertEqual(str(a), 'list[int]')
+        self.assertIs(a.__origin__, list)
+        self.assertEqual(a.__args__, (int,))
+        self.assertEqual(a.__parameters__, ())
+
+    def test_parameters(self):
+        from typing import TypeVar
+        T = TypeVar('T')
+        K = TypeVar('K')
+        V = TypeVar('V')
+        D0 = dict[str, int]
+        self.assertEqual(D0.__args__, (str, int))
+        self.assertEqual(D0.__parameters__, ())
+        D1a = dict[str, V]
+        self.assertEqual(D1a.__args__, (str, V))
+        self.assertEqual(D1a.__parameters__, (V,))
+        D1b = dict[K, int]
+        self.assertEqual(D1b.__args__, (K, int))
+        self.assertEqual(D1b.__parameters__, (K,))
+        D2a = dict[K, V]
+        self.assertEqual(D2a.__args__, (K, V))
+        self.assertEqual(D2a.__parameters__, (K, V))
+        D2b = dict[T, T]
+        self.assertEqual(D2b.__args__, (T, T))
+        self.assertEqual(D2b.__parameters__, (T,))
+        L0 = list[str]
+        self.assertEqual(L0.__args__, (str,))
+        self.assertEqual(L0.__parameters__, ())
+        L1 = list[T]
+        self.assertEqual(L1.__args__, (T,))
+        self.assertEqual(L1.__parameters__, (T,))
+
+    def test_parameter_chaining(self):
+        from typing import TypeVar
+        T = TypeVar('T')
+        self.assertEqual(list[T][int], list[int])
+        self.assertEqual(dict[str, T][int], dict[str, int])
+        self.assertEqual(dict[T, int][str], dict[str, int])
+        self.assertEqual(dict[T, T][int], dict[int, int])
+        with self.assertRaises(TypeError):
+            list[int][int]
+            dict[T, int][str, int]
+            dict[str, T][str, int]
+            dict[T, T][str, int]
+
+    def test_equality(self):
+        self.assertEqual(list[int], list[int])
+        self.assertEqual(dict[str, int], dict[str, int])
+        self.assertNotEqual(dict[str, int], dict[str, str])
+        self.assertNotEqual(list, list[int])
+        self.assertNotEqual(list[int], list)
+
+    def test_isinstance(self):
+        self.assertTrue(isinstance([], list))
+        with self.assertRaises(TypeError):
+            isinstance([], list[str])
+
+    def test_issubclass(self):
+        class L(list): ...
+        self.assertTrue(issubclass(L, list))
+        with self.assertRaises(TypeError):
+            issubclass(L, list[str])
+
+    def test_type_generic(self):
+        t = type[int]
+        Test = t('Test', (), {})
+        self.assertTrue(isinstance(Test, type))
+        test = Test()
+        self.assertEqual(t(test), Test)
+        self.assertEqual(t(0), int)
+
+    def test_type_subclass_generic(self):
+        class MyType(type):
+            pass
+        with self.assertRaises(TypeError):
+            MyType[int]
+
+    def test_pickle(self):
+        alias = GenericAlias(list, T)
+        s = pickle.dumps(alias)
+        loaded = pickle.loads(s)
+        self.assertEqual(alias.__origin__, loaded.__origin__)
+        self.assertEqual(alias.__args__, loaded.__args__)
+        self.assertEqual(alias.__parameters__, loaded.__parameters__)
+
+    def test_union(self):
+        a = typing.Union[list[int], list[str]]
+        self.assertEqual(a.__args__, (list[int], list[str]))
+        self.assertEqual(a.__parameters__, ())
+
+    def test_union_generic(self):
+        T = typing.TypeVar('T')
+        a = typing.Union[list[T], tuple[T, ...]]
+        self.assertEqual(a.__args__, (list[T], tuple[T, ...]))
+        self.assertEqual(a.__parameters__, (T,))
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_os.py

@@ -25,6 +25,7 @@ import sysconfig
 import tempfile
 import threading
 import time
+import types
 import unittest
 import uuid
 import warnings
@@ -4191,7 +4192,7 @@ class TestPEP519(unittest.TestCase):
         self.assertTrue(issubclass(FakePath, os.PathLike))
 
     def test_pathlike_class_getitem(self):
-        self.assertIs(os.PathLike[bytes], os.PathLike)
+        self.assertIsInstance(os.PathLike[bytes], types.GenericAlias)
 
 
 class TimesTests(unittest.TestCase):

Lib/test/test_subprocess.py

@@ -11,6 +11,7 @@ import errno
 import tempfile
 import time
 import traceback
+import types
 import selectors
 import sysconfig
 import select
@@ -1435,8 +1436,8 @@ class ProcessTestCase(BaseTestCase):
         self.assertEqual(c.exception.filename, '/some/nonexistent/directory')
 
     def test_class_getitems(self):
-        self.assertIs(subprocess.Popen[bytes], subprocess.Popen)
-        self.assertIs(subprocess.CompletedProcess[str], subprocess.CompletedProcess)
+        self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias)
+        self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias)
 
 class RunFuncTestCase(BaseTestCase):
     def run_python(self, code, **kwargs):

Lib/test/test_tempfile.py

@@ -10,6 +10,7 @@ import re
 import warnings
 import contextlib
 import stat
+import types
 import weakref
 from unittest import mock
 
@@ -1222,8 +1223,8 @@ class TestSpooledTemporaryFile(BaseTestCase):
         self.assertEqual(os.fstat(f.fileno()).st_size, 20)
 
     def test_class_getitem(self):
-        self.assertIs(tempfile.SpooledTemporaryFile[bytes],
-                      tempfile.SpooledTemporaryFile)
+        self.assertIsInstance(tempfile.SpooledTemporaryFile[bytes],
+                      types.GenericAlias)
 
 if tempfile.NamedTemporaryFile is not tempfile.TemporaryFile:
 

Lib/test/test_types.py

@@ -622,6 +622,7 @@ class MappingProxyTests(unittest.TestCase):
         self.assertEqual(attrs, {
              '__contains__',
              '__getitem__',
+             '__class_getitem__',
              '__ior__',
              '__iter__',
              '__len__',

Lib/test/test_typing.py

@@ -1775,10 +1775,11 @@ class GenericTests(BaseTestCase):
         self.assertEqual(T1[int, T].__origin__, T1)
 
         self.assertEqual(T2.__parameters__, (T,))
-        with self.assertRaises(TypeError):
-            T1[int]
-        with self.assertRaises(TypeError):
-            T2[int, str]
+        # These don't work because of tuple.__class_item__
+        ## with self.assertRaises(TypeError):
+        ##     T1[int]
+        ## with self.assertRaises(TypeError):
+        ##     T2[int, str]
 
         self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]')
         self.assertEqual(C2.__parameters__, ())
@@ -1817,22 +1818,22 @@ class GenericTests(BaseTestCase):
         self.clear_caches()
         class MyTup(Tuple[T, T]): ...
         self.assertIs(MyTup[int]().__class__, MyTup)
-        self.assertIs(MyTup[int]().__orig_class__, MyTup[int])
+        self.assertEqual(MyTup[int]().__orig_class__, MyTup[int])
         class MyCall(Callable[..., T]):
             def __call__(self): return None
         self.assertIs(MyCall[T]().__class__, MyCall)
-        self.assertIs(MyCall[T]().__orig_class__, MyCall[T])
+        self.assertEqual(MyCall[T]().__orig_class__, MyCall[T])
         class MyDict(typing.Dict[T, T]): ...
         self.assertIs(MyDict[int]().__class__, MyDict)
-        self.assertIs(MyDict[int]().__orig_class__, MyDict[int])
+        self.assertEqual(MyDict[int]().__orig_class__, MyDict[int])
         class MyDef(typing.DefaultDict[str, T]): ...
         self.assertIs(MyDef[int]().__class__, MyDef)
-        self.assertIs(MyDef[int]().__orig_class__, MyDef[int])
+        self.assertEqual(MyDef[int]().__orig_class__, MyDef[int])
         # ChainMap was added in 3.3
         if sys.version_info >= (3, 3):
             class MyChain(typing.ChainMap[str, T]): ...
             self.assertIs(MyChain[int]().__class__, MyChain)
-            self.assertIs(MyChain[int]().__orig_class__, MyChain[int])
+            self.assertEqual(MyChain[int]().__orig_class__, MyChain[int])
 
     def test_all_repr_eq_any(self):
         objs = (getattr(typing, el) for el in typing.__all__)

Lib/types.py

@@ -293,4 +293,7 @@ def coroutine(func):
     return wrapped
 
 
+GenericAlias = type(list[int])
+
+
 __all__ = [n for n in globals() if n[:1] != '_']

Lib/typing.py

@@ -26,7 +26,7 @@ import operator
 import re as stdlib_re  # Avoid confusion with the re we export.
 import sys
 import types
-from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType
+from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias
 
 # Please keep __all__ alphabetized within each category.
 __all__ = [
@@ -180,7 +180,8 @@ def _collect_type_vars(types):
     for t in types:
         if isinstance(t, TypeVar) and t not in tvars:
             tvars.append(t)
-        if isinstance(t, _GenericAlias) and not t._special:
+        if ((isinstance(t, _GenericAlias) and not t._special)
+                or isinstance(t, GenericAlias)):
             tvars.extend([t for t in t.__parameters__ if t not in tvars])
     return tuple(tvars)
 
@@ -947,7 +948,7 @@ _TYPING_INTERNALS = ['__parameters__', '__orig_bases__',  '__orig_class__',
 
 _SPECIAL_NAMES = ['__abstractmethods__', '__annotations__', '__dict__', '__doc__',
                   '__init__', '__module__', '__new__', '__slots__',
-                  '__subclasshook__', '__weakref__']
+                  '__subclasshook__', '__weakref__', '__class_getitem__']
 
 # These special attributes will be not collected as protocol members.
 EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS + _SPECIAL_NAMES + ['_MutableMapping__marker']