bpo-39481: Implementation for PEP 585 (#18239)

This implements things like `list[int]`,
which returns an object of type `types.GenericAlias`.
This object mostly acts as a proxy for `list`,
but has attributes `__origin__` and `__args__`
that allow recovering the parts (with values `list` and `(int,)`.

There is also an approximate notion of type variables;
e.g. `list[T]` has a `__parameters__` attribute equal to `(T,)`.
Type variables are objects of type `typing.TypeVar`.

Include/Python.h

@@ -120,6 +120,7 @@
 #include "iterobject.h"
 #include "genobject.h"
 #include "descrobject.h"
+#include "genericaliasobject.h"
 #include "warnings.h"
 #include "weakrefobject.h"
 #include "structseq.h"

Include/genericaliasobject.h

@@ -0,0 +1,14 @@
+// Implementation of PEP 585: support list[int] etc.
+#ifndef Py_GENERICALIASOBJECT_H
+#define Py_GENERICALIASOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *);
+PyAPI_DATA(PyTypeObject) Py_GenericAliasType;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_GENERICALIASOBJECT_H */

Lib/_collections_abc.py

@@ -9,6 +9,8 @@ Unit tests are in test_collections.
 from abc import ABCMeta, abstractmethod
 import sys
 
+GenericAlias = type(list[int])
+
 __all__ = ["Awaitable", "Coroutine",
            "AsyncIterable", "AsyncIterator", "AsyncGenerator",
            "Hashable", "Iterable", "Iterator", "Generator", "Reversible",
@@ -110,6 +112,8 @@ class Awaitable(metaclass=ABCMeta):
             return _check_methods(C, "__await__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class Coroutine(Awaitable):
 
@@ -169,6 +173,8 @@ class AsyncIterable(metaclass=ABCMeta):
             return _check_methods(C, "__aiter__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class AsyncIterator(AsyncIterable):
 
@@ -255,6 +261,8 @@ class Iterable(metaclass=ABCMeta):
             return _check_methods(C, "__iter__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class Iterator(Iterable):
 
@@ -274,6 +282,7 @@ class Iterator(Iterable):
             return _check_methods(C, '__iter__', '__next__')
         return NotImplemented
 
+
 Iterator.register(bytes_iterator)
 Iterator.register(bytearray_iterator)
 #Iterator.register(callable_iterator)
@@ -353,6 +362,7 @@ class Generator(Iterator):
                                   'send', 'throw', 'close')
         return NotImplemented
 
+
 Generator.register(generator)
 
 
@@ -385,6 +395,9 @@ class Container(metaclass=ABCMeta):
             return _check_methods(C, "__contains__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
+
 class Collection(Sized, Iterable, Container):
 
     __slots__ = ()
@@ -395,6 +408,7 @@ class Collection(Sized, Iterable, Container):
             return _check_methods(C,  "__len__", "__iter__", "__contains__")
         return NotImplemented
 
+
 class Callable(metaclass=ABCMeta):
 
     __slots__ = ()
@@ -409,6 +423,8 @@ class Callable(metaclass=ABCMeta):
             return _check_methods(C, "__call__")
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 ### SETS ###
 
@@ -550,6 +566,7 @@ class Set(Collection):
             h = 590923713
         return h
 
+
 Set.register(frozenset)
 
 
@@ -632,6 +649,7 @@ class MutableSet(Set):
                 self.discard(value)
         return self
 
+
 MutableSet.register(set)
 
 
@@ -688,6 +706,7 @@ class Mapping(Collection):
 
     __reversed__ = None
 
+
 Mapping.register(mappingproxy)
 
 
@@ -704,6 +723,8 @@ class MappingView(Sized):
     def __repr__(self):
         return '{0.__class__.__name__}({0._mapping!r})'.format(self)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class KeysView(MappingView, Set):
 
@@ -719,6 +740,7 @@ class KeysView(MappingView, Set):
     def __iter__(self):
         yield from self._mapping
 
+
 KeysView.register(dict_keys)
 
 
@@ -743,6 +765,7 @@ class ItemsView(MappingView, Set):
         for key in self._mapping:
             yield (key, self._mapping[key])
 
+
 ItemsView.register(dict_items)
 
 
@@ -761,6 +784,7 @@ class ValuesView(MappingView, Collection):
         for key in self._mapping:
             yield self._mapping[key]
 
+
 ValuesView.register(dict_values)
 
 
@@ -847,6 +871,7 @@ class MutableMapping(Mapping):
             self[key] = default
         return default
 
+
 MutableMapping.register(dict)
 
 
@@ -914,6 +939,7 @@ class Sequence(Reversible, Collection):
         'S.count(value) -> integer -- return number of occurrences of value'
         return sum(1 for v in self if v is value or v == value)
 
+
 Sequence.register(tuple)
 Sequence.register(str)
 Sequence.register(range)
@@ -1000,5 +1026,6 @@ class MutableSequence(Sequence):
         self.extend(values)
         return self
 
+
 MutableSequence.register(list)
 MutableSequence.register(bytearray)  # Multiply inheriting, see ByteString

Lib/contextlib.py

@@ -4,7 +4,7 @@ import sys
 import _collections_abc
 from collections import deque
 from functools import wraps
-from types import MethodType
+from types import MethodType, GenericAlias
 
 __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
            "AbstractContextManager", "AbstractAsyncContextManager",
@@ -16,6 +16,8 @@ class AbstractContextManager(abc.ABC):
 
     """An abstract base class for context managers."""
 
+    __class_getitem__ = classmethod(GenericAlias)
+
     def __enter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -36,6 +38,8 @@ class AbstractAsyncContextManager(abc.ABC):
 
     """An abstract base class for asynchronous context managers."""
 
+    __class_getitem__ = classmethod(GenericAlias)
+
     async def __aenter__(self):
         """Return `self` upon entering the runtime context."""
         return self

Lib/os.py

@@ -28,6 +28,8 @@ import stat as st
 
 from _collections_abc import _check_methods
 
+GenericAlias = type(list[int])
+
 _names = sys.builtin_module_names
 
 # Note:  more names are added to __all__ later.
@@ -1074,8 +1076,7 @@ class PathLike(abc.ABC):
             return _check_methods(subclass, '__fspath__')
         return NotImplemented
 
-    def __class_getitem__(cls, type):
+    __class_getitem__ = classmethod(GenericAlias)
-        return cls
 
 
 if name == 'nt':

Lib/subprocess.py

@@ -52,6 +52,7 @@ import threading
 import warnings
 import contextlib
 from time import monotonic as _time
+import types
 
 try:
     import pwd
@@ -446,17 +447,7 @@ class CompletedProcess(object):
             args.append('stderr={!r}'.format(self.stderr))
         return "{}({})".format(type(self).__name__, ', '.join(args))
 
-    def __class_getitem__(cls, type):
+    __class_getitem__ = classmethod(types.GenericAlias)
-        """Provide minimal support for using this class as generic
-        (for example in type annotations).
-
-        See PEP 484 and PEP 560 for more details. For example,
-        `CompletedProcess[bytes]` is a valid expression at runtime
-        (type argument `bytes` indicates the type used for stdout).
-        Note, no type checking happens at runtime, but a static type
-        checker can be used.
-        """
-        return cls
 
 
     def check_returncode(self):
@@ -1000,16 +991,7 @@ class Popen(object):
             obj_repr = obj_repr[:76] + "...>"
         return obj_repr
 
-    def __class_getitem__(cls, type):
+    __class_getitem__ = classmethod(types.GenericAlias)
-        """Provide minimal support for using this class as generic
-        (for example in type annotations).
-
-        See PEP 484 and PEP 560 for more details. For example, `Popen[bytes]`
-        is a valid expression at runtime (type argument `bytes` indicates the
-        type used for stdout). Note, no type checking happens at runtime, but
-        a static type checker can be used.
-        """
-        return cls
 
     @property
     def universal_newlines(self):

Lib/tempfile.py

@@ -44,6 +44,7 @@ import shutil as _shutil
 import errno as _errno
 from random import Random as _Random
 import sys as _sys
+import types as _types
 import weakref as _weakref
 import _thread
 _allocate_lock = _thread.allocate_lock
@@ -643,17 +644,7 @@ class SpooledTemporaryFile:
                                    'encoding': encoding, 'newline': newline,
                                    'dir': dir, 'errors': errors}
 
-    def __class_getitem__(cls, type):
+    __class_getitem__ = classmethod(_types.GenericAlias)
-        """Provide minimal support for using this class as generic
-        (for example in type annotations).
-
-        See PEP 484 and PEP 560 for more details. For example,
-        `SpooledTemporaryFile[str]` is a valid expression at runtime (type
-        argument `str` indicates whether the file is open in bytes or text
-        mode). Note, no type checking happens at runtime, but a static type
-        checker can be used.
-        """
-        return cls
 
     def _check(self, file):
         if self._rolled: return

Lib/test/test_descrtut.py

@@ -167,6 +167,7 @@ You can get the information from the list type:
     >>> pprint.pprint(dir(list))    # like list.__dict__.keys(), but sorted
     ['__add__',
      '__class__',
+     '__class_getitem__',
      '__contains__',
      '__delattr__',
      '__delitem__',

Lib/test/test_doctest.py

@@ -665,7 +665,7 @@ plain ol' Python and is guaranteed to be available.
 
     >>> import builtins
     >>> tests = doctest.DocTestFinder().find(builtins)
-    >>> 800 < len(tests) < 820 # approximate number of objects with docstrings
+    >>> 810 < len(tests) < 830 # approximate number of objects with docstrings
     True
     >>> real_tests = [t for t in tests if len(t.examples) > 0]
     >>> len(real_tests) # objects that actually have doctests

Lib/test/test_genericalias.py

@@ -0,0 +1,214 @@
+"""Tests for C-implemented GenericAlias."""
+
+import unittest
+import pickle
+from collections import (
+    defaultdict, deque, OrderedDict, Counter, UserDict, UserList
+)
+from collections.abc import *
+from contextlib import AbstractContextManager, AbstractAsyncContextManager
+from re import Pattern, Match
+from types import GenericAlias, MappingProxyType
+import typing
+
+from typing import TypeVar
+T = TypeVar('T')
+
+class BaseTest(unittest.TestCase):
+    """Test basics."""
+
+    def test_subscriptable(self):
+        for t in (type, tuple, list, dict, set, frozenset,
+                  defaultdict, deque,
+                  OrderedDict, Counter, UserDict, UserList,
+                  Pattern, Match,
+                  AbstractContextManager, AbstractAsyncContextManager,
+                  Awaitable, Coroutine,
+                  AsyncIterable, AsyncIterator,
+                  AsyncGenerator, Generator,
+                  Iterable, Iterator,
+                  Reversible,
+                  Container, Collection,
+                  Callable,
+                  Set, MutableSet,
+                  Mapping, MutableMapping, MappingView,
+                  KeysView, ItemsView, ValuesView,
+                  Sequence, MutableSequence,
+                  MappingProxyType,
+                  ):
+            tname = t.__name__
+            with self.subTest(f"Testing {tname}"):
+                alias = t[int]
+                self.assertIs(alias.__origin__, t)
+                self.assertEqual(alias.__args__, (int,))
+                self.assertEqual(alias.__parameters__, ())
+
+    def test_unsubscriptable(self):
+        for t in int, str, float, Sized, Hashable:
+            tname = t.__name__
+            with self.subTest(f"Testing {tname}"):
+                with self.assertRaises(TypeError):
+                    t[int]
+
+    def test_instantiate(self):
+        for t in tuple, list, dict, set, frozenset, defaultdict, deque:
+            tname = t.__name__
+            with self.subTest(f"Testing {tname}"):
+                alias = t[int]
+                self.assertEqual(alias(), t())
+                if t is dict:
+                    self.assertEqual(alias(iter([('a', 1), ('b', 2)])), dict(a=1, b=2))
+                    self.assertEqual(alias(a=1, b=2), dict(a=1, b=2))
+                elif t is defaultdict:
+                    def default():
+                        return 'value'
+                    a = alias(default)
+                    d = defaultdict(default)
+                    self.assertEqual(a['test'], d['test'])
+                else:
+                    self.assertEqual(alias(iter((1, 2, 3))), t((1, 2, 3)))
+
+    def test_unbound_methods(self):
+        t = list[int]
+        a = t()
+        t.append(a, 'foo')
+        self.assertEqual(a, ['foo'])
+        x = t.__getitem__(a, 0)
+        self.assertEqual(x, 'foo')
+        self.assertEqual(t.__len__(a), 1)
+
+    def test_subclassing(self):
+        class C(list[int]):
+            pass
+        self.assertEqual(C.__bases__, (list,))
+        self.assertEqual(C.__class__, type)
+
+    def test_class_methods(self):
+        t = dict[int, None]
+        self.assertEqual(dict.fromkeys(range(2)), {0: None, 1: None})  # This works
+        self.assertEqual(t.fromkeys(range(2)), {0: None, 1: None})  # Should be equivalent
+
+    def test_no_chaining(self):
+        t = list[int]
+        with self.assertRaises(TypeError):
+            t[int]
+
+    def test_generic_subclass(self):
+        class MyList(list):
+            pass
+        t = MyList[int]
+        self.assertIs(t.__origin__, MyList)
+        self.assertEqual(t.__args__, (int,))
+        self.assertEqual(t.__parameters__, ())
+
+    def test_repr(self):
+        class MyList(list):
+            pass
+        self.assertEqual(repr(list[str]), 'list[str]')
+        self.assertEqual(repr(list[()]), 'list[()]')
+        self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]')
+        self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr.<locals>.MyList[int]'))
+        self.assertEqual(repr(list[str]()), '[]')  # instances should keep their normal repr
+
+    def test_exposed_type(self):
+        import types
+        a = types.GenericAlias(list, int)
+        self.assertEqual(str(a), 'list[int]')
+        self.assertIs(a.__origin__, list)
+        self.assertEqual(a.__args__, (int,))
+        self.assertEqual(a.__parameters__, ())
+
+    def test_parameters(self):
+        from typing import TypeVar
+        T = TypeVar('T')
+        K = TypeVar('K')
+        V = TypeVar('V')
+        D0 = dict[str, int]
+        self.assertEqual(D0.__args__, (str, int))
+        self.assertEqual(D0.__parameters__, ())
+        D1a = dict[str, V]
+        self.assertEqual(D1a.__args__, (str, V))
+        self.assertEqual(D1a.__parameters__, (V,))
+        D1b = dict[K, int]
+        self.assertEqual(D1b.__args__, (K, int))
+        self.assertEqual(D1b.__parameters__, (K,))
+        D2a = dict[K, V]
+        self.assertEqual(D2a.__args__, (K, V))
+        self.assertEqual(D2a.__parameters__, (K, V))
+        D2b = dict[T, T]
+        self.assertEqual(D2b.__args__, (T, T))
+        self.assertEqual(D2b.__parameters__, (T,))
+        L0 = list[str]
+        self.assertEqual(L0.__args__, (str,))
+        self.assertEqual(L0.__parameters__, ())
+        L1 = list[T]
+        self.assertEqual(L1.__args__, (T,))
+        self.assertEqual(L1.__parameters__, (T,))
+
+    def test_parameter_chaining(self):
+        from typing import TypeVar
+        T = TypeVar('T')
+        self.assertEqual(list[T][int], list[int])
+        self.assertEqual(dict[str, T][int], dict[str, int])
+        self.assertEqual(dict[T, int][str], dict[str, int])
+        self.assertEqual(dict[T, T][int], dict[int, int])
+        with self.assertRaises(TypeError):
+            list[int][int]
+            dict[T, int][str, int]
+            dict[str, T][str, int]
+            dict[T, T][str, int]
+
+    def test_equality(self):
+        self.assertEqual(list[int], list[int])
+        self.assertEqual(dict[str, int], dict[str, int])
+        self.assertNotEqual(dict[str, int], dict[str, str])
+        self.assertNotEqual(list, list[int])
+        self.assertNotEqual(list[int], list)
+
+    def test_isinstance(self):
+        self.assertTrue(isinstance([], list))
+        with self.assertRaises(TypeError):
+            isinstance([], list[str])
+
+    def test_issubclass(self):
+        class L(list): ...
+        self.assertTrue(issubclass(L, list))
+        with self.assertRaises(TypeError):
+            issubclass(L, list[str])
+
+    def test_type_generic(self):
+        t = type[int]
+        Test = t('Test', (), {})
+        self.assertTrue(isinstance(Test, type))
+        test = Test()
+        self.assertEqual(t(test), Test)
+        self.assertEqual(t(0), int)
+
+    def test_type_subclass_generic(self):
+        class MyType(type):
+            pass
+        with self.assertRaises(TypeError):
+            MyType[int]
+
+    def test_pickle(self):
+        alias = GenericAlias(list, T)
+        s = pickle.dumps(alias)
+        loaded = pickle.loads(s)
+        self.assertEqual(alias.__origin__, loaded.__origin__)
+        self.assertEqual(alias.__args__, loaded.__args__)
+        self.assertEqual(alias.__parameters__, loaded.__parameters__)
+
+    def test_union(self):
+        a = typing.Union[list[int], list[str]]
+        self.assertEqual(a.__args__, (list[int], list[str]))
+        self.assertEqual(a.__parameters__, ())
+
+    def test_union_generic(self):
+        T = typing.TypeVar('T')
+        a = typing.Union[list[T], tuple[T, ...]]
+        self.assertEqual(a.__args__, (list[T], tuple[T, ...]))
+        self.assertEqual(a.__parameters__, (T,))
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_os.py

@@ -25,6 +25,7 @@ import sysconfig
 import tempfile
 import threading
 import time
+import types
 import unittest
 import uuid
 import warnings
@@ -4191,7 +4192,7 @@ class TestPEP519(unittest.TestCase):
         self.assertTrue(issubclass(FakePath, os.PathLike))
 
     def test_pathlike_class_getitem(self):
-        self.assertIs(os.PathLike[bytes], os.PathLike)
+        self.assertIsInstance(os.PathLike[bytes], types.GenericAlias)
 
 
 class TimesTests(unittest.TestCase):

Lib/test/test_subprocess.py

@@ -11,6 +11,7 @@ import errno
 import tempfile
 import time
 import traceback
+import types
 import selectors
 import sysconfig
 import select
@@ -1435,8 +1436,8 @@ class ProcessTestCase(BaseTestCase):
         self.assertEqual(c.exception.filename, '/some/nonexistent/directory')
 
     def test_class_getitems(self):
-        self.assertIs(subprocess.Popen[bytes], subprocess.Popen)
+        self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias)
-        self.assertIs(subprocess.CompletedProcess[str], subprocess.CompletedProcess)
+        self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias)
 
 class RunFuncTestCase(BaseTestCase):
     def run_python(self, code, **kwargs):

Lib/test/test_tempfile.py

@@ -10,6 +10,7 @@ import re
 import warnings
 import contextlib
 import stat
+import types
 import weakref
 from unittest import mock
 
@@ -1222,8 +1223,8 @@ class TestSpooledTemporaryFile(BaseTestCase):
         self.assertEqual(os.fstat(f.fileno()).st_size, 20)
 
     def test_class_getitem(self):
-        self.assertIs(tempfile.SpooledTemporaryFile[bytes],
+        self.assertIsInstance(tempfile.SpooledTemporaryFile[bytes],
-                      tempfile.SpooledTemporaryFile)
+                      types.GenericAlias)
 
 if tempfile.NamedTemporaryFile is not tempfile.TemporaryFile:
 

Lib/test/test_types.py

@@ -622,6 +622,7 @@ class MappingProxyTests(unittest.TestCase):
         self.assertEqual(attrs, {
              '__contains__',
              '__getitem__',
+             '__class_getitem__',
              '__ior__',
              '__iter__',
              '__len__',

Lib/test/test_typing.py

@@ -1775,10 +1775,11 @@ class GenericTests(BaseTestCase):
         self.assertEqual(T1[int, T].__origin__, T1)
 
         self.assertEqual(T2.__parameters__, (T,))
-        with self.assertRaises(TypeError):
+        # These don't work because of tuple.__class_item__
-            T1[int]
+        ## with self.assertRaises(TypeError):
-        with self.assertRaises(TypeError):
+        ##     T1[int]
-            T2[int, str]
+        ## with self.assertRaises(TypeError):
+        ##     T2[int, str]
 
         self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]')
         self.assertEqual(C2.__parameters__, ())
@@ -1817,22 +1818,22 @@ class GenericTests(BaseTestCase):
         self.clear_caches()
         class MyTup(Tuple[T, T]): ...
         self.assertIs(MyTup[int]().__class__, MyTup)
-        self.assertIs(MyTup[int]().__orig_class__, MyTup[int])
+        self.assertEqual(MyTup[int]().__orig_class__, MyTup[int])
         class MyCall(Callable[..., T]):
             def __call__(self): return None
         self.assertIs(MyCall[T]().__class__, MyCall)
-        self.assertIs(MyCall[T]().__orig_class__, MyCall[T])
+        self.assertEqual(MyCall[T]().__orig_class__, MyCall[T])
         class MyDict(typing.Dict[T, T]): ...
         self.assertIs(MyDict[int]().__class__, MyDict)
-        self.assertIs(MyDict[int]().__orig_class__, MyDict[int])
+        self.assertEqual(MyDict[int]().__orig_class__, MyDict[int])
         class MyDef(typing.DefaultDict[str, T]): ...
         self.assertIs(MyDef[int]().__class__, MyDef)
-        self.assertIs(MyDef[int]().__orig_class__, MyDef[int])
+        self.assertEqual(MyDef[int]().__orig_class__, MyDef[int])
         # ChainMap was added in 3.3
         if sys.version_info >= (3, 3):
             class MyChain(typing.ChainMap[str, T]): ...
             self.assertIs(MyChain[int]().__class__, MyChain)
-            self.assertIs(MyChain[int]().__orig_class__, MyChain[int])
+            self.assertEqual(MyChain[int]().__orig_class__, MyChain[int])
 
     def test_all_repr_eq_any(self):
         objs = (getattr(typing, el) for el in typing.__all__)

Lib/types.py

@@ -293,4 +293,7 @@ def coroutine(func):
     return wrapped
 
 
+GenericAlias = type(list[int])
+
+
 __all__ = [n for n in globals() if n[:1] != '_']

Lib/typing.py

@@ -26,7 +26,7 @@ import operator
 import re as stdlib_re  # Avoid confusion with the re we export.
 import sys
 import types
-from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType
+from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias
 
 # Please keep __all__ alphabetized within each category.
 __all__ = [
@@ -180,7 +180,8 @@ def _collect_type_vars(types):
     for t in types:
         if isinstance(t, TypeVar) and t not in tvars:
             tvars.append(t)
-        if isinstance(t, _GenericAlias) and not t._special:
+        if ((isinstance(t, _GenericAlias) and not t._special)
+                or isinstance(t, GenericAlias)):
             tvars.extend([t for t in t.__parameters__ if t not in tvars])
     return tuple(tvars)
 
@@ -947,7 +948,7 @@ _TYPING_INTERNALS = ['__parameters__', '__orig_bases__',  '__orig_class__',
 
 _SPECIAL_NAMES = ['__abstractmethods__', '__annotations__', '__dict__', '__doc__',
                   '__init__', '__module__', '__new__', '__slots__',
-                  '__subclasshook__', '__weakref__']
+                  '__subclasshook__', '__weakref__', '__class_getitem__']
 
 # These special attributes will be not collected as protocol members.
 EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS + _SPECIAL_NAMES + ['_MutableMapping__marker']

Makefile.pre.in

@@ -393,6 +393,7 @@ OBJECT_OBJS=	\
 		Objects/descrobject.o \
 		Objects/enumobject.o \
 		Objects/exceptions.o \
+		Objects/genericaliasobject.o \
 		Objects/genobject.o \
 		Objects/fileobject.o \
 		Objects/floatobject.o \

Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst

@@ -0,0 +1 @@
+Implement PEP 585. This supports list[int], tuple[str, ...] etc.

Modules/_collectionsmodule.c

@@ -1609,6 +1609,8 @@ static PyMethodDef deque_methods[] = {
         METH_FASTCALL,            rotate_doc},
     {"__sizeof__",              (PyCFunction)deque_sizeof,
         METH_NOARGS,             sizeof_doc},
+    {"__class_getitem__",       (PyCFunction)Py_GenericAlias,
+        METH_O|METH_CLASS,       PyDoc_STR("See PEP 585")},
     {NULL,              NULL}   /* sentinel */
 };
 
@@ -2074,6 +2076,8 @@ static PyMethodDef defdict_methods[] = {
      defdict_copy_doc},
     {"__reduce__", (PyCFunction)defdict_reduce, METH_NOARGS,
      reduce_doc},
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS,
+     PyDoc_STR("See PEP 585")},
     {NULL}
 };
 

Modules/_sre.c

@@ -2568,6 +2568,8 @@ static PyMethodDef pattern_methods[] = {
     _SRE_SRE_PATTERN_SCANNER_METHODDEF
     _SRE_SRE_PATTERN___COPY___METHODDEF
     _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS,
+     PyDoc_STR("See PEP 585")},
     {NULL, NULL}
 };
 
@@ -2638,6 +2640,8 @@ static PyMethodDef match_methods[] = {
     _SRE_SRE_MATCH_EXPAND_METHODDEF
     _SRE_SRE_MATCH___COPY___METHODDEF
     _SRE_SRE_MATCH___DEEPCOPY___METHODDEF
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS,
+     PyDoc_STR("See PEP 585")},
     {NULL, NULL}
 };
 

Objects/abstract.c

@@ -176,6 +176,12 @@ PyObject_GetItem(PyObject *o, PyObject *key)
     if (PyType_Check(o)) {
         PyObject *meth, *result;
         _Py_IDENTIFIER(__class_getitem__);
+        
+        // Special case type[int], but disallow other types so str[int] fails
+        if ((PyTypeObject*)o == &PyType_Type) {
+            return Py_GenericAlias(o, key);
+        }
+
         if (_PyObject_LookupAttrId(o, &PyId___class_getitem__, &meth) < 0) {
             return NULL;
         }

Objects/descrobject.c

@@ -1092,6 +1092,8 @@ static PyMethodDef mappingproxy_methods[] = {
      PyDoc_STR("D.items() -> list of D's (key, value) pairs, as 2-tuples")},
     {"copy",      (PyCFunction)mappingproxy_copy,       METH_NOARGS,
      PyDoc_STR("D.copy() -> a shallow copy of D")},
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS,
+     PyDoc_STR("See PEP 585")},
     {0}
 };
 

Objects/dictobject.c

@@ -3252,6 +3252,7 @@ static PyMethodDef mapp_methods[] = {
     {"copy",            (PyCFunction)dict_copy,         METH_NOARGS,
      copy__doc__},
     DICT___REVERSED___METHODDEF
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")},
     {NULL,              NULL}   /* sentinel */
 };
 

Objects/genericaliasobject.c

@@ -0,0 +1,507 @@
+// types.GenericAlias -- used to represent e.g. list[int].
+
+#include "Python.h"
+#include "pycore_object.h"
+#include "structmember.h"
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *origin;
+    PyObject *args;
+    PyObject *parameters;
+} gaobject;
+
+static void
+ga_dealloc(PyObject *self)
+{
+    gaobject *alias = (gaobject *)self;
+
+    _PyObject_GC_UNTRACK(self);
+    Py_XDECREF(alias->origin);
+    Py_XDECREF(alias->args);
+    Py_XDECREF(alias->parameters);
+    self->ob_type->tp_free(self);
+}
+
+static int
+ga_traverse(PyObject *self, visitproc visit, void *arg)
+{
+    gaobject *alias = (gaobject *)self;
+    Py_VISIT(alias->origin);
+    Py_VISIT(alias->args);
+    Py_VISIT(alias->parameters);
+    return 0;
+}
+
+static int
+ga_repr_item(_PyUnicodeWriter *writer, PyObject *p)
+{
+    _Py_IDENTIFIER(__module__);
+    _Py_IDENTIFIER(__qualname__);
+    _Py_IDENTIFIER(__origin__);
+    _Py_IDENTIFIER(__args__);
+    PyObject *qualname = NULL;
+    PyObject *module = NULL;
+    PyObject *r = NULL;
+    PyObject *tmp;
+    int err;
+
+    if (p == Py_Ellipsis) {
+        // The Ellipsis object
+        r = PyUnicode_FromString("...");
+        goto done;
+    }
+
+    if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) {
+        goto done;
+    }
+    if (tmp != NULL) {
+        Py_DECREF(tmp);
+        if (_PyObject_LookupAttrId(p, &PyId___args__, &tmp) < 0) {
+            goto done;
+        }
+        if (tmp != NULL) {
+            Py_DECREF(tmp);
+            // It looks like a GenericAlias
+            goto use_repr;
+        }
+    }
+
+    if (_PyObject_LookupAttrId(p, &PyId___qualname__, &qualname) < 0) {
+        goto done;
+    }
+    if (qualname == NULL) {
+        goto use_repr;
+    }
+    if (_PyObject_LookupAttrId(p, &PyId___module__, &module) < 0) {
+        goto done;
+    }
+    if (module == NULL || module == Py_None) {
+        goto use_repr;
+    }
+
+    // Looks like a class
+    if (PyUnicode_Check(module) &&
+        _PyUnicode_EqualToASCIIString(module, "builtins"))
+    {
+        // builtins don't need a module name
+        r = PyObject_Str(qualname);
+        goto done;
+    }
+    else {
+        r = PyUnicode_FromFormat("%S.%S", module, qualname);
+        goto done;
+    }
+
+use_repr:
+    r = PyObject_Repr(p);
+
+done:
+    Py_XDECREF(qualname);
+    Py_XDECREF(module);
+    if (r == NULL) {
+        // error if any of the above PyObject_Repr/PyUnicode_From* fail
+        err = -1;
+    }
+    else {
+        err = _PyUnicodeWriter_WriteStr(writer, r);
+        Py_DECREF(r);
+    }
+    return err;
+}
+
+static PyObject *
+ga_repr(PyObject *self)
+{
+    gaobject *alias = (gaobject *)self;
+    Py_ssize_t len = PyTuple_GET_SIZE(alias->args);
+
+    _PyUnicodeWriter writer;
+    _PyUnicodeWriter_Init(&writer);
+    
+    if (ga_repr_item(&writer, alias->origin) < 0) {
+        goto error;
+    }
+    if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) {
+        goto error;
+    }
+    for (Py_ssize_t i = 0; i < len; i++) {
+        if (i > 0) {
+            if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) {
+                goto error;
+            }
+        }
+        PyObject *p = PyTuple_GET_ITEM(alias->args, i);
+        if (ga_repr_item(&writer, p) < 0) {
+            goto error;
+        }
+    }
+    if (len == 0) {
+        // for something like tuple[()] we should print a "()"
+        if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) {
+            goto error;
+        }
+    }
+    if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) {
+        goto error;
+    }
+    return _PyUnicodeWriter_Finish(&writer);
+error:
+    _PyUnicodeWriter_Dealloc(&writer);
+    return NULL;
+}
+
+// isinstance(obj, TypeVar) without importing typing.py.
+// Returns -1 for errors.
+static int
+is_typevar(PyObject *obj)
+{
+    PyTypeObject *type = Py_TYPE(obj);
+    if (strcmp(type->tp_name, "TypeVar") != 0) {
+        return 0;
+    }
+    PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__");
+    if (module == NULL) {
+        return -1;
+    }
+    int res = PyUnicode_Check(module)
+        && _PyUnicode_EqualToASCIIString(module, "typing");
+    Py_DECREF(module);
+    return res;
+}
+
+// Index of item in self[:len], or -1 if not found (self is a tuple)
+static Py_ssize_t
+tuple_index(PyObject *self, Py_ssize_t len, PyObject *item)
+{
+    for (Py_ssize_t i = 0; i < len; i++) {
+        if (PyTuple_GET_ITEM(self, i) == item) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+// tuple(t for t in args if isinstance(t, TypeVar))
+static PyObject *
+make_parameters(PyObject *args)
+{
+    Py_ssize_t len = PyTuple_GET_SIZE(args);
+    PyObject *parameters = PyTuple_New(len);
+    if (parameters == NULL)
+        return NULL;
+    Py_ssize_t iparam = 0;
+    for (Py_ssize_t iarg = 0; iarg < len; iarg++) {
+        PyObject *t = PyTuple_GET_ITEM(args, iarg);
+        int typevar = is_typevar(t);
+        if (typevar < 0) {
+            Py_XDECREF(parameters);
+            return NULL;
+        }
+        if (typevar) {
+            if (tuple_index(parameters, iparam, t) < 0) {
+                Py_INCREF(t);
+                PyTuple_SET_ITEM(parameters, iparam, t);
+                iparam++;
+            }
+        }
+    }
+    if (iparam < len) {
+        if (_PyTuple_Resize(&parameters, iparam) < 0) {
+            Py_XDECREF(parameters);
+            return NULL;
+        }
+    }
+    return parameters;
+}
+
+static PyObject *
+ga_getitem(PyObject *self, PyObject *item)
+{
+    gaobject *alias = (gaobject *)self;
+    // do a lookup for __parameters__ so it gets populated (if not already)
+    if (alias->parameters == NULL) {
+        alias->parameters = make_parameters(alias->args);
+        if (alias->parameters == NULL) {
+            return NULL;
+        }
+    }
+    Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters);
+    if (nparams == 0) {
+        return PyErr_Format(PyExc_TypeError,
+                            "There are no type variables left in %R",
+                            self);
+    }
+    int is_tuple = PyTuple_Check(item);
+    Py_ssize_t nitem = is_tuple ? PyTuple_GET_SIZE(item) : 1;
+    if (nitem != nparams) {
+        return PyErr_Format(PyExc_TypeError,
+                            "Too %s arguments for %R",
+                            nitem > nparams ? "many" : "few",
+                            self);
+    }
+    Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args);
+    PyObject *newargs = PyTuple_New(nargs);
+    if (newargs == NULL)
+        return NULL;
+    for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) {
+        PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg);
+        int typevar = is_typevar(arg);
+        if (typevar < 0) {
+            Py_DECREF(newargs);
+            return NULL;
+        }
+        if (typevar) {
+            Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg);
+            assert(iparam >= 0);
+            if (is_tuple) {
+                arg = PyTuple_GET_ITEM(item, iparam);
+            }
+            else {
+                assert(iparam == 0);
+                arg = item;
+            }
+        }
+        Py_INCREF(arg);
+        PyTuple_SET_ITEM(newargs, iarg, arg);
+    }
+    PyObject *res = Py_GenericAlias(alias->origin, newargs);
+    Py_DECREF(newargs);
+    return res;
+}
+
+static PyMappingMethods ga_as_mapping = {
+    .mp_subscript = ga_getitem,
+};
+
+static Py_hash_t
+ga_hash(PyObject *self)
+{
+    gaobject *alias = (gaobject *)self;
+    // TODO: Hash in the hash for the origin
+    Py_hash_t h0 = PyObject_Hash(alias->origin);
+    if (h0 == -1) {
+        return -1;
+    }
+    Py_hash_t h1 = PyObject_Hash(alias->args);
+    if (h1 == -1) {
+        return -1;
+    }
+    return h0 ^ h1;
+}
+
+static PyObject *
+ga_call(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    gaobject *alias = (gaobject *)self;
+    PyObject *obj = PyObject_Call(alias->origin, args, kwds);
+    if (obj != NULL) {
+        if (PyObject_SetAttrString(obj, "__orig_class__", self) < 0) {
+            if (!PyErr_ExceptionMatches(PyExc_AttributeError) &&
+                !PyErr_ExceptionMatches(PyExc_TypeError))
+            {
+                Py_DECREF(obj);
+                return NULL;
+            }
+            PyErr_Clear();
+        }
+    }
+    return obj;
+}
+
+static const char* const attr_exceptions[] = {
+    "__origin__",
+    "__args__",
+    "__parameters__",
+    "__mro_entries__",
+    "__reduce_ex__",  // needed so we don't look up object.__reduce_ex__
+    "__reduce__",
+    NULL,
+};
+
+static PyObject *
+ga_getattro(PyObject *self, PyObject *name)
+{
+    gaobject *alias = (gaobject *)self;
+    if (PyUnicode_Check(name)) {
+        for (const char * const *p = attr_exceptions; ; p++) {
+            if (*p == NULL) {
+                return PyObject_GetAttr(alias->origin, name);
+            }
+            if (_PyUnicode_EqualToASCIIString(name, *p)) {
+                break;
+            }
+        }
+    }
+    return PyObject_GenericGetAttr(self, name);
+}
+
+static PyObject *
+ga_richcompare(PyObject *a, PyObject *b, int op)
+{
+    if (Py_TYPE(a) != &Py_GenericAliasType ||
+        Py_TYPE(b) != &Py_GenericAliasType ||
+        (op != Py_EQ && op != Py_NE))
+    {
+        Py_RETURN_NOTIMPLEMENTED;
+    }
+
+    if (op == Py_NE) {
+        PyObject *eq = ga_richcompare(a, b, Py_EQ);
+        if (eq == NULL)
+            return NULL;
+        Py_DECREF(eq);
+        if (eq == Py_True) {
+            Py_RETURN_FALSE;
+        }
+        else {
+            Py_RETURN_TRUE;
+        }
+    }
+
+    gaobject *aa = (gaobject *)a;
+    gaobject *bb = (gaobject *)b;
+    int eq = PyObject_RichCompareBool(aa->origin, bb->origin, Py_EQ);
+    if (eq < 0) {
+        return NULL;
+    }
+    if (!eq) {
+        Py_RETURN_FALSE;
+    }
+    return PyObject_RichCompare(aa->args, bb->args, Py_EQ);
+}
+
+static PyObject *
+ga_mro_entries(PyObject *self, PyObject *args)
+{
+    gaobject *alias = (gaobject *)self;
+    return PyTuple_Pack(1, alias->origin);
+}
+
+static PyObject *
+ga_instancecheck(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyErr_SetString(PyExc_TypeError,
+                    "isinstance() argument 2 cannot be a parameterized generic");
+    return NULL;
+}
+
+static PyObject *
+ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyErr_SetString(PyExc_TypeError,
+                    "issubclass() argument 2 cannot be a parameterized generic");
+    return NULL;
+}
+
+static PyObject *
+ga_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    gaobject *alias = (gaobject *)self;
+    return Py_BuildValue("O(OO)", Py_TYPE(alias),
+                         alias->origin, alias->args);
+}
+
+static PyMethodDef ga_methods[] = {
+    {"__mro_entries__", ga_mro_entries, METH_O},
+    {"__instancecheck__", ga_instancecheck, METH_O},
+    {"__subclasscheck__", ga_subclasscheck, METH_O},
+    {"__reduce__", ga_reduce, METH_NOARGS},
+    {0}
+};
+
+static PyMemberDef ga_members[] = {
+    {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY},
+    {"__args__", T_OBJECT, offsetof(gaobject, args), READONLY},
+    {0}
+};
+
+static PyObject *
+ga_parameters(PyObject *self, void *unused)
+{
+    gaobject *alias = (gaobject *)self;
+    if (alias->parameters == NULL) {
+        alias->parameters = make_parameters(alias->args);
+        if (alias->parameters == NULL) {
+            return NULL;
+        }
+    }
+    Py_INCREF(alias->parameters);
+    return alias->parameters;
+}
+
+static PyGetSetDef ga_properties[] = {
+    {"__parameters__", ga_parameters, (setter)NULL, "Type variables in the GenericAlias.", NULL},
+    {0}
+};
+
+static PyObject *
+ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) {
+        PyErr_SetString(PyExc_TypeError, "GenericAlias does not support keyword arguments");
+        return NULL;
+    }
+    if (PyTuple_GET_SIZE(args) != 2) {
+        PyErr_SetString(PyExc_TypeError, "GenericAlias expects 2 positional arguments");
+        return NULL;
+    }
+    PyObject *origin = PyTuple_GET_ITEM(args, 0);
+    PyObject *arguments = PyTuple_GET_ITEM(args, 1);
+    return Py_GenericAlias(origin, arguments);
+}
+
+// TODO:
+// - argument clinic?
+// - __doc__?
+// - cache?
+PyTypeObject Py_GenericAliasType = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    .tp_name = "types.GenericAlias",
+    .tp_doc = "Represent a PEP 585 generic type\n"
+              "\n"
+              "E.g. for t = list[int], t.origin is list and t.args is (int,).",
+    .tp_basicsize = sizeof(gaobject),
+    .tp_dealloc = ga_dealloc,
+    .tp_repr = ga_repr,
+    .tp_as_mapping = &ga_as_mapping,
+    .tp_hash = ga_hash,
+    .tp_call = ga_call,
+    .tp_getattro = ga_getattro,
+    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
+    .tp_traverse = ga_traverse,
+    .tp_richcompare = ga_richcompare,
+    .tp_methods = ga_methods,
+    .tp_members = ga_members,
+    .tp_alloc = PyType_GenericAlloc,
+    .tp_new = ga_new,
+    .tp_free = PyObject_GC_Del,
+    .tp_getset = ga_properties,
+};
+
+PyObject *
+Py_GenericAlias(PyObject *origin, PyObject *args)
+{
+    if (!PyTuple_Check(args)) {
+        args = PyTuple_Pack(1, args);
+        if (args == NULL) {
+            return NULL;
+        }
+    }
+    else {
+        Py_INCREF(args);
+    }
+
+    gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType);
+    if (alias == NULL) {
+        Py_DECREF(args);
+        return NULL;
+    }
+
+    Py_INCREF(origin);
+    alias->origin = origin;
+    alias->args = args;
+    alias->parameters = NULL;
+    _PyObject_GC_TRACK(alias);
+    return (PyObject *)alias;
+}

Objects/listobject.c

@@ -2780,6 +2780,7 @@ static PyMethodDef list_methods[] = {
     LIST_COUNT_METHODDEF
     LIST_REVERSE_METHODDEF
     LIST_SORT_METHODDEF
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")},
     {NULL,              NULL}           /* sentinel */
 };
 

Objects/setobject.c

@@ -2132,6 +2132,7 @@ static PyMethodDef set_methods[] = {
      union_doc},
     {"update",          (PyCFunction)set_update,        METH_VARARGS,
      update_doc},
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")},
     {NULL,              NULL}   /* sentinel */
 };
 
@@ -2244,6 +2245,7 @@ static PyMethodDef frozenset_methods[] = {
      symmetric_difference_doc},
     {"union",           (PyCFunction)set_union,         METH_VARARGS,
      union_doc},
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")},
     {NULL,              NULL}   /* sentinel */
 };
 

Objects/tupleobject.c

@@ -832,6 +832,7 @@ static PyMethodDef tuple_methods[] = {
     TUPLE___GETNEWARGS___METHODDEF
     TUPLE_INDEX_METHODDEF
     TUPLE_COUNT_METHODDEF
+    {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")},
     {NULL,              NULL}           /* sentinel */
 };
 

PC/python3.def

@@ -734,6 +734,8 @@ EXPORTS
   Py_FileSystemDefaultEncoding=python39.Py_FileSystemDefaultEncoding DATA
   Py_Finalize=python39.Py_Finalize
   Py_FinalizeEx=python39.Py_FinalizeEx
+  Py_GenericAlias=python39.Py_GenericAlias
+  Py_GenericAliasType=python39.Py_GenericAliasType
   Py_GetBuildInfo=python39.Py_GetBuildInfo
   Py_GetCompiler=python39.Py_GetCompiler
   Py_GetCopyright=python39.Py_GetCopyright

PCbuild/pythoncore.vcxproj

@@ -383,6 +383,7 @@
     <ClCompile Include="..\Objects\floatobject.c" />
     <ClCompile Include="..\Objects\frameobject.c" />
     <ClCompile Include="..\Objects\funcobject.c" />
+    <ClCompile Include="..\Objects\genericaliasobject.c" />
     <ClCompile Include="..\Objects\genobject.c" />
     <ClCompile Include="..\Objects\interpreteridobject.c" />
     <ClCompile Include="..\Objects\iterobject.c" />