From cf5b109dbb39bcff1bc5b5d22036866d11de971b Mon Sep 17 00:00:00 2001 From: Jakub Stasiak Date: Wed, 5 Feb 2020 02:10:19 +0100 Subject: [PATCH] bpo-39491: Merge PEP 593 (typing.Annotated) support (#18260) * bpo-39491: Merge PEP 593 (typing.Annotated) support PEP 593 has been accepted some time ago. I got a green light for merging this from Till, so I went ahead and combined the code contributed to typing_extensions[1] and the documentation from the PEP 593 text[2]. My changes were limited to: * removing code designed for typing_extensions to run on older Python versions * removing some irrelevant parts of the PEP text when copying it over as documentation and otherwise changing few small bits to better serve the purpose * changing the get_type_hints signature to match reality (parameter names) I wasn't entirely sure how to go about crediting the authors but I used my best judgment, let me know if something needs changing in this regard. [1] https://github.com/python/typing/blob/8280de241fd8c8afe727c7860254b753e383b360/typing_extensions/src_py3/typing_extensions.py [2] https://github.com/python/peps/blob/17710b879882454d55f82c2d44596e8e9f8e4bff/pep-0593.rst --- Doc/library/typing.rst | 102 +++++++- Doc/whatsnew/3.9.rst | 8 + Lib/test/test_typing.py | 234 ++++++++++++++++++ Lib/typing.py | 126 +++++++++- .../2020-01-29-22-47-12.bpo-39491.tdl17b.rst | 3 + 5 files changed, 467 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-01-29-22-47-12.bpo-39491.tdl17b.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 323dac20822..d3bab94c6dd 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1028,7 +1028,7 @@ The module defines the following classes, functions and decorators: runtime we intentionally don't check anything (we want this to be as fast as possible). -.. function:: get_type_hints(obj[, globals[, locals]]) +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) Return a dictionary containing type hints for a function, method, module or class object. @@ -1041,6 +1041,22 @@ The module defines the following classes, functions and decorators: a dictionary constructed by merging all the ``__annotations__`` along ``C.__mro__`` in reverse order. + The function recursively replaces all ``Annotated[T, ...]`` with ``T``, + unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for + more information). For example:: + + class Student(NamedTuple): + name: Annotated[str, 'some marker'] + + get_type_hints(Student) == {'name': str} + get_type_hints(Student, include_extras=False) == {'name': str} + get_type_hints(Student, include_extras=True) == { + 'name': Annotated[str, 'some marker'] + } + + .. versionchanged:: 3.9 + Added ``include_extras`` parameter as part of :pep:`593`. + .. function:: get_origin(tp) .. function:: get_args(tp) @@ -1372,3 +1388,87 @@ The module defines the following classes, functions and decorators: evaluated, so the second annotation does not need to be enclosed in quotes. .. versionadded:: 3.5.2 + +.. data:: Annotated + + A type, introduced in :pep:`593` (``Flexible function and variable + annotations``), to decorate existing types with context-specific metadata + (possibly multiple pieces of it, as ``Annotated`` is variadic). + Specifically, a type ``T`` can be annotated with metadata ``x`` via the + typehint ``Annotated[T, x]``. This metadata can be used for either static + analysis or at runtime. If a library (or tool) encounters a typehint + ``Annotated[T, x]`` and has no special logic for metadata ``x``, it + should ignore it and simply treat the type as ``T``. Unlike the + ``no_type_check`` functionality that currently exists in the ``typing`` + module which completely disables typechecking annotations on a function + or a class, the ``Annotated`` type allows for both static typechecking + of ``T`` (e.g., via mypy or Pyre, which can safely ignore ``x``) + together with runtime access to ``x`` within a specific application. + + Ultimately, the responsibility of how to interpret the annotations (if + at all) is the responsibility of the tool or library encountering the + ``Annotated`` type. A tool or library encountering an ``Annotated`` type + can scan through the annotations to determine if they are of interest + (e.g., using ``isinstance()``). + + When a tool or a library does not support annotations or encounters an + unknown annotation it should just ignore it and treat annotated type as + the underlying type. + + It's up to the tool consuming the annotations to decide whether the + client is allowed to have several annotations on one type and how to + merge those annotations. + + Since the ``Annotated`` type allows you to put several annotations of + the same (or different) type(s) on any node, the tools or libraries + consuming those annotations are in charge of dealing with potential + duplicates. For example, if you are doing value range analysis you might + allow this:: + + T1 = Annotated[int, ValueRange(-10, 5)] + T2 = Annotated[T1, ValueRange(-20, 3)] + + Passing ``include_extras=True`` to :func:`get_type_hints` lets one + access the extra annotations at runtime. + + The details of the syntax: + + * The first argument to ``Annotated`` must be a valid type + + * Multiple type annotations are supported (``Annotated`` supports variadic + arguments):: + + Annotated[int, ValueRange(3, 10), ctype("char")] + + * ``Annotated`` must be called with at least two arguments ( + ``Annotated[int]`` is not valid) + + * The order of the annotations is preserved and matters for equality + checks:: + + Annotated[int, ValueRange(3, 10), ctype("char")] != Annotated[ + int, ctype("char"), ValueRange(3, 10) + ] + + * Nested ``Annotated`` types are flattened, with metadata ordered + starting with the innermost annotation:: + + Annotated[Annotated[int, ValueRange(3, 10)], ctype("char")] == Annotated[ + int, ValueRange(3, 10), ctype("char") + ] + + * Duplicated annotations are not removed:: + + Annotated[int, ValueRange(3, 10)] != Annotated[ + int, ValueRange(3, 10), ValueRange(3, 10) + ] + + * ``Annotated`` can be used with nested and generic aliases:: + + T = TypeVar('T') + Vec = Annotated[List[Tuple[T, T]], MaxLen(10)] + V = Vec[int] + + V == Annotated[List[Tuple[int, int]], MaxLen(10)] + + .. versionadded:: 3.9 diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 6e080c7033c..66caf3f6f82 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -303,6 +303,14 @@ signal Exposed the Linux-specific :func:`signal.pidfd_send_signal` for sending to signals to a process using a file descriptor instead of a pid. (:issue:`38712`) +typing +------ + +:pep:`593` introduced an :data:`typing.Annotated` type to decorate existing +types with context-specific metadata and new ``include_extras`` parameter to +:func:`typing.get_type_hints` to access the metadata at runtime. (Contributed +by Till Varoquaux and Konstantin Kashin.) + Optimizations ============= diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 5b4916f9c32..bc6a3db4e00 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -22,6 +22,7 @@ from typing import NewType from typing import NamedTuple, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match +from typing import Annotated import abc import typing import weakref @@ -2891,6 +2892,64 @@ class GetTypeHintTests(BaseTestCase): self.assertEqual(gth(ForRefExample.func), expects) self.assertEqual(gth(ForRefExample.nested), expects) + def test_get_type_hints_annotated(self): + def foobar(x: List['X']): ... + X = Annotated[int, (1, 10)] + self.assertEqual( + get_type_hints(foobar, globals(), locals()), + {'x': List[int]} + ) + self.assertEqual( + get_type_hints(foobar, globals(), locals(), include_extras=True), + {'x': List[Annotated[int, (1, 10)]]} + ) + BA = Tuple[Annotated[T, (1, 0)], ...] + def barfoo(x: BA): ... + self.assertEqual(get_type_hints(barfoo, globals(), locals())['x'], Tuple[T, ...]) + self.assertIs( + get_type_hints(barfoo, globals(), locals(), include_extras=True)['x'], + BA + ) + def barfoo2(x: typing.Callable[..., Annotated[List[T], "const"]], + y: typing.Union[int, Annotated[T, "mutable"]]): ... + self.assertEqual( + get_type_hints(barfoo2, globals(), locals()), + {'x': typing.Callable[..., List[T]], 'y': typing.Union[int, T]} + ) + BA2 = typing.Callable[..., List[T]] + def barfoo3(x: BA2): ... + self.assertIs( + get_type_hints(barfoo3, globals(), locals(), include_extras=True)["x"], + BA2 + ) + + def test_get_type_hints_annotated_refs(self): + + Const = Annotated[T, "Const"] + + class MySet(Generic[T]): + + def __ior__(self, other: "Const[MySet[T]]") -> "MySet[T]": + ... + + def __iand__(self, other: Const["MySet[T]"]) -> "MySet[T]": + ... + + self.assertEqual( + get_type_hints(MySet.__iand__, globals(), locals()), + {'other': MySet[T], 'return': MySet[T]} + ) + + self.assertEqual( + get_type_hints(MySet.__iand__, globals(), locals(), include_extras=True), + {'other': Const[MySet[T]], 'return': MySet[T]} + ) + + self.assertEqual( + get_type_hints(MySet.__ior__, globals(), locals()), + {'other': MySet[T], 'return': MySet[T]} + ) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -2906,6 +2965,7 @@ class GetUtilitiesTestCase(TestCase): self.assertIs(get_origin(Generic), Generic) self.assertIs(get_origin(Generic[T]), Generic) self.assertIs(get_origin(List[Tuple[T, T]][int]), list) + self.assertIs(get_origin(Annotated[T, 'thing']), Annotated) def test_get_args(self): T = TypeVar('T') @@ -2926,6 +2986,7 @@ class GetUtilitiesTestCase(TestCase): (int, Callable[[Tuple[T, ...]], str])) self.assertEqual(get_args(Tuple[int, ...]), (int, ...)) self.assertEqual(get_args(Tuple[()]), ((),)) + self.assertEqual(get_args(Annotated[T, 'one', 2, ['three']]), (T, 'one', 2, ['three'])) class CollectionsAbcTests(BaseTestCase): @@ -3844,6 +3905,179 @@ class RETests(BaseTestCase): "type 're.Match' is not an acceptable base type") +class AnnotatedTests(BaseTestCase): + + def test_repr(self): + self.assertEqual( + repr(Annotated[int, 4, 5]), + "typing.Annotated[int, 4, 5]" + ) + self.assertEqual( + repr(Annotated[List[int], 4, 5]), + "typing.Annotated[typing.List[int], 4, 5]" + ) + + def test_flatten(self): + A = Annotated[Annotated[int, 4], 5] + self.assertEqual(A, Annotated[int, 4, 5]) + self.assertEqual(A.__metadata__, (4, 5)) + self.assertEqual(A.__origin__, int) + + def test_specialize(self): + L = Annotated[List[T], "my decoration"] + LI = Annotated[List[int], "my decoration"] + self.assertEqual(L[int], Annotated[List[int], "my decoration"]) + self.assertEqual(L[int].__metadata__, ("my decoration",)) + self.assertEqual(L[int].__origin__, List[int]) + with self.assertRaises(TypeError): + LI[int] + with self.assertRaises(TypeError): + L[int, float] + + def test_hash_eq(self): + self.assertEqual(len({Annotated[int, 4, 5], Annotated[int, 4, 5]}), 1) + self.assertNotEqual(Annotated[int, 4, 5], Annotated[int, 5, 4]) + self.assertNotEqual(Annotated[int, 4, 5], Annotated[str, 4, 5]) + self.assertNotEqual(Annotated[int, 4], Annotated[int, 4, 4]) + self.assertEqual( + {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, + {Annotated[int, 4, 5], Annotated[T, 4, 5]} + ) + + def test_instantiate(self): + class C: + classvar = 4 + + def __init__(self, x): + self.x = x + + def __eq__(self, other): + if not isinstance(other, C): + return NotImplemented + return other.x == self.x + + A = Annotated[C, "a decoration"] + a = A(5) + c = C(5) + self.assertEqual(a, c) + self.assertEqual(a.x, c.x) + self.assertEqual(a.classvar, c.classvar) + + def test_instantiate_generic(self): + MyCount = Annotated[typing.Counter[T], "my decoration"] + self.assertEqual(MyCount([4, 4, 5]), {4: 2, 5: 1}) + self.assertEqual(MyCount[int]([4, 4, 5]), {4: 2, 5: 1}) + + def test_cannot_instantiate_forward(self): + A = Annotated["int", (5, 6)] + with self.assertRaises(TypeError): + A(5) + + def test_cannot_instantiate_type_var(self): + A = Annotated[T, (5, 6)] + with self.assertRaises(TypeError): + A(5) + + def test_cannot_getattr_typevar(self): + with self.assertRaises(AttributeError): + Annotated[T, (5, 7)].x + + def test_attr_passthrough(self): + class C: + classvar = 4 + + A = Annotated[C, "a decoration"] + self.assertEqual(A.classvar, 4) + A.x = 5 + self.assertEqual(C.x, 5) + + def test_hash_eq(self): + self.assertEqual(len({Annotated[int, 4, 5], Annotated[int, 4, 5]}), 1) + self.assertNotEqual(Annotated[int, 4, 5], Annotated[int, 5, 4]) + self.assertNotEqual(Annotated[int, 4, 5], Annotated[str, 4, 5]) + self.assertNotEqual(Annotated[int, 4], Annotated[int, 4, 4]) + self.assertEqual( + {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, + {Annotated[int, 4, 5], Annotated[T, 4, 5]} + ) + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): + class C(Annotated): + pass + + def test_cannot_check_instance(self): + with self.assertRaises(TypeError): + isinstance(5, Annotated[int, "positive"]) + + def test_cannot_check_subclass(self): + with self.assertRaises(TypeError): + issubclass(int, Annotated[int, "positive"]) + + def test_pickle(self): + samples = [typing.Any, typing.Union[int, str], + typing.Optional[str], Tuple[int, ...], + typing.Callable[[str], bytes]] + + for t in samples: + x = Annotated[t, "a"] + + for prot in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=prot, type=t): + pickled = pickle.dumps(x, prot) + restored = pickle.loads(pickled) + self.assertEqual(x, restored) + + global _Annotated_test_G + + class _Annotated_test_G(Generic[T]): + x = 1 + + G = Annotated[_Annotated_test_G[int], "A decoration"] + G.foo = 42 + G.bar = 'abc' + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(G, proto) + x = pickle.loads(z) + self.assertEqual(x.foo, 42) + self.assertEqual(x.bar, 'abc') + self.assertEqual(x.x, 1) + + def test_subst(self): + dec = "a decoration" + dec2 = "another decoration" + + S = Annotated[T, dec2] + self.assertEqual(S[int], Annotated[int, dec2]) + + self.assertEqual(S[Annotated[int, dec]], Annotated[int, dec, dec2]) + L = Annotated[List[T], dec] + + self.assertEqual(L[int], Annotated[List[int], dec]) + with self.assertRaises(TypeError): + L[int, int] + + self.assertEqual(S[L[int]], Annotated[List[int], dec, dec2]) + + D = Annotated[typing.Dict[KT, VT], dec] + self.assertEqual(D[str, int], Annotated[typing.Dict[str, int], dec]) + with self.assertRaises(TypeError): + D[int] + + It = Annotated[int, dec] + with self.assertRaises(TypeError): + It[None] + + LI = L[int] + with self.assertRaises(TypeError): + LI[None] + + def test_annotated_in_other_types(self): + X = List[Annotated[T, 5]] + self.assertEqual(X[int], List[Annotated[int, 5]]) + + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Lib/typing.py b/Lib/typing.py index 28c887ed35e..5a7077c27c4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -31,6 +31,7 @@ from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType # Please keep __all__ alphabetized within each category. __all__ = [ # Super-special typing primitives. + 'Annotated', 'Any', 'Callable', 'ClassVar', @@ -1118,6 +1119,101 @@ class Protocol(Generic, metaclass=_ProtocolMeta): cls.__init__ = _no_init +class _AnnotatedAlias(_GenericAlias, _root=True): + """Runtime representation of an annotated type. + + At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' + with extra annotations. The alias behaves like a normal typing alias, + instantiating is the same as instantiating the underlying type, binding + it to types is also the same. + """ + def __init__(self, origin, metadata): + if isinstance(origin, _AnnotatedAlias): + metadata = origin.__metadata__ + metadata + origin = origin.__origin__ + super().__init__(origin, origin) + self.__metadata__ = metadata + + def copy_with(self, params): + assert len(params) == 1 + new_type = params[0] + return _AnnotatedAlias(new_type, self.__metadata__) + + def __repr__(self): + return "typing.Annotated[{}, {}]".format( + _type_repr(self.__origin__), + ", ".join(repr(a) for a in self.__metadata__) + ) + + def __reduce__(self): + return operator.getitem, ( + Annotated, (self.__origin__,) + self.__metadata__ + ) + + def __eq__(self, other): + if not isinstance(other, _AnnotatedAlias): + return NotImplemented + if self.__origin__ != other.__origin__: + return False + return self.__metadata__ == other.__metadata__ + + def __hash__(self): + return hash((self.__origin__, self.__metadata__)) + + +class Annotated: + """Add context specific metadata to a type. + + Example: Annotated[int, runtime_check.Unsigned] indicates to the + hypothetical runtime_check module that this type is an unsigned int. + Every other consumer of this type can ignore this metadata and treat + this type as int. + + The first argument to Annotated must be a valid type. + + Details: + + - It's an error to call `Annotated` with less than two arguments. + - Nested Annotated are flattened:: + + Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3] + + - Instantiating an annotated type is equivalent to instantiating the + underlying type:: + + Annotated[C, Ann1](5) == C(5) + + - Annotated can be used as a generic type alias:: + + Optimized = Annotated[T, runtime.Optimize()] + Optimized[int] == Annotated[int, runtime.Optimize()] + + OptimizedList = Annotated[List[T], runtime.Optimize()] + OptimizedList[int] == Annotated[List[int], runtime.Optimize()] + """ + + __slots__ = () + + def __new__(cls, *args, **kwargs): + raise TypeError("Type Annotated cannot be instantiated.") + + @_tp_cache + def __class_getitem__(cls, params): + if not isinstance(params, tuple) or len(params) < 2: + raise TypeError("Annotated[...] should be used " + "with at least two arguments (a type and an " + "annotation).") + msg = "Annotated[t, ...]: t must be a type." + origin = _type_check(params[0], msg) + metadata = tuple(params[1:]) + return _AnnotatedAlias(origin, metadata) + + def __init_subclass__(cls, *args, **kwargs): + raise TypeError( + "Cannot subclass {}.Annotated".format(cls.__module__) + ) + + def runtime_checkable(cls): """Mark a protocol class as a runtime protocol. @@ -1179,12 +1275,13 @@ _allowed_types = (types.FunctionType, types.BuiltinFunctionType, WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) -def get_type_hints(obj, globalns=None, localns=None): +def get_type_hints(obj, globalns=None, localns=None, include_extras=False): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals, and if necessary - adds Optional[t] if a default value equal to None is set. + forward references encoded as string literals, adds Optional[t] if a + default value equal to None is set and recursively replaces all + 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations are returned as a dictionary. For classes, annotations include also @@ -1228,7 +1325,7 @@ def get_type_hints(obj, globalns=None, localns=None): value = ForwardRef(value, is_argument=False) value = _eval_type(value, base_globals, localns) hints[name] = value - return hints + return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} if globalns is None: if isinstance(obj, types.ModuleType): @@ -1262,7 +1359,22 @@ def get_type_hints(obj, globalns=None, localns=None): if name in defaults and defaults[name] is None: value = Optional[value] hints[name] = value - return hints + return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} + + +def _strip_annotations(t): + """Strips the annotations from a given type. + """ + if isinstance(t, _AnnotatedAlias): + return _strip_annotations(t.__origin__) + if isinstance(t, _GenericAlias): + stripped_args = tuple(_strip_annotations(a) for a in t.__args__) + if stripped_args == t.__args__: + return t + res = t.copy_with(stripped_args) + res._special = t._special + return res + return t def get_origin(tp): @@ -1279,6 +1391,8 @@ def get_origin(tp): get_origin(Union[T, int]) is Union get_origin(List[Tuple[T, T]][int]) == list """ + if isinstance(tp, _AnnotatedAlias): + return Annotated if isinstance(tp, _GenericAlias): return tp.__origin__ if tp is Generic: @@ -1297,6 +1411,8 @@ def get_args(tp): get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) get_args(Callable[[], T][int]) == ([], int) """ + if isinstance(tp, _AnnotatedAlias): + return (tp.__origin__,) + tp.__metadata__ if isinstance(tp, _GenericAlias): res = tp.__args__ if get_origin(tp) is collections.abc.Callable and res[0] is not Ellipsis: diff --git a/Misc/NEWS.d/next/Library/2020-01-29-22-47-12.bpo-39491.tdl17b.rst b/Misc/NEWS.d/next/Library/2020-01-29-22-47-12.bpo-39491.tdl17b.rst new file mode 100644 index 00000000000..1dd36454dc2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-01-29-22-47-12.bpo-39491.tdl17b.rst @@ -0,0 +1,3 @@ +Add :data:`typing.Annotated` and ``include_extras`` parameter to +:func:`typing.get_type_hints` as part of :pep:`593`. Patch by Till +Varoquaux, documentation by Till Varoquaux and Konstantin Kashin.