From ac6c3de03c5bb06a9a463701fb297148f5a5746f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 12 Apr 2022 12:31:02 -0700 Subject: [PATCH] gh-91243: Add typing.Required and NotRequired (PEP 655) (GH-32419) I talked to @davidfstr and I offered to implement the runtime part of PEP 655 to make sure we can get it in before the feature freeze. We're going to defer the documentation to a separate PR, because it can wait until after the feature freeze. The runtime implementation conveniently already exists in typing-extensions, so I largely copied that. Co-authored-by: David Foster --- Lib/test/_typed_dict_helper.py | 8 +- Lib/test/test_typing.py | 174 +++++++++++++++++- Lib/typing.py | 63 ++++++- .../2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst | 2 + 4 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst diff --git a/Lib/test/_typed_dict_helper.py b/Lib/test/_typed_dict_helper.py index d333db19318..3328330c995 100644 --- a/Lib/test/_typed_dict_helper.py +++ b/Lib/test/_typed_dict_helper.py @@ -6,13 +6,19 @@ look something like this: class Bar(_typed_dict_helper.Foo, total=False): b: int + +In addition, it uses multiple levels of Annotated to test the interaction +between the __future__ import, Annotated, and Required. """ from __future__ import annotations -from typing import Optional, TypedDict +from typing import Annotated, Optional, Required, TypedDict OptionalIntType = Optional[int] class Foo(TypedDict): a: OptionalIntType + +class VeryAnnotated(TypedDict, total=False): + a: Annotated[Annotated[Annotated[Required[int], "a"], "b"], "c"] diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b884f7b2cce..2afa5449bc2 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -23,7 +23,7 @@ from typing import is_typeddict from typing import reveal_type from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NamedTuple, TypedDict +from typing import NamedTuple, NotRequired, Required, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef @@ -3993,6 +3993,26 @@ class Options(TypedDict, total=False): log_level: int log_path: str +class TotalMovie(TypedDict): + title: str + year: NotRequired[int] + +class NontotalMovie(TypedDict, total=False): + title: Required[str] + year: int + +class AnnotatedMovie(TypedDict): + title: Annotated[Required[str], "foobar"] + year: NotRequired[Annotated[int, 2000]] + +class DeeplyAnnotatedMovie(TypedDict): + title: Annotated[Annotated[Required[str], "foobar"], "another level"] + year: NotRequired[Annotated[int, 2000]] + +class WeirdlyQuotedMovie(TypedDict): + title: Annotated['Annotated[Required[str], "foobar"]', "another level"] + year: NotRequired['Annotated[int, 2000]'] + class HasForeignBaseClass(mod_generics_cache.A): some_xrepr: 'XRepr' other_a: 'mod_generics_cache.A' @@ -4280,6 +4300,36 @@ class GetTypeHintTests(BaseTestCase): ): get_type_hints(ann_module6) + def test_get_type_hints_typeddict(self): + self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(TotalMovie, include_extras=True), { + 'title': str, + 'year': NotRequired[int], + }) + + self.assertEqual(get_type_hints(AnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(AnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(WeirdlyQuotedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(WeirdlyQuotedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated), {'a': int}) + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated, include_extras=True), { + 'a': Annotated[Required[int], "a", "b", "c"] + }) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -4305,6 +4355,8 @@ class GetUtilitiesTestCase(TestCase): self.assertIs(get_origin(list | str), types.UnionType) self.assertIs(get_origin(P.args), P) self.assertIs(get_origin(P.kwargs), P) + self.assertIs(get_origin(Required[int]), Required) + self.assertIs(get_origin(NotRequired[int]), NotRequired) def test_get_args(self): T = TypeVar('T') @@ -4342,6 +4394,8 @@ class GetUtilitiesTestCase(TestCase): self.assertEqual(get_args(Callable[Concatenate[int, P], int]), (Concatenate[int, P], int)) self.assertEqual(get_args(list | str), (list, str)) + self.assertEqual(get_args(Required[int]), (int,)) + self.assertEqual(get_args(NotRequired[int]), (int,)) class CollectionsAbcTests(BaseTestCase): @@ -5299,6 +5353,32 @@ class TypedDictTests(BaseTestCase): 'voice': str, } + def test_required_notrequired_keys(self): + self.assertEqual(NontotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(NontotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(TotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(TotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(_typed_dict_helper.VeryAnnotated.__required_keys__, + frozenset()) + self.assertEqual(_typed_dict_helper.VeryAnnotated.__optional_keys__, + frozenset({"a"})) + + self.assertEqual(AnnotatedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(AnnotatedMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(WeirdlyQuotedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(WeirdlyQuotedMovie.__optional_keys__, + frozenset({"year"})) + def test_multiple_inheritance(self): class One(TypedDict): one: int @@ -5399,6 +5479,98 @@ class TypedDictTests(BaseTestCase): ) +class RequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + Required[NotRequired] + with self.assertRaises(TypeError): + Required[int, str] + with self.assertRaises(TypeError): + Required[int][str] + + def test_repr(self): + self.assertEqual(repr(Required), 'typing.Required') + cv = Required[int] + self.assertEqual(repr(cv), 'typing.Required[int]') + cv = Required[Employee] + self.assertEqual(repr(cv), f'typing.Required[{__name__}.Employee]') + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class C(type(Required)): + pass + with self.assertRaises(TypeError): + class C(type(Required[int])): + pass + with self.assertRaises(TypeError): + class C(Required): + pass + with self.assertRaises(TypeError): + class C(Required[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + Required() + with self.assertRaises(TypeError): + type(Required)() + with self.assertRaises(TypeError): + type(Required[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, Required[int]) + with self.assertRaises(TypeError): + issubclass(int, Required) + + +class NotRequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + NotRequired[Required] + with self.assertRaises(TypeError): + NotRequired[int, str] + with self.assertRaises(TypeError): + NotRequired[int][str] + + def test_repr(self): + self.assertEqual(repr(NotRequired), 'typing.NotRequired') + cv = NotRequired[int] + self.assertEqual(repr(cv), 'typing.NotRequired[int]') + cv = NotRequired[Employee] + self.assertEqual(repr(cv), f'typing.NotRequired[{__name__}.Employee]') + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class C(type(NotRequired)): + pass + with self.assertRaises(TypeError): + class C(type(NotRequired[int])): + pass + with self.assertRaises(TypeError): + class C(NotRequired): + pass + with self.assertRaises(TypeError): + class C(NotRequired[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + NotRequired() + with self.assertRaises(TypeError): + type(NotRequired)() + with self.assertRaises(TypeError): + type(NotRequired[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, NotRequired[int]) + with self.assertRaises(TypeError): + issubclass(int, NotRequired) + + class IOTests(BaseTestCase): def test_io(self): diff --git a/Lib/typing.py b/Lib/typing.py index ec8cbbd8b20..46fc72218dc 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -132,9 +132,11 @@ __all__ = [ 'no_type_check', 'no_type_check_decorator', 'NoReturn', + 'NotRequired', 'overload', 'ParamSpecArgs', 'ParamSpecKwargs', + 'Required', 'reveal_type', 'runtime_checkable', 'Self', @@ -2262,6 +2264,8 @@ def _strip_annotations(t): """ if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired): + return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: @@ -2786,10 +2790,22 @@ class _TypedDictMeta(type): optional_keys.update(base.__dict__.get('__optional_keys__', ())) annotations.update(own_annotations) - if total: - required_keys.update(own_annotation_keys) - else: - optional_keys.update(own_annotation_keys) + for annotation_key, annotation_type in own_annotations.items(): + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + annotation_origin = get_origin(annotation_type) + + if annotation_origin is Required: + required_keys.add(annotation_key) + elif annotation_origin is NotRequired: + optional_keys.add(annotation_key) + elif total: + required_keys.add(annotation_key) + else: + optional_keys.add(annotation_key) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) @@ -2874,6 +2890,45 @@ _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {}) TypedDict.__mro_entries__ = lambda bases: (_TypedDict,) +@_SpecialForm +def Required(self, parameters): + """A special typing construct to mark a key of a total=False TypedDict + as required. For example: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + + There is no runtime checking that a required key is actually provided + when instantiating a related TypedDict. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + +@_SpecialForm +def NotRequired(self, parameters): + """A special typing construct to mark a key of a TypedDict as + potentially missing. For example: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + class NewType: """NewType creates simple unique types with almost zero runtime overhead. NewType(name, tp) is considered a subtype of tp diff --git a/Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst b/Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst new file mode 100644 index 00000000000..33996632140 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst @@ -0,0 +1,2 @@ +Implement ``typing.Required`` and ``typing.NotRequired`` (:pep:`655`). Patch +by Jelle Zijlstra.