bpo-42059: Fix required/optional keys for TypedDict(..., total=False) (GH-22736)

This allows easier Enum construction in unusual cases, such as including dynamic member definitions into a class definition:

# created dynamically
foo_defines = {'FOO_CAT': 'aloof', 'BAR_DOG': 'friendly', 'FOO_HORSE': 'big'}

class Foo(Enum):
    vars().update({
            k: v
            for k, v in foo_defines.items()
            if k.startswith('FOO_')
            })
    def upper(self):
        # example method
        return self.value.upper()

Doc/library/enum.rst

@@ -67,10 +67,12 @@ helper, :class:`auto`.
 
 .. class:: auto
 
-    Instances are replaced with an appropriate value for Enum members.  By default, the initial value starts at 1.
+    Instances are replaced with an appropriate value for Enum members.
+    :class:`StrEnum` defaults to the lower-cased version of the member name,
+    while other Enums default to 1 and increase from there.
 
 .. versionadded:: 3.6  ``Flag``, ``IntFlag``, ``auto``
-
+.. versionadded:: 3.10  ``StrEnum``
 
 Creating an Enum
 ----------------

Doc/library/unittest.mock.rst

@@ -262,9 +262,10 @@ the *new_callable* argument to :func:`patch`.
       this is a new Mock (created on first access). See the
       :attr:`return_value` attribute.
 
-    * *unsafe*: By default if any attribute starts with *assert* or
-      *assret* will raise an :exc:`AttributeError`. Passing ``unsafe=True``
-      will allow access to these attributes.
+    * *unsafe*: By default, accessing any attribute with name starting with
+      *assert*, *assret*, *asert*, *aseert* or *assrt* will raise an
+      :exc:`AttributeError`. Passing ``unsafe=True`` will allow access to
+      these attributes.
 
       .. versionadded:: 3.5
 

Include/pylifecycle.h

@@ -32,6 +32,8 @@ PyAPI_FUNC(void) _Py_NO_RETURN Py_Exit(int);
 /* Bootstrap __main__ (defined in Modules/main.c) */
 PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
 
+PyAPI_FUNC(int) Py_FrozenMain(int argc, char **argv);
+
 PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv);
 
 /* In pathconfig.c */

Lib/enum.py

@@ -136,7 +136,7 @@ class _EnumDict(dict):
                 key = '_order_'
         elif key in self._member_names:
             # descriptor overwriting an enum?
-            raise TypeError('Attempted to reuse key: %r' % key)
+            raise TypeError('%r already defined as: %r' % (key, self[key]))
         elif key in self._ignore:
             pass
         elif not _is_descriptor(value):
@@ -157,6 +157,16 @@ class _EnumDict(dict):
             self._last_values.append(value)
         super().__setitem__(key, value)
 
+    def update(self, members, **more_members):
+        try:
+            for name in members.keys():
+                self[name] = members[name]
+        except AttributeError:
+            for name, value in members:
+                self[name] = value
+        for name, value in more_members.items():
+            self[name] = value
+
 
 # Dummy value for Enum as EnumMeta explicitly checks for it, but of course
 # until EnumMeta finishes running the first time the Enum class doesn't exist.
@@ -826,6 +836,12 @@ class StrEnum(str, Enum):
 
     __str__ = str.__str__
 
+    def _generate_next_value_(name, start, count, last_values):
+        """
+        Return the lower-cased version of the member name.
+        """
+        return name.lower()
+
 
 def _reduce_ex_by_name(self, proto):
     return self.name

Lib/test/test_enum.py

@@ -2179,6 +2179,40 @@ class TestEnum(unittest.TestCase):
         self.assertEqual(Private._Private__corporal, 'Radar')
         self.assertEqual(Private._Private__major_, 'Hoolihan')
 
+    def test_strenum_auto(self):
+        class Strings(StrEnum):
+            ONE = auto()
+            TWO = auto()
+        self.assertEqual([Strings.ONE, Strings.TWO], ['one', 'two'])
+
+
+    def test_dynamic_members_with_static_methods(self):
+        #
+        foo_defines = {'FOO_CAT': 'aloof', 'BAR_DOG': 'friendly', 'FOO_HORSE': 'big'}
+        class Foo(Enum):
+            vars().update({
+                    k: v
+                    for k, v in foo_defines.items()
+                    if k.startswith('FOO_')
+                    })
+            def upper(self):
+                return self.value.upper()
+        self.assertEqual(list(Foo), [Foo.FOO_CAT, Foo.FOO_HORSE])
+        self.assertEqual(Foo.FOO_CAT.value, 'aloof')
+        self.assertEqual(Foo.FOO_HORSE.upper(), 'BIG')
+        #
+        with self.assertRaisesRegex(TypeError, "'FOO_CAT' already defined as: 'aloof'"):
+            class FooBar(Enum):
+                vars().update({
+                        k: v
+                        for k, v in foo_defines.items()
+                        if k.startswith('FOO_')
+                        },
+                        **{'FOO_CAT': 'small'},
+                        )
+                def upper(self):
+                    return self.value.upper()
+
 
 class TestOrder(unittest.TestCase):
 

Lib/test/test_typing.py

@@ -3895,10 +3895,14 @@ class TypedDictTests(BaseTestCase):
         self.assertEqual(D(), {})
         self.assertEqual(D(x=1), {'x': 1})
         self.assertEqual(D.__total__, False)
+        self.assertEqual(D.__required_keys__, frozenset())
+        self.assertEqual(D.__optional_keys__, {'x'})
 
         self.assertEqual(Options(), {})
         self.assertEqual(Options(log_level=2), {'log_level': 2})
         self.assertEqual(Options.__total__, False)
+        self.assertEqual(Options.__required_keys__, frozenset())
+        self.assertEqual(Options.__optional_keys__, {'log_level', 'log_path'})
 
     def test_optional_keys(self):
         class Point2Dor3D(Point2D, total=False):

Lib/typing.py

@@ -2043,14 +2043,14 @@ def TypedDict(typename, fields=None, /, *, total=True, **kwargs):
         raise TypeError("TypedDict takes either a dict or keyword arguments,"
                         " but not both")
 
-    ns = {'__annotations__': dict(fields), '__total__': total}
+    ns = {'__annotations__': dict(fields)}
     try:
         # Setting correct module is necessary to make typed dict classes pickleable.
         ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__')
     except (AttributeError, ValueError):
         pass
 
-    return _TypedDictMeta(typename, (), ns)
+    return _TypedDictMeta(typename, (), ns, total=total)
 
 _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {})
 TypedDict.__mro_entries__ = lambda bases: (_TypedDict,)

Lib/unittest/mock.py

@@ -632,8 +632,9 @@ class NonCallableMock(Base):
             raise AttributeError(name)
         if not self._mock_unsafe:
             if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')):
-                raise AttributeError("Attributes cannot start with 'assert' "
-                                     "or its misspellings")
+                raise AttributeError(
+                    f"{name} is not a valid assertion. Use a spec "
+                    f"for the mock if {name} is meant to be an attribute.")
 
         result = self._mock_children.get(name)
         if result is _deleted:

Lib/unittest/test/testmock/testmock.py

@@ -1598,7 +1598,7 @@ class MockTest(unittest.TestCase):
     #Issue21238
     def test_mock_unsafe(self):
         m = Mock()
-        msg = "Attributes cannot start with 'assert' or its misspellings"
+        msg = "is not a valid assertion. Use a spec for the mock"
         with self.assertRaisesRegex(AttributeError, msg):
             m.assert_foo_call()
         with self.assertRaisesRegex(AttributeError, msg):

Misc/NEWS.d/next/C API/2020-12-10-10-43-03.bpo-42591.CXNY8G.rst

@@ -0,0 +1,3 @@
+Export the :c:func:`Py_FrozenMain` function: fix a Python 3.9.0 regression.
+Python 3.9 uses ``-fvisibility=hidden`` and the function was not exported
+explicitly and so not exported.

Misc/NEWS.d/next/Library/2020-10-17-12-42-08.bpo-42059.ZGMZ3D.rst

@@ -0,0 +1 @@
+:class:`typing.TypedDict` types created using the alternative call-style syntax now correctly respect the ``total`` keyword argument when setting their ``__required_keys__`` and ``__optional_keys__`` class attributes.

Misc/NEWS.d/next/Library/2020-12-09-14-15-48.bpo-34750.x8TASR.rst

@@ -0,0 +1 @@
+[Enum] `_EnumDict.update()` is now supported

Misc/NEWS.d/next/Library/2020-12-09-19-45-32.bpo-42385.boGbjo.rst

@@ -0,0 +1 @@
+StrEnum: fix _generate_next_value_ to return a str

Misc/NEWS.d/next/Library/2020-12-10-09-24-44.bpo-41877.iJSCvM.rst

@@ -0,0 +1 @@
+AttributeError for suspected misspellings of assertions on mocks are now pointing out that the cause are misspelled assertions and also what to do if the misspelling is actually an intended attribute name. The unittest.mock document is also updated to reflect the current set of recognised misspellings.

PC/python3dll.c

@@ -49,6 +49,7 @@ EXPORT_FUNC(Py_Exit)
 EXPORT_FUNC(Py_FatalError)
 EXPORT_FUNC(Py_Finalize)
 EXPORT_FUNC(Py_FinalizeEx)
+EXPORT_FUNC(Py_FrozenMain)
 EXPORT_FUNC(Py_GenericAlias)
 EXPORT_FUNC(Py_GenericAliasType)
 EXPORT_FUNC(Py_GetArgcArgv)