Close #19030: improvements to inspect and Enum.

inspect.getmembers and inspect.classify_class_attrs now search the metaclass
mro for types.DynamicClassAttributes (what use to be called
enum._RouteClassAttributeToGetattr); in part this means that these two
functions no longer rely solely on dir().

Besides now returning more accurate information, these improvements also
allow a more helpful help() on Enum classes.

Lib/enum.py

@@ -1,36 +1,10 @@
 import sys
 from collections import OrderedDict
-from types import MappingProxyType
+from types import MappingProxyType, DynamicClassAttribute
 
 __all__ = ['Enum', 'IntEnum', 'unique']
 
 
-class _RouteClassAttributeToGetattr:
-    """Route attribute access on a class to __getattr__.
-
-    This is a descriptor, used to define attributes that act differently when
-    accessed through an instance and through a class.  Instance access remains
-    normal, but access to an attribute through a class will be routed to the
-    class's __getattr__ method; this is done by raising AttributeError.
-
-    """
-    def __init__(self, fget=None):
-        self.fget = fget
-        if fget.__doc__ is not None:
-            self.__doc__ = fget.__doc__
-
-    def __get__(self, instance, ownerclass=None):
-        if instance is None:
-            raise AttributeError()
-        return self.fget(instance)
-
-    def __set__(self, instance, value):
-        raise AttributeError("can't set attribute")
-
-    def __delete__(self, instance):
-        raise AttributeError("can't delete attribute")
-
-
 def _is_descriptor(obj):
     """Returns True if obj is a descriptor, False otherwise."""
     return (
@@ -504,12 +478,12 @@ class Enum(metaclass=EnumMeta):
     # members are not set directly on the enum class -- __getattr__ is
     # used to look them up.
 
-    @_RouteClassAttributeToGetattr
+    @DynamicClassAttribute
     def name(self):
         """The name of the Enum member."""
         return self._name_
 
-    @_RouteClassAttributeToGetattr
+    @DynamicClassAttribute
     def value(self):
         """The value of the Enum member."""
         return self._value_

Lib/inspect.py

@@ -267,11 +267,25 @@ def getmembers(object, predicate=None):
     else:
         mro = ()
     results = []
-    for key in dir(object):
+    processed = set()
+    names = dir(object)
+    # add any virtual attributes to the list of names if object is a class
+    # this may result in duplicate entries if, for example, a virtual
+    # attribute with the same name as a member property exists
+    try:
+        for base in object.__bases__:
+            for k, v in base.__dict__.items():
+                if isinstance(v, types.DynamicClassAttribute):
+                    names.append(k)
+    except AttributeError:
+        pass
+    for key in names:
         # First try to get the value via __dict__. Some descriptors don't
         # like calling their __get__ (see bug #1785).
         for base in mro:
-            if key in base.__dict__:
+            if key in base.__dict__ and key not in processed:
+                # handle the normal case first; if duplicate entries exist
+                # they will be handled second
                 value = base.__dict__[key]
                 break
         else:
@@ -281,7 +295,8 @@ def getmembers(object, predicate=None):
                 continue
         if not predicate or predicate(value):
             results.append((key, value))
-    results.sort()
+        processed.add(key)
+    results.sort(key=lambda pair: pair[0])
     return results
 
 Attribute = namedtuple('Attribute', 'name kind defining_class object')
@@ -298,16 +313,15 @@ def classify_class_attrs(cls):
                'class method'    created via classmethod()
                'static method'   created via staticmethod()
                'property'        created via property()
-               'method'          any other flavor of method
+               'method'          any other flavor of method or descriptor
                'data'            not a method
 
         2. The class which defined this attribute (a class).
 
-        3. The object as obtained directly from the defining class's
+        3. The object as obtained by calling getattr; if this fails, or if the
-           __dict__, not via getattr.  This is especially important for
+           resulting object does not live anywhere in the class' mro (including
-           data attributes:  C.data is just a data object, but
+           metaclasses) then the object is looked up in the defining class's
-           C.__dict__['data'] may be a data descriptor with additional
+           dict (found by walking the mro).
-           info, like a __doc__ string.
 
     If one of the items in dir(cls) is stored in the metaclass it will now
     be discovered and not have None be listed as the class in which it was
@@ -316,46 +330,72 @@ def classify_class_attrs(cls):
 
     mro = getmro(cls)
     metamro = getmro(type(cls)) # for attributes stored in the metaclass
+    metamro = tuple([cls for cls in metamro if cls not in (type, object)])
+    possible_bases = (cls,) + mro + metamro
     names = dir(cls)
+    # add any virtual attributes to the list of names
+    # this may result in duplicate entries if, for example, a virtual
+    # attribute with the same name as a member property exists
+    for base in cls.__bases__:
+        for k, v in base.__dict__.items():
+            if isinstance(v, types.DynamicClassAttribute):
+                names.append(k)
     result = []
+    processed = set()
+    sentinel = object()
     for name in names:
         # Get the object associated with the name, and where it was defined.
+        # Normal objects will be looked up with both getattr and directly in
+        # its class' dict (in case getattr fails [bug #1785], and also to look
+        # for a docstring).
+        # For VirtualAttributes on the second pass we only look in the
+        # class's dict.
+        #
         # Getting an obj from the __dict__ sometimes reveals more than
         # using getattr.  Static and class methods are dramatic examples.
-        # Furthermore, some objects may raise an Exception when fetched with
-        # getattr(). This is the case with some descriptors (bug #1785).
-        # Thus, we only use getattr() as a last resort.
         homecls = None
-        for base in (cls,) + mro + metamro:
+        get_obj = sentinel
-            if name in base.__dict__:
+        dict_obj = sentinel
-                obj = base.__dict__[name]
-                homecls = base
-                break
-        else:
-            obj = getattr(cls, name)
-            homecls = getattr(obj, "__objclass__", homecls)
 
-        # Classify the object.
+
+        if name not in processed:
+            try:
+                get_obj = getattr(cls, name)
+            except Exception as exc:
+                pass
+            else:
+                homecls = getattr(get_obj, "__class__")
+                homecls = getattr(get_obj, "__objclass__", homecls)
+                if homecls not in possible_bases:
+                    # if the resulting object does not live somewhere in the
+                    # mro, drop it and go with the dict_obj version only
+                    homecls = None
+                    get_obj = sentinel
+
+        for base in possible_bases:
+            if name in base.__dict__:
+                dict_obj = base.__dict__[name]
+                homecls = homecls or base
+                break
+
+        # Classify the object or its descriptor.
+        if get_obj is not sentinel:
+            obj = get_obj
+        else:
+            obj = dict_obj
         if isinstance(obj, staticmethod):
             kind = "static method"
         elif isinstance(obj, classmethod):
             kind = "class method"
         elif isinstance(obj, property):
             kind = "property"
-        elif ismethoddescriptor(obj):
+        elif isfunction(obj) or ismethoddescriptor(obj):
             kind = "method"
-        elif isdatadescriptor(obj):
-            kind = "data"
         else:
-            obj_via_getattr = getattr(cls, name)
+            kind = "data"
-            if (isfunction(obj_via_getattr) or
-                ismethoddescriptor(obj_via_getattr)):
-                kind = "method"
-            else:
-                kind = "data"
-            obj = obj_via_getattr
 
         result.append(Attribute(name, kind, homecls, obj))
+        processed.add(name)
 
     return result
 

Lib/test/test_inspect.py

@@ -652,6 +652,14 @@ class TestClassesAndFunctions(unittest.TestCase):
             if isinstance(builtin, type):
                 inspect.classify_class_attrs(builtin)
 
+    def test_classify_VirtualAttribute(self):
+        class VA:
+            @types.DynamicClassAttribute
+            def ham(self):
+                return 'eggs'
+        should_find = inspect.Attribute('ham', 'data', VA, VA.__dict__['ham'])
+        self.assertIn(should_find, inspect.classify_class_attrs(VA))
+
     def test_getmembers_descriptors(self):
         class A(object):
             dd = _BrokenDataDescriptor()
@@ -695,6 +703,13 @@ class TestClassesAndFunctions(unittest.TestCase):
         self.assertIn(('f', b.f), inspect.getmembers(b))
         self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod))
 
+    def test_getmembers_VirtualAttribute(self):
+        class A:
+            @types.DynamicClassAttribute
+            def eggs(self):
+                return 'spam'
+        self.assertIn(('eggs', A.__dict__['eggs']), inspect.getmembers(A))
+
 
 _global_ref = object()
 class TestGetClosureVars(unittest.TestCase):
@@ -1082,6 +1097,15 @@ class TestGetattrStatic(unittest.TestCase):
 
         self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.x)
 
+    def test_classVirtualAttribute(self):
+        class Thing(object):
+            @types.DynamicClassAttribute
+            def x(self):
+                return self._x
+            _x = object()
+
+        self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.__dict__['x'])
+
     def test_inherited_classattribute(self):
         class Thing(object):
             x = object()

Lib/types.py

@@ -99,3 +99,60 @@ def _calculate_meta(meta, bases):
                         "must be a (non-strict) subclass "
                         "of the metaclasses of all its bases")
     return winner
+
+class DynamicClassAttribute:
+    """Route attribute access on a class to __getattr__.
+
+    This is a descriptor, used to define attributes that act differently when
+    accessed through an instance and through a class.  Instance access remains
+    normal, but access to an attribute through a class will be routed to the
+    class's __getattr__ method; this is done by raising AttributeError.
+
+    This allows one to have properties active on an instance, and have virtual
+    attributes on the class with the same name (see Enum for an example).
+
+    """
+    def __init__(self, fget=None, fset=None, fdel=None, doc=None):
+        self.fget = fget
+        self.fset = fset
+        self.fdel = fdel
+        # next two lines make DynamicClassAttribute act the same as property
+        self.__doc__ = doc or fget.__doc__ or self.__doc__
+        self.overwrite_doc = doc is None
+        # support for abstract methods
+        self.__isabstractmethod__ = bool(getattr(fget, '__isabstractmethod__', False))
+
+    def __get__(self, instance, ownerclass=None):
+        if instance is None:
+            if self.__isabstractmethod__:
+                return self
+            raise AttributeError()
+        elif self.fget is None:
+            raise AttributeError("unreadable attribute")
+        return self.fget(instance)
+
+    def __set__(self, instance, value):
+        if self.fset is None:
+            raise AttributeError("can't set attribute")
+        self.fset(instance, value)
+
+    def __delete__(self, instance):
+        if self.fdel is None:
+            raise AttributeError("can't delete attribute")
+        self.fdel(instance)
+
+    def getter(self, fget):
+        fdoc = fget.__doc__ if self.overwrite_doc else None
+        result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__)
+        result.overwrite_doc = self.overwrite_doc
+        return result
+
+    def setter(self, fset):
+        result = type(self)(self.fget, fset, self.fdel, self.__doc__)
+        result.overwrite_doc = self.overwrite_doc
+        return result
+
+    def deleter(self, fdel):
+        result = type(self)(self.fget, self.fset, fdel, self.__doc__)
+        result.overwrite_doc = self.overwrite_doc
+        return result