Issue #19030: final pieces for proper location of various class attributes located in the metaclass.

Okay, hopefully the very last patch for this issue.  :/

I realized when playing with Enum that the metaclass attributes weren't always displayed properly.

New patch properly locates DynamicClassAttributes, virtual class attributes (returned by __getattr__ and friends), and metaclass class attributes (if they are also in the metaclass __dir__ method).

Also had to change one line in pydoc to get this to work.

Added tests in test_inspect and test_pydoc to cover these situations.

Lib/inspect.py

@@ -269,9 +269,9 @@ def getmembers(object, predicate=None):
     results = []
     processed = set()
     names = dir(object)
-    # add any virtual attributes to the list of names if object is a class
+    # :dd any DynamicClassAttributes to the list of names if object is a class;
     # this may result in duplicate entries if, for example, a virtual
-    # attribute with the same name as a member property exists
+    # attribute with the same name as a DynamicClassAttribute exists
     try:
         for base in object.__bases__:
             for k, v in base.__dict__.items():
@@ -329,79 +329,88 @@ def classify_class_attrs(cls):
 
     If one of the items in dir(cls) is stored in the metaclass it will now
     be discovered and not have None be listed as the class in which it was
-    defined.
+    defined.  Any items whose home class cannot be discovered are skipped.
     """
 
     mro = getmro(cls)
     metamro = getmro(type(cls)) # for attributes stored in the metaclass
     metamro = tuple([cls for cls in metamro if cls not in (type, object)])
-    possible_bases = (cls,) + mro + metamro
+    class_bases = (cls,) + mro
+    all_bases = class_bases + metamro
     names = dir(cls)
-    # add any virtual attributes to the list of names
+    # :dd any DynamicClassAttributes to the list of names;
     # this may result in duplicate entries if, for example, a virtual
-    # attribute with the same name as a member property exists
+    # attribute with the same name as a DynamicClassAttribute exists.
     for base in mro:
         for k, v in base.__dict__.items():
             if isinstance(v, types.DynamicClassAttribute):
                 names.append(k)
     result = []
     processed = set()
-    sentinel = object()
+
     for name in names:
         # Get the object associated with the name, and where it was defined.
         # Normal objects will be looked up with both getattr and directly in
         # its class' dict (in case getattr fails [bug #1785], and also to look
         # for a docstring).
-        # For VirtualAttributes on the second pass we only look in the
+        # For DynamicClassAttributes on the second pass we only look in the
         # class's dict.
         #
         # Getting an obj from the __dict__ sometimes reveals more than
         # using getattr.  Static and class methods are dramatic examples.
         homecls = None
-        get_obj = sentinel
+        get_obj = None
-        dict_obj = sentinel
+        dict_obj = None
         if name not in processed:
             try:
                 if name == '__dict__':
-                    raise Exception("__dict__ is special, we don't want the proxy")
+                    raise Exception("__dict__ is special, don't want the proxy")
                 get_obj = getattr(cls, name)
             except Exception as exc:
                 pass
             else:
                 homecls = getattr(get_obj, "__objclass__", homecls)
-                if homecls not in possible_bases:
+                if homecls not in class_bases:
                     # if the resulting object does not live somewhere in the
                     # mro, drop it and search the mro manually
                     homecls = None
                     last_cls = None
-                    last_obj = None
+                    # first look in the classes
-                    for srch_cls in ((cls,) + mro):
+                    for srch_cls in class_bases:
                         srch_obj = getattr(srch_cls, name, None)
-                        if srch_obj is get_obj:
+                        if srch_obj == get_obj:
+                            last_cls = srch_cls
+                    # then check the metaclasses
+                    for srch_cls in metamro:
+                        try:
+                            srch_obj = srch_cls.__getattr__(cls, name)
+                        except AttributeError:
+                            continue
+                        if srch_obj == get_obj:
                             last_cls = srch_cls
-                            last_obj = srch_obj
                     if last_cls is not None:
                         homecls = last_cls
-        for base in possible_bases:
+        for base in all_bases:
             if name in base.__dict__:
                 dict_obj = base.__dict__[name]
-                homecls = homecls or base
+                if homecls not in metamro:
+                    homecls = base
                 break
         if homecls is None:
             # unable to locate the attribute anywhere, most likely due to
             # buggy custom __dir__; discard and move on
             continue
+        obj = get_obj or dict_obj
         # Classify the object or its descriptor.
-        if get_obj is not sentinel:
-            obj = get_obj
-        else:
-            obj = dict_obj
         if isinstance(dict_obj, staticmethod):
             kind = "static method"
+            obj = dict_obj
         elif isinstance(dict_obj, classmethod):
             kind = "class method"
-        elif isinstance(obj, property):
+            obj = dict_obj
+        elif isinstance(dict_obj, property):
             kind = "property"
+            obj = dict_obj
         elif isfunction(obj) or ismethoddescriptor(obj):
             kind = "method"
         else:

Lib/pydoc.py

@@ -1235,7 +1235,8 @@ def spilldata(msg, attrs, predicate):
                         doc = getdoc(value)
                     else:
                         doc = None
-                    push(self.docother(getattr(object, name),
+                    push(self.docother(
+                        getattr(object, name, None) or homecls.__dict__[name],
                         name, mod, maxlen=70, doc=doc) + '\n')
             return attrs
 
@@ -1258,7 +1259,6 @@ def spilldata(msg, attrs, predicate):
             else:
                 tag = "inherited from %s" % classname(thisclass,
                                                       object.__module__)
-
             # Sort attrs by name.
             attrs.sort()
 
@@ -1273,6 +1273,7 @@ def spilldata(msg, attrs, predicate):
                                      lambda t: t[1] == 'data descriptor')
             attrs = spilldata("Data and other attributes %s:\n" % tag, attrs,
                               lambda t: t[1] == 'data')
+
             assert attrs == []
             attrs = inherited
 

Lib/test/test_inspect.py

@@ -667,9 +667,19 @@ def ham(self):
                 return 'eggs'
         should_find_dca = inspect.Attribute('ham', 'data', VA, VA.__dict__['ham'])
         self.assertIn(should_find_dca, inspect.classify_class_attrs(VA))
-        should_find_ga = inspect.Attribute('ham', 'data', VA, 'spam')
+        should_find_ga = inspect.Attribute('ham', 'data', Meta, 'spam')
         self.assertIn(should_find_ga, inspect.classify_class_attrs(VA))
 
+    def test_classify_metaclass_class_attribute(self):
+        class Meta(type):
+            fish = 'slap'
+            def __dir__(self):
+                return ['__class__', '__modules__', '__name__', 'fish']
+        class Class(metaclass=Meta):
+            pass
+        should_find = inspect.Attribute('fish', 'data', Meta, 'slap')
+        self.assertIn(should_find, inspect.classify_class_attrs(Class))
+
     def test_classify_VirtualAttribute(self):
         class Meta(type):
             def __dir__(cls):
@@ -680,7 +690,7 @@ def __getattr__(self, name):
                 return super().__getattr(name)
         class Class(metaclass=Meta):
             pass
-        should_find = inspect.Attribute('BOOM', 'data', Class, 42)
+        should_find = inspect.Attribute('BOOM', 'data', Meta, 42)
         self.assertIn(should_find, inspect.classify_class_attrs(Class))
 
     def test_classify_VirtualAttribute_multi_classes(self):
@@ -711,9 +721,9 @@ class Class1(metaclass=Meta1):
         class Class2(Class1, metaclass=Meta3):
             pass
 
-        should_find1 = inspect.Attribute('one', 'data', Class1, 1)
+        should_find1 = inspect.Attribute('one', 'data', Meta1, 1)
-        should_find2 = inspect.Attribute('two', 'data', Class2, 2)
+        should_find2 = inspect.Attribute('two', 'data', Meta2, 2)
-        should_find3 = inspect.Attribute('three', 'data', Class2, 3)
+        should_find3 = inspect.Attribute('three', 'data', Meta3, 3)
         cca = inspect.classify_class_attrs(Class2)
         for sf in (should_find1, should_find2, should_find3):
             self.assertIn(sf, cca)

Lib/test/test_pydoc.py

@@ -11,6 +11,7 @@
 import string
 import test.support
 import time
+import types
 import unittest
 import xml.etree
 import textwrap
@@ -208,6 +209,77 @@ class B(builtins.object)
 # output pattern for module with bad imports
 badimport_pattern = "problem in %s - ImportError: No module named %r"
 
+expected_dynamicattribute_pattern = """
+Help on class DA in module %s:
+
+class DA(builtins.object)
+ |  Data descriptors defined here:
+ |
+ |  __dict__
+ |      dictionary for instance variables (if defined)
+ |
+ |  __weakref__
+ |      list of weak references to the object (if defined)
+ |
+ |  ham
+ |
+ |  ----------------------------------------------------------------------
+ |  Data and other attributes inherited from Meta:
+ |
+ |  ham = 'spam'
+""".strip()
+
+expected_virtualattribute_pattern1 = """
+Help on class Class in module %s:
+
+class Class(builtins.object)
+ |  Data and other attributes inherited from Meta:
+ |
+ |  LIFE = 42
+""".strip()
+
+expected_virtualattribute_pattern2 = """
+Help on class Class1 in module %s:
+
+class Class1(builtins.object)
+ |  Data and other attributes inherited from Meta1:
+ |
+ |  one = 1
+""".strip()
+
+expected_virtualattribute_pattern3 = """
+Help on class Class2 in module %s:
+
+class Class2(Class1)
+ |  Method resolution order:
+ |      Class2
+ |      Class1
+ |      builtins.object
+ |
+ |  Data and other attributes inherited from Meta1:
+ |
+ |  one = 1
+ |
+ |  ----------------------------------------------------------------------
+ |  Data and other attributes inherited from Meta3:
+ |
+ |  three = 3
+ |
+ |  ----------------------------------------------------------------------
+ |  Data and other attributes inherited from Meta2:
+ |
+ |  two = 2
+""".strip()
+
+expected_missingattribute_pattern = """
+Help on class C in module %s:
+
+class C(builtins.object)
+ |  Data and other attributes defined here:
+ |
+ |  here = 'present!'
+""".strip()
+
 def run_pydoc(module_name, *args, **env):
     """
     Runs pydoc on the specified module. Returns the stripped
@@ -636,6 +708,108 @@ def test_keywords(self):
         self.assertEqual(sorted(pydoc.Helper.keywords),
                          sorted(keyword.kwlist))
 
+class PydocWithMetaClasses(unittest.TestCase):
+    def test_DynamicClassAttribute(self):
+        class Meta(type):
+            def __getattr__(self, name):
+                if name == 'ham':
+                    return 'spam'
+                return super().__getattr__(name)
+        class DA(metaclass=Meta):
+            @types.DynamicClassAttribute
+            def ham(self):
+                return 'eggs'
+        output = StringIO()
+        helper = pydoc.Helper(output=output)
+        helper(DA)
+        expected_text = expected_dynamicattribute_pattern % __name__
+        result = output.getvalue().strip()
+        if result != expected_text:
+            print_diffs(expected_text, result)
+            self.fail("outputs are not equal, see diff above")
+
+    def test_virtualClassAttributeWithOneMeta(self):
+        class Meta(type):
+            def __dir__(cls):
+                return ['__class__', '__module__', '__name__', 'LIFE']
+            def __getattr__(self, name):
+                if name =='LIFE':
+                    return 42
+                return super().__getattr(name)
+        class Class(metaclass=Meta):
+            pass
+        output = StringIO()
+        helper = pydoc.Helper(output=output)
+        helper(Class)
+        expected_text = expected_virtualattribute_pattern1 % __name__
+        result = output.getvalue().strip()
+        if result != expected_text:
+            print_diffs(expected_text, result)
+            self.fail("outputs are not equal, see diff above")
+
+    def test_virtualClassAttributeWithTwoMeta(self):
+        class Meta1(type):
+            def __dir__(cls):
+                return ['__class__', '__module__', '__name__', 'one']
+            def __getattr__(self, name):
+                if name =='one':
+                    return 1
+                return super().__getattr__(name)
+        class Meta2(type):
+            def __dir__(cls):
+                return ['__class__', '__module__', '__name__', 'two']
+            def __getattr__(self, name):
+                if name =='two':
+                    return 2
+                return super().__getattr__(name)
+        class Meta3(Meta1, Meta2):
+            def __dir__(cls):
+                return list(sorted(set(
+                    ['__class__', '__module__', '__name__', 'three'] +
+                    Meta1.__dir__(cls) + Meta2.__dir__(cls))))
+            def __getattr__(self, name):
+                if name =='three':
+                    return 3
+                return super().__getattr__(name)
+        class Class1(metaclass=Meta1):
+            pass
+        class Class2(Class1, metaclass=Meta3):
+            pass
+        fail1 = fail2 = False
+        output = StringIO()
+        helper = pydoc.Helper(output=output)
+        helper(Class1)
+        expected_text1 = expected_virtualattribute_pattern2 % __name__
+        result1 = output.getvalue().strip()
+        if result1 != expected_text1:
+            print_diffs(expected_text1, result1)
+            fail1 = True
+        output = StringIO()
+        helper = pydoc.Helper(output=output)
+        helper(Class2)
+        expected_text2 = expected_virtualattribute_pattern3 % __name__
+        result2 = output.getvalue().strip()
+        if result2 != expected_text2:
+            print_diffs(expected_text2, result2)
+            fail2 = True
+        if fail1 or fail2:
+            self.fail("outputs are not equal, see diff above")
+
+    def test_buggy_dir(self):
+        class M(type):
+            def __dir__(cls):
+                return ['__class__', '__name__', 'missing', 'here']
+        class C(metaclass=M):
+            here = 'present!'
+        output = StringIO()
+        helper = pydoc.Helper(output=output)
+        helper(C)
+        expected_text = expected_missingattribute_pattern % __name__
+        result = output.getvalue().strip()
+        if result != expected_text:
+            print_diffs(expected_text, result)
+            self.fail("outputs are not equal, see diff above")
+
 @reap_threads
 def test_main():
     try:
@@ -645,6 +819,7 @@ def test_main():
                                   PydocServerTest,
                                   PydocUrlHandlerTest,
                                   TestHelper,
+                                  PydocWithMetaClasses,
                                   )
     finally:
         reap_children()