stubgenc: add support for including class and property docstrings (#17964)

<!-- If this pull request fixes an issue, add "Fixes #NNN" with the
issue number. -->

Prior to this change passing `--include-docstrings` did not generate
docstrings for classes or properties, only functions. This PR brings
c-extensions up to parity with pure-python modules.

I used this feature to generate stubs for this project:
https://github.com/LumaPictures/cg-stubs/blob/master/usd/stubs/pxr/Usd/__init__.pyi

<!--
Checklist:
- Read the [Contributing
Guidelines](https://github.com/python/mypy/blob/master/CONTRIBUTING.md)
- Add tests for all changed behaviour.
- If you can't add a test, please explain why and how you verified your
changes work.
- Make sure CI passes.
- Please do not force push to the PR once it has been reviewed.
-->

Author: chadrik

mypy/stubdoc.py

@@ -78,6 +78,7 @@ class FunctionSig(NamedTuple):
     args: list[ArgSig]
     ret_type: str | None
     type_args: str = ""  # TODO implement in stubgenc and remove the default
+    docstring: str | None = None
 
     def is_special_method(self) -> bool:
         return bool(
@@ -110,6 +111,7 @@ def format_sig(
         is_async: bool = False,
         any_val: str | None = None,
         docstring: str | None = None,
+        include_docstrings: bool = False,
     ) -> str:
         args: list[str] = []
         for arg in self.args:
@@ -144,8 +146,11 @@ def format_sig(
 
         prefix = "async " if is_async else ""
         sig = f"{indent}{prefix}def {self.name}{self.type_args}({', '.join(args)}){retfield}:"
-        if docstring:
-            suffix = f"\n{indent}    {mypy.util.quote_docstring(docstring)}"
+        # if this object has a docstring it's probably produced by a SignatureGenerator, so it
+        # takes precedence over the passed docstring, which acts as a fallback.
+        doc = (self.docstring or docstring) if include_docstrings else None
+        if doc:
+            suffix = f"\n{indent}    {mypy.util.quote_docstring(doc)}"
         else:
             suffix = " ..."
         return f"{sig}{suffix}"

mypy/stubgenc.py

@@ -38,6 +38,7 @@
     infer_method_arg_types,
     infer_method_ret_type,
 )
+from mypy.util import quote_docstring
 
 
 class ExternalSignatureGenerator(SignatureGenerator):
@@ -649,8 +650,7 @@ def generate_function_stub(
                 if inferred[0].args and inferred[0].args[0].name == "cls":
                     decorators.append("@classmethod")
 
-        if docstring:
-            docstring = self._indent_docstring(docstring)
+        docstring = self._indent_docstring(ctx.docstring) if ctx.docstring else None
         output.extend(self.format_func_def(inferred, decorators=decorators, docstring=docstring))
         self._fix_iter(ctx, inferred, output)
 
@@ -754,9 +754,14 @@ def generate_property_stub(
             )
         else:  # regular property
             if readonly:
+                docstring = self._indent_docstring(ctx.docstring) if ctx.docstring else None
                 ro_properties.append(f"{self._indent}@property")
-                sig = FunctionSig(name, [ArgSig("self")], inferred_type)
-                ro_properties.append(sig.format_sig(indent=self._indent))
+                sig = FunctionSig(name, [ArgSig("self")], inferred_type, docstring=docstring)
+                ro_properties.append(
+                    sig.format_sig(
+                        indent=self._indent, include_docstrings=self._include_docstrings
+                    )
+                )
             else:
                 if inferred_type is None:
                     inferred_type = self.add_name("_typeshed.Incomplete")
@@ -875,8 +880,17 @@ def generate_class_stub(
             bases_str = "(%s)" % ", ".join(bases)
         else:
             bases_str = ""
-        if types or static_properties or rw_properties or methods or ro_properties:
+
+        if class_info.docstring and self._include_docstrings:
+            doc = quote_docstring(self._indent_docstring(class_info.docstring))
+            doc = f"    {self._indent}{doc}"
+            docstring = doc.splitlines(keepends=False)
+        else:
+            docstring = []
+
+        if docstring or types or static_properties or rw_properties or methods or ro_properties:
             output.append(f"{self._indent}class {class_name}{bases_str}:")
+            output.extend(docstring)
             for line in types:
                 if (
                     output
@@ -886,14 +900,10 @@ def generate_class_stub(
                 ):
                     output.append("")
                 output.append(line)
-            for line in static_properties:
-                output.append(line)
-            for line in rw_properties:
-                output.append(line)
-            for line in methods:
-                output.append(line)
-            for line in ro_properties:
-                output.append(line)
+            output.extend(static_properties)
+            output.extend(rw_properties)
+            output.extend(methods)
+            output.extend(ro_properties)
         else:
             output.append(f"{self._indent}class {class_name}{bases_str}: ...")
 

mypy/stubutil.py

@@ -803,7 +803,8 @@ def format_func_def(
                 signature.format_sig(
                     indent=self._indent,
                     is_async=is_coroutine,
-                    docstring=docstring if self._include_docstrings else None,
+                    docstring=docstring,
+                    include_docstrings=self._include_docstrings,
                 )
             )
         return lines

test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi

@@ -38,7 +38,10 @@ class TestStruct:
     def __init__(self, *args, **kwargs) -> None:
         """Initialize self.  See help(type(self)) for accurate signature."""
     @property
-    def field_readonly(self) -> int: ...
+    def field_readonly(self) -> int:
+        """some docstring
+        (arg0: pybind11_fixtures.TestStruct) -> int
+        """
 
 def func_incomplete_signature(*args, **kwargs):
     """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding"""

test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi

@@ -5,6 +5,11 @@ __version__: str
 
 class Point:
     class AngleUnit:
+        """Members:
+
+          radian
+
+          degree"""
         __members__: ClassVar[dict] = ...  # read-only
         __entries: ClassVar[dict] = ...
         degree: ClassVar[Point.AngleUnit] = ...
@@ -22,11 +27,23 @@ class Point:
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
         @property
-        def name(self) -> str: ...
+        def name(self) -> str:
+            """name(self: handle) -> str
+
+            name(self: handle) -> str
+            """
         @property
-        def value(self) -> int: ...
+        def value(self) -> int:
+            """(arg0: pybind11_fixtures.demo.Point.AngleUnit) -> int"""
 
     class LengthUnit:
+        """Members:
+
+          mm
+
+          pixel
+
+          inch"""
         __members__: ClassVar[dict] = ...  # read-only
         __entries: ClassVar[dict] = ...
         inch: ClassVar[Point.LengthUnit] = ...
@@ -45,9 +62,14 @@ class Point:
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
         @property
-        def name(self) -> str: ...
+        def name(self) -> str:
+            """name(self: handle) -> str
+
+            name(self: handle) -> str
+            """
         @property
-        def value(self) -> int: ...
+        def value(self) -> int:
+            """(arg0: pybind11_fixtures.demo.Point.LengthUnit) -> int"""
     angle_unit: ClassVar[Point.AngleUnit] = ...
     length_unit: ClassVar[Point.LengthUnit] = ...
     x_axis: ClassVar[Point] = ...  # read-only
@@ -94,7 +116,8 @@ class Point:
         2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float
         """
     @property
-    def length(self) -> float: ...
+    def length(self) -> float:
+        """(arg0: pybind11_fixtures.demo.Point) -> float"""
 
 def answer() -> int:
     '''answer() -> int