From 5d53f92a0ce8547e23ef40ab49b307d09d2cc251 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Sun, 1 Sep 2024 18:00:11 -0700 Subject: [PATCH] Fix all class members missing when documenting a module with the same name as a standard library module Fixes #478 --- autoapi/_astroid_utils.py | 30 ------- autoapi/_mapper.py | 28 +++++++ autoapi/_parser.py | 113 ++++++++++++++++---------- docs/changes/478.bugfix.rst | 4 + tests/python/conftest.py | 2 - tests/python/pystdlib/conf.py | 24 ++++++ tests/python/pystdlib/index.rst | 25 ++++++ tests/python/pystdlib/stdlib/myast.py | 10 +++ tests/python/pystdlib/stdlib/myssl.py | 10 +++ tests/python/pystdlib/stdlib/ssl.py | 9 ++ tests/python/test_pyintegration.py | 49 ++++++++++- 11 files changed, 226 insertions(+), 78 deletions(-) create mode 100644 docs/changes/478.bugfix.rst create mode 100644 tests/python/pystdlib/conf.py create mode 100644 tests/python/pystdlib/index.rst create mode 100644 tests/python/pystdlib/stdlib/myast.py create mode 100644 tests/python/pystdlib/stdlib/myssl.py create mode 100644 tests/python/pystdlib/stdlib/ssl.py diff --git a/autoapi/_astroid_utils.py b/autoapi/_astroid_utils.py index e3d073e4..4547a2e2 100644 --- a/autoapi/_astroid_utils.py +++ b/autoapi/_astroid_utils.py @@ -644,36 +644,6 @@ def get_return_annotation(node: astroid.nodes.FunctionDef) -> str | None: return return_annotation -def get_func_docstring(node: astroid.nodes.FunctionDef) -> str: - """Get the docstring of a node, using a parent docstring if needed. - - Args: - node: The node to get a docstring for. - - Returns: - The docstring of the function, or the empty string if no docstring - was found or defined. - """ - doc = node.doc_node.value if node.doc_node else "" - - if not doc and isinstance(node.parent, astroid.nodes.ClassDef): - for base in node.parent.ancestors(): - if node.name in ("__init__", "__new__"): - base_module = base.qname().split(".", 1)[0] - if in_stdlib(base_module): - continue - - for child in base.get_children(): - if ( - isinstance(child, node.__class__) - and child.name == node.name - and child.doc_node is not None - ): - return child.doc_node.value - - return doc - - def get_class_docstring(node: astroid.nodes.ClassDef) -> str: """Get the docstring of a node, using a parent docstring if needed. diff --git a/autoapi/_mapper.py b/autoapi/_mapper.py index 75d8d7a0..e798f142 100644 --- a/autoapi/_mapper.py +++ b/autoapi/_mapper.py @@ -5,6 +5,7 @@ import operator import os import re +import sys from jinja2 import Environment, FileSystemLoader import sphinx @@ -31,6 +32,14 @@ ) from .settings import OWN_PAGE_LEVELS, TEMPLATE_DIR +if sys.version_info < (3, 10): # PY310 + from stdlib_list import in_stdlib +else: + + def in_stdlib(module_name: str) -> bool: + return module_name in sys.stdlib_module_names + + LOGGER = sphinx.util.logging.getLogger(__name__) @@ -451,6 +460,24 @@ def read_file(self, path, **kwargs): ) return None + def _skip_if_stdlib(self): + documented_modules = {obj["full_name"] for obj in self.paths.values()} + + q = collections.deque(self.paths.values()) + while q: + obj = q.popleft() + if "children" in obj: + q.extend(obj["children"]) + + if obj.get("inherited", False): + module = obj["inherited_from"]["full_name"].split(".", 1)[0] + if ( + in_stdlib(module) + and not obj["inherited_from"]["is_abstract"] + and module not in documented_modules + ): + obj["hide"] = True + def _resolve_placeholders(self): """Resolve objects that have been imported from elsewhere.""" modules = {} @@ -477,6 +504,7 @@ def _hide_yo_kids(self): child["hide"] = True def map(self, options=None): + self._skip_if_stdlib() self._resolve_placeholders() self._hide_yo_kids() self.app.env.autoapi_annotations = {} diff --git a/autoapi/_parser.py b/autoapi/_parser.py index 0ca92130..ff28e1d9 100644 --- a/autoapi/_parser.py +++ b/autoapi/_parser.py @@ -1,7 +1,5 @@ import collections -import itertools import os -import sys import astroid import astroid.builder @@ -10,14 +8,6 @@ from . import _astroid_utils -if sys.version_info < (3, 10): # PY310 - from stdlib_list import in_stdlib -else: - - def in_stdlib(module_name: str) -> bool: - return module_name in sys.stdlib_module_names - - def _prepare_docstring(doc): return "\n".join(sphinx.util.docstrings.prepare_docstring(doc)) @@ -117,55 +107,69 @@ def _parse_assign(self, node): return [data] - def parse_classdef(self, node, data=None): + def _parse_classdef(self, node, use_name_stacks): + if use_name_stacks: + qual_name = self._get_qual_name(node.name) + full_name = self._get_full_name(node.name) + + self._qual_name_stack.append(node.name) + self._full_name_stack.append(node.name) + else: + qual_name = node.qname()[len(node.root().qname()) + 1 :] + full_name = node.qname() + type_ = "class" if _astroid_utils.is_exception(node): type_ = "exception" - basenames = list(_astroid_utils.get_full_basenames(node)) - data = { "type": type_, "name": node.name, - "qual_name": self._get_qual_name(node.name), - "full_name": self._get_full_name(node.name), - "bases": basenames, + "qual_name": qual_name, + "full_name": full_name, + "bases": list(_astroid_utils.get_full_basenames(node)), "doc": _prepare_docstring(_astroid_utils.get_class_docstring(node)), "from_line_no": node.fromlineno, "to_line_no": node.tolineno, "children": [], + "is_abstract": _astroid_utils.is_abstract_class(node), } - self._qual_name_stack.append(node.name) - self._full_name_stack.append(node.name) - overridden = set() overloads = {} + for child in node.get_children(): + children_data = self.parse(child) + for child_data in children_data: + if _parse_child(child_data, overloads): + data["children"].append(child_data) + + data["children"] = list(self._resolve_inheritance(data)) + + return data + + def _resolve_inheritance(self, *mro_data): + overridden = set() children = {} - for base in itertools.chain(iter((node,)), node.ancestors()): + for i, cls_data in enumerate(mro_data): seen = set() base_children = [] + overloads = {} - # Don't document members inherited from standard library classes - # unless that class is abstract. - base_module = base.qname().split(".", 1)[0] - if in_stdlib(base_module) and not _astroid_utils.is_abstract_class(base): - continue - - for child in base.get_children(): - children_data = self.parse(child) - for child_data in children_data: - name = child_data["name"] + for child_data in cls_data["children"]: + name = child_data["name"] - existing_child = children.get(name) - if existing_child and not existing_child["doc"]: - existing_child["doc"] = child_data["doc"] + existing_child = children.get(name) + if existing_child and not existing_child["doc"]: + existing_child["doc"] = child_data["doc"] - if name in overridden: - continue + if name in overridden: + continue - seen.add(name) - if _parse_child(node, child_data, overloads, base): - base_children.append(child_data) + seen.add(name) + if _parse_child(child_data, overloads): + base_children.append(child_data) + child_data["inherited"] = i != 0 + if child_data["inherited"]: + child_data["inherited_from"] = cls_data overridden.update(seen) @@ -185,7 +189,29 @@ def parse_classdef(self, node, data=None): children[base_child["name"]] = base_child - data["children"].extend(children.values()) + return children.values() + + def _relevant_ancestors(self, node): + for base in node.ancestors(): + if base.qname() in ( + "__builtins__.object", + "builtins.object", + "builtins.type", + ): + continue + + yield base + + def parse_classdef(self, node): + data = self._parse_classdef(node, use_name_stacks=True) + + ancestors = self._relevant_ancestors(node) + ancestor_data = [ + self._parse_classdef(base, use_name_stacks=False) for base in ancestors + ] + if ancestor_data: + data["children"] = list(self._resolve_inheritance(data, *ancestor_data)) + self._qual_name_stack.pop() self._full_name_stack.pop() @@ -227,7 +253,7 @@ def parse_functiondef(self, node): "qual_name": self._get_qual_name(node.name), "full_name": self._get_full_name(node.name), "args": _astroid_utils.get_args_info(node.args), - "doc": _prepare_docstring(_astroid_utils.get_func_docstring(node)), + "doc": _prepare_docstring(node.doc_node.value if node.doc_node else ""), "from_line_no": node.fromlineno, "to_line_no": node.tolineno, "return_annotation": _astroid_utils.get_return_annotation(node), @@ -302,7 +328,7 @@ def parse_module(self, node): children_data = self.parse(child) for child_data in children_data: - if _parse_child(node, child_data, overloads): + if _parse_child(child_data, overloads): data["children"].append(child_data) return data @@ -352,7 +378,7 @@ def parse(self, node): return data -def _parse_child(node, child_data, overloads, base=None) -> bool: +def _parse_child(child_data, overloads) -> bool: if child_data["type"] in ("function", "method", "property"): name = child_data["name"] if name in overloads: @@ -369,7 +395,4 @@ def _parse_child(node, child_data, overloads, base=None) -> bool: if child_data["is_overload"] and name not in overloads: overloads[name] = child_data - if base: - child_data["inherited"] = base is not node - return True diff --git a/docs/changes/478.bugfix.rst b/docs/changes/478.bugfix.rst new file mode 100644 index 00000000..f8bf4f41 --- /dev/null +++ b/docs/changes/478.bugfix.rst @@ -0,0 +1,4 @@ +Fix all class members missing when documenting a module with the same name as a standard library module + +Members inherited from the standard library can also have their skip value +overridden by autoapi-skip-member. diff --git a/tests/python/conftest.py b/tests/python/conftest.py index 7daa5721..8d5864a5 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -1,8 +1,6 @@ -import io import os import pathlib import shutil -from unittest.mock import call from bs4 import BeautifulSoup import pytest diff --git a/tests/python/pystdlib/conf.py b/tests/python/pystdlib/conf.py new file mode 100644 index 00000000..739b8c9a --- /dev/null +++ b/tests/python/pystdlib/conf.py @@ -0,0 +1,24 @@ +templates_path = ["_templates"] +source_suffix = ".rst" +master_doc = "index" +project = "pystdlib" +copyright = "2015, readthedocs" +author = "readthedocs" +version = "0.1" +release = "0.1" +language = "en" +exclude_patterns = ["_build"] +pygments_style = "sphinx" +todo_include_todos = False +html_theme = "alabaster" +htmlhelp_basename = "pystdlibdoc" +extensions = ["autoapi.extension"] +autoapi_dirs = ["stdlib"] +autoapi_file_pattern = "*.py" +autoapi_keep_files = True +autoapi_options = [ + "members", + "undoc-members" "special-members", + "imported-members", + "inherited-members", +] diff --git a/tests/python/pystdlib/index.rst b/tests/python/pystdlib/index.rst new file mode 100644 index 00000000..cf6549b1 --- /dev/null +++ b/tests/python/pystdlib/index.rst @@ -0,0 +1,25 @@ +.. pypackageexample documentation master file, created by + sphinx-quickstart on Fri May 29 13:34:37 2015. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to pypackageexample's documentation! +============================================ + +.. toctree:: + + autoapi/index + +Contents: + +.. toctree:: + :maxdepth: 2 + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tests/python/pystdlib/stdlib/myast.py b/tests/python/pystdlib/stdlib/myast.py new file mode 100644 index 00000000..bc83729c --- /dev/null +++ b/tests/python/pystdlib/stdlib/myast.py @@ -0,0 +1,10 @@ +"""This is a docstring.""" + +import ast + + +class MyVisitor(ast.NodeVisitor): + """My custom visitor.""" + + def my_visit(self): + """My visit method.""" diff --git a/tests/python/pystdlib/stdlib/myssl.py b/tests/python/pystdlib/stdlib/myssl.py new file mode 100644 index 00000000..a3d41179 --- /dev/null +++ b/tests/python/pystdlib/stdlib/myssl.py @@ -0,0 +1,10 @@ +"""This is a docstring.""" + +from ssl import SSLContext + + +class MySSLContext(SSLContext): + """This is a class.""" + + def my_method(self): + """This is a method.""" diff --git a/tests/python/pystdlib/stdlib/ssl.py b/tests/python/pystdlib/stdlib/ssl.py new file mode 100644 index 00000000..dfb311e7 --- /dev/null +++ b/tests/python/pystdlib/stdlib/ssl.py @@ -0,0 +1,9 @@ +"""ssl is the same name as a stdlib module.""" + + +class SSLContext: + """Do things with ssl.""" + + def wrap_socket(self, sock): + """Wrap a socket.""" + return sock diff --git a/tests/python/test_pyintegration.py b/tests/python/test_pyintegration.py index 28e16ca0..0bda9ffa 100644 --- a/tests/python/test_pyintegration.py +++ b/tests/python/test_pyintegration.py @@ -1,4 +1,3 @@ -import io import os import pathlib import sys @@ -1243,3 +1242,51 @@ def test_nothing_to_render_raises_warning(builder): builder("pynorender", warningiserror=True) assert "No modules were rendered" in str(exc_info.value) + + +class TestStdLib: + """Check that modules with standard library names are still documented.""" + + @pytest.fixture(autouse=True) + def built(self, builder): + builder("pystdlib") + + def test_integration(self, parse): + ssl_file = parse("_build/html/autoapi/ssl/index.html") + + ssl_mod = ssl_file.find(id="ssl") + assert "ssl is the same name" in ssl_mod.parent.text + + assert ssl_file.find(id="ssl.SSLContext") + wrap_socket = ssl_file.find(id="ssl.SSLContext.wrap_socket") + assert wrap_socket + wrap_docstring = wrap_socket.parent.find("dd").text.strip() + assert wrap_docstring == "Wrap a socket." + + myssl_file = parse("_build/html/autoapi/myssl/index.html") + + # Find members that are not inherited from local standard library classes. + ctx = myssl_file.find(id="myssl.MySSLContext") + assert ctx + meth = myssl_file.find(id="myssl.MySSLContext.my_method") + assert meth + meth_docstring = meth.parent.find("dd").text.strip() + assert meth_docstring == "This is a method." + + # Find members that are inherited from local standard library classes. + wrap_socket = myssl_file.find(id="myssl.MySSLContext.wrap_socket") + assert not wrap_socket + + myast_file = parse("_build/html/autoapi/myast/index.html") + + # Find members that are not inherited from standard library classes. + visitor = myast_file.find(id="myast.MyVisitor") + assert visitor + meth = myast_file.find(id="myast.MyVisitor.my_visit") + assert meth + meth_docstring = meth.parent.find("dd").text.strip() + assert meth_docstring == "My visit method." + + # Find members that are inherited from standard library classes. + meth = myast_file.find(id="myast.MyVisitor.visit") + assert not meth