From 731e8b4142a02318dda792262d5fe290d4400c58 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 6 Sep 2019 00:36:07 +0200 Subject: [PATCH] documentation/python: include documented underscored names. --- doc/documentation/python.rst | 43 +++++-- documentation/python.py | 86 +++++++++----- .../inspect_all_property/__init__.py | 2 +- .../inspect_string.DerivedException.html | 8 ++ .../inspect_string/inspect_string/__init__.py | 36 +----- .../inspect_string/_private_module.py | 1 - .../test_python/inspect_underscored/docs.rst | 33 ++++++ .../inspect_underscored.Class.html | 88 +++++++++++++++ .../inspect_underscored.html | 105 ++++++++++++++++++ .../inspect_underscored/__init__.py | 75 +++++++++++++ .../inspect_underscored/_submodule.py | 1 + .../_submodule_external.py | 0 .../_submodule_undocumented.py | 0 documentation/test_python/test_inspect.py | 11 ++ documentation/test_python/test_page.py | 4 +- 15 files changed, 420 insertions(+), 73 deletions(-) delete mode 100644 documentation/test_python/inspect_string/inspect_string/_private_module.py create mode 100644 documentation/test_python/inspect_underscored/docs.rst create mode 100644 documentation/test_python/inspect_underscored/inspect_underscored.Class.html create mode 100644 documentation/test_python/inspect_underscored/inspect_underscored.html create mode 100644 documentation/test_python/inspect_underscored/inspect_underscored/__init__.py create mode 100644 documentation/test_python/inspect_underscored/inspect_underscored/_submodule.py create mode 100644 documentation/test_python/inspect_underscored/inspect_underscored/_submodule_external.py create mode 100644 documentation/test_python/inspect_underscored/inspect_underscored/_submodule_undocumented.py diff --git a/doc/documentation/python.rst b/doc/documentation/python.rst index 15856bd0..c13a05ec 100644 --- a/doc/documentation/python.rst +++ b/doc/documentation/python.rst @@ -425,17 +425,17 @@ dashes: `Module inspection`_ ==================== -By default, if a module contains the :py:`__all__` attribute, all names listed -there are exposed in the documentation. Otherwise, all module (and class) -members are extracted using :py:`inspect.getmembers()`, skipping names -:py:`import`\ ed from elsewhere and underscored names. +By default, if a module contains the :py:`__all__` attribute, *all* names +listed there are exposed in the documentation. Otherwise, all module (and +class) members are extracted using :py:`inspect.getmembers()`, skipping names +:py:`import`\ ed from elsewhere and undocumented underscored names. Detecting if a module is a submodule of the current package or if it's :py:`import`\ ed from elsewhere is tricky, the script thus includes only -submodules that have their :py:`__package__` property the same or one level below -the parent package. If a module's :py:`__package__` is empty, it's assumed to -be a plain module (instead of a package) and since those can't have submodules, -all found submodules in it are ignored. +submodules that have their :py:`__package__` property the same or one level +below the parent package. If a module's :py:`__package__` is empty, it's +assumed to be a plain module (instead of a package) and since those can't have +submodules, all found submodules in it are ignored. .. block-success:: Overriding the set of included names, module reorganization @@ -623,6 +623,33 @@ non-obvious way to document enum values as well. The documentation output for enums includes enum value values and the class it was derived from, so it's possible to know whether it's an enum or a flag. +`Including underscored names in the output`_ +-------------------------------------------- + +By default, names starting with an underscore (except for :py:`__dunder__` +methods) are treated as private and not listed in the output. One way to expose +them is to list them in :py:`__all__`, however that works for module content +only. For exposing general underscored names, you either need to provide a +docstring or `external documentation content`_ (and in case of plain data, +external documentation content is the only option). + +Note that at the point where modules and classes are crawled for members, +docstrings are *not* parsed yet --- so e.g. a data documentation via a +:rst:`:data:` option of the :rst:`.. py:class::` `m.sphinx`_ directive won't be +visible to the initial crawl and thus the data will stay hidden. + +Sometimes, however, you'll want the inverse --- keeping an underscored name +hidden, even though it has a docstring. Solution is to remove the docstring +while generating the docs, directly in the ``conf.py`` file during module +import: + +.. code:: py + + import mymodule + mymodule._private_thing.__doc__ = None + + INPUT_MODULES = [mymodule] + `Pages`_ ======== diff --git a/documentation/python.py b/documentation/python.py index 3e039017..7ffeda0d 100755 --- a/documentation/python.py +++ b/documentation/python.py @@ -228,6 +228,32 @@ def object_type(state: State, object, name) -> EntryType: # caller should print a warning in this case return None # pragma: no cover +def is_docstring_useless(type: EntryType, docstring): + # Enum doc is by default set to a generic value. That's very useless. + if type == EntryType.ENUM and docstring == 'An enumeration.': return True + return not docstring or not docstring.strip() + +def is_underscored_and_undocumented(state: State, type, path, docstring): + if type == EntryType.MODULE: + external_docs = state.module_docs + elif type == EntryType.CLASS: + external_docs = state.class_docs + elif type == EntryType.ENUM: + external_docs = state.enum_docs + elif type in [EntryType.FUNCTION, EntryType.OVERLOADED_FUNCTION]: + external_docs = state.function_docs + elif type == EntryType.PROPERTY: + external_docs = state.property_docs + elif type == EntryType.DATA: + external_docs = state.data_docs + # Data don't have docstrings, those are from their type instead + docstring = None + else: + assert type is None, type + external_docs = {} + + return path[-1].startswith('_') and '.'.join(path) not in external_docs and is_docstring_useless(type, docstring) + # Builtin dunder functions have hardcoded docstrings. This is totally useless # to have in the docs, so filter them out. Uh... kinda ugly. _filtered_builtin_functions = set([ @@ -359,13 +385,13 @@ def crawl_class(state: State, path: List[str], class_): # name_map) if type == EntryType.CLASS: if name in ['__base__', '__class__']: continue # TODO - if name.startswith('_'): continue + if is_underscored_and_undocumented(state, type, subpath, object.__doc__): continue crawl_class(state, subpath, object) # Crawl enum values (they also add itself ot the name_map) elif type == EntryType.ENUM: - if name.startswith('_'): continue + if is_underscored_and_undocumented(state, type, subpath, object.__doc__): continue crawl_enum(state, subpath, object, class_entry.url) @@ -373,9 +399,11 @@ def crawl_class(state: State, path: List[str], class_): else: # Filter out private / unwanted members if type in [EntryType.FUNCTION, EntryType.OVERLOADED_FUNCTION]: - # Filter out underscored methods (but not dunder methods such - # as __init__) - if name.startswith('_') and not (name.startswith('__') and name.endswith('__')): continue + # Filter out undocumented underscored methods (but not dunder + # methods such as __init__) + # TODO: this won't look into docs saved under a signature but + # for that we'd need to parse the signature first, ugh + if not (name.startswith('__') and name.endswith('__')) and is_underscored_and_undocumented(state, type, subpath, object.__doc__): continue # Filter out dunder methods that ... if name.startswith('__'): # ... don't have their own docs @@ -394,9 +422,10 @@ def crawl_class(state: State, path: List[str], class_): pass elif type == EntryType.PROPERTY: if (name, object.__doc__) in _filtered_builtin_properties: continue - if name.startswith('_'): continue # TODO: are there any dunder props? + # TODO: are there any interesting dunder props? + if is_underscored_and_undocumented(state, type, subpath, object.__doc__): continue elif type == EntryType.DATA: - if name.startswith('_'): continue + if is_underscored_and_undocumented(state, type, subpath, object.__doc__): continue else: # pragma: no cover assert type is None; continue # ignore unknown object types @@ -414,7 +443,11 @@ def crawl_class(state: State, path: List[str], class_): # places. if state.config['ATTRS_COMPATIBILITY'] and hasattr(class_, '__attrs_attrs__'): for attrib in class_.__attrs_attrs__: - if attrib.name.startswith('_'): continue + subpath = path + [attrib.name] + + # No docstrings for attrs (the best we could get would be a + # docstring of the variable type, nope to that) + if is_underscored_and_undocumented(state, EntryType.PROPERTY, subpath, None): continue # In some cases, the attribute can be present also among class # data (for example when using slots). Prefer the info provided by @@ -423,8 +456,6 @@ def crawl_class(state: State, path: List[str], class_): if attrib.name not in class_entry.members: class_entry.members += [attrib.name] - subpath = path + [attrib.name] - entry = Empty() entry.type = EntryType.PROPERTY entry.object = attrib @@ -541,9 +572,6 @@ def crawl_module(state: State, path: List[str], module) -> List[Tuple[List[str], # have __module__ equivalent to `path`. else: for name, object in inspect.getmembers(module): - # Filter out underscored names - if name.startswith('_'): continue - # If this is not a module, check if the enclosing module of the # object is what expected. If not, it's a class/function/... # imported from elsewhere and we don't want those. @@ -582,9 +610,12 @@ def crawl_module(state: State, path: List[str], module) -> List[Tuple[List[str], type_ = object_type(state, object, name) subpath = path + [name] + # Filter out undocumented underscored names + if is_underscored_and_undocumented(state, type_, subpath, object.__doc__): continue + # Crawl the submodules and subclasses recursively (they also add # itself to the name_map), add other members directly. - if not type_: # pragma: no cover + if type_ is None: # pragma: no cover # Ignore unknown object types (with __all__ we warn instead) continue elif type_ == EntryType.MODULE: @@ -1158,7 +1189,7 @@ def extract_enum_doc(state: State, entry: Empty): # The happy case if issubclass(entry.object, enum.Enum): # Enum doc is by default set to a generic value. That's useless as well. - if entry.object.__doc__ == 'An enumeration.': + if is_docstring_useless(EntryType.ENUM, entry.object.__doc__): docstring = '' else: docstring = entry.object.__doc__ @@ -1599,13 +1630,13 @@ def extract_property_doc(state: State, parent, entry: Empty): # fget / fset / fdel, instead we need to look into __get__ / __set__ / # __delete__ directly. This is fairly rare (datetime.date is one and # BaseException.args is another I could find), so don't bother with it much - # --- assume readonly and no docstrings / annotations whatsoever. + # --- assume readonly. Some docstrings are there for properties; see the + # inspect_string.DerivedException test class for details. if entry.object.__class__.__name__ == 'getset_descriptor' and entry.object.__class__.__module__ == 'builtins': out.is_gettable = True out.is_settable = False out.is_deletable = False - # Unfortunately we can't get any docstring for these - out.summary, out.content = extract_docs(state, state.property_docs, entry.type, entry.path, '') + out.summary, out.content = extract_docs(state, state.property_docs, entry.type, entry.path, entry.object.__doc__) out.has_details = bool(out.content) out.type = None return out @@ -2304,6 +2335,15 @@ def run(basedir, config, *, templates=default_templates, search_add_lookahead_ba hooks_pre_page=state.hooks_pre_page, hooks_post_run=state.hooks_post_run) + # First process the doc input files so we have all data for rendering + # module/class pages. This needs to be done first so the crawl after can + # have a look at the external data and include documented underscored + # members as well. On the other hand, this means nothing in render_doc() + # has access to the module hierarchy -- all actual content rendering has to + # happen later. + for file in config['INPUT_DOCS']: + render_doc(state, os.path.join(basedir, file)) + # Crawl all input modules to gather the name tree, put their names into a # list for the index. The crawl is done breadth-first, so the function # returns a list of submodules to be crawled next. @@ -2351,16 +2391,6 @@ def run(basedir, config, *, templates=default_templates, search_add_lookahead_ba for hook in state.hooks_post_crawl: hook(name_map=state.name_map) - # Call all registered page begin hooks for the doc rendering - for hook in state.hooks_pre_page: - hook(path=[]) - - # Then process the doc input files so we have all data for rendering - # module pages. This needs to be done *after* the initial crawl so - # cross-linking works as expected. - for file in config['INPUT_DOCS']: - render_doc(state, os.path.join(basedir, file)) - # Go through all crawled names and render modules, classes and pages. A # side effect of the render is entry.summary (and entry.name for pages) # being filled. diff --git a/documentation/test_python/inspect_all_property/inspect_all_property/__init__.py b/documentation/test_python/inspect_all_property/inspect_all_property/__init__.py index b6a737da..3a5341af 100644 --- a/documentation/test_python/inspect_all_property/inspect_all_property/__init__.py +++ b/documentation/test_python/inspect_all_property/inspect_all_property/__init__.py @@ -16,7 +16,7 @@ def hidden_func(a, b, c): pass def _private_but_exposed_func(): - # A private thing but exposed in __all__ + # A private undocumented thing but exposed in __all__ pass hidden_data = 34 diff --git a/documentation/test_python/inspect_string/inspect_string.DerivedException.html b/documentation/test_python/inspect_string/inspect_string.DerivedException.html index e46b4a2f..1c0354b7 100644 --- a/documentation/test_python/inspect_string/inspect_string.DerivedException.html +++ b/documentation/test_python/inspect_string/inspect_string.DerivedException.html @@ -74,6 +74,14 @@ set self.__traceback__ to tb and return self.

Properties

+
+ __cause__ get +
+
exception cause
+
+ __context__ get +
+
exception context
args get
diff --git a/documentation/test_python/inspect_string/inspect_string/__init__.py b/documentation/test_python/inspect_string/inspect_string/__init__.py index 5c67e46d..67751326 100644 --- a/documentation/test_python/inspect_string/inspect_string/__init__.py +++ b/documentation/test_python/inspect_string/inspect_string/__init__.py @@ -6,9 +6,8 @@ import enum # This one is a package that shouldn't be exposed import xml -# These are descendant packages / modules that should be exposed if not -# underscored -from . import subpackage, another_module, _private_module +# These are descendant packages / modules that should be exposed +from . import subpackage, another_module # These are variables from an external modules, shouldn't be exposed either from re import I @@ -41,38 +40,20 @@ class Foo: """A subclass of Foo""" pass - class _PrivateSubclass: - """A private subclass""" - pass - def func(self, a, b): """A method""" pass - def _private_func(self, a, b): - """A private function""" - pass - @classmethod def func_on_class(cls, a): """A class method""" pass - @classmethod - def _private_func_on_class(cls, a): - """A private class method""" - pass - @staticmethod def static_func(a): """A static method""" pass - @staticmethod - def _private_static_func(a): - """A private static method""" - pass - @property def a_property(self): """A property""" @@ -102,11 +83,6 @@ class Foo: writeonly_property = property(None, writeonly_property) - @property - def _private_property(self): - """A private property""" - pass - class FooSlots: """A class with slots. Can't have docstrings for these.""" @@ -147,18 +123,10 @@ class UndocumentedEnum(enum.IntFlag): FLAG_ONE = 1 FLAG_SEVENTEEN = 17 -class _PrivateClass: - """Private class""" - pass - def function(): """A function""" pass -def _private_function(): - """A private function""" - pass - A_CONSTANT = 3.24 ENUM_THING = MyEnum.YAY diff --git a/documentation/test_python/inspect_string/inspect_string/_private_module.py b/documentation/test_python/inspect_string/inspect_string/_private_module.py deleted file mode 100644 index f2ec18f2..00000000 --- a/documentation/test_python/inspect_string/inspect_string/_private_module.py +++ /dev/null @@ -1 +0,0 @@ -"""Private module.""" diff --git a/documentation/test_python/inspect_underscored/docs.rst b/documentation/test_python/inspect_underscored/docs.rst new file mode 100644 index 00000000..9f0872b5 --- /dev/null +++ b/documentation/test_python/inspect_underscored/docs.rst @@ -0,0 +1,33 @@ +.. py:module:: inspect_underscored + :data _DATA_EXTERNAL_IN_MODULE: Externally in-module documented underscored + data + +.. py:module:: inspect_underscored._submodule_external + :summary: Externally documented underscored submodule + +.. py:class:: inspect_underscored._ClassExternal + :summary: Externally documented underscored class + +.. py:enum:: inspect_underscored._EnumExternal + :summary: Externally documented underscored enum + +.. py:function:: inspect_underscored._function_external + :summary: Externally documented underscored function + +.. py:data:: inspect_underscored._DATA_EXTERNAL + :summary: Externally documented underscored data + +.. py:class:: inspect_underscored.Class + :property _property_external_in_class: Externally in-class documented + underscored property + :data _DATA_EXTERNAL_IN_CLASS: Externally in-class documented underscored + data + +.. py:function:: inspect_underscored.Class._function_external + :summary: Externally documented underscored function + +.. py:property:: inspect_underscored.Class._property_external + :summary: Externally documented underscored property + +.. py:data:: inspect_underscored.Class._DATA_EXTERNAL + :summary: Externally documented underscored data diff --git a/documentation/test_python/inspect_underscored/inspect_underscored.Class.html b/documentation/test_python/inspect_underscored/inspect_underscored.Class.html new file mode 100644 index 00000000..b413fc88 --- /dev/null +++ b/documentation/test_python/inspect_underscored/inspect_underscored.Class.html @@ -0,0 +1,88 @@ + + + + + inspect_underscored.Class | My Python Project + + + + + +
+
+
+
+
+

+ inspect_underscored.Class class +

+
+

Contents

+ +
+
+

Methods

+
+
+ def _function(self) +
+
Documented underscored function
+
+ def _function_external(self) +
+
Externally documented underscored function
+
+
+
+

Properties

+
+
+ _property get +
+
Documented underscored property
+
+ _property_external get +
+
Externally documented underscored property
+
+ _property_external_in_class get +
+
Externally in-class documented +underscored property
+
+
+
+

Data

+
+
+ _DATA_EXTERNAL: int = 5 +
+
Externally documented underscored data
+
+ _DATA_EXTERNAL_IN_CLASS: int = 6 +
+
Externally in-class documented underscored +data
+
+
+
+
+
+
+ + diff --git a/documentation/test_python/inspect_underscored/inspect_underscored.html b/documentation/test_python/inspect_underscored/inspect_underscored.html new file mode 100644 index 00000000..fdc4f7ce --- /dev/null +++ b/documentation/test_python/inspect_underscored/inspect_underscored.html @@ -0,0 +1,105 @@ + + + + + inspect_underscored | My Python Project + + + + + +
+
+
+
+
+

+ inspect_underscored module +

+
+

Contents

+ +
+
+

Modules

+
+
module _submodule
+
Documented underscored submodule
+
module _submodule_external
+
Externally documented underscored submodule
+
+
+
+

Classes

+
+
class Class
+
+
class _Class
+
Documented underscored class
+
class _ClassExternal
+
Externally documented underscored class
+
+
+
+

Enums

+
+
+ class _Enum(enum.Enum): +
+
Documented underscored enum
+
+ class _EnumExternal(enum.Enum): +
+
Externally documented underscored enum
+
+
+
+

Functions

+
+
+ def _function() +
+
Documented undercored function
+
+ def _function_external() +
+
Externally documented underscored function
+
+
+
+

Data

+
+
+ _DATA_EXTERNAL: int = 1 +
+
Externally documented underscored data
+
+ _DATA_EXTERNAL_IN_MODULE: int = 2 +
+
Externally in-module documented underscored +data
+
+
+
+
+
+
+ + diff --git a/documentation/test_python/inspect_underscored/inspect_underscored/__init__.py b/documentation/test_python/inspect_underscored/inspect_underscored/__init__.py new file mode 100644 index 00000000..49ca4113 --- /dev/null +++ b/documentation/test_python/inspect_underscored/inspect_underscored/__init__.py @@ -0,0 +1,75 @@ +""".. + +:data _DATA_IN_MODULE: In-module documented underscored data. This won't be + picked up by the initial crawl, unfortunately, as the docstrings are + processed much later. +""" + +import enum + +from . import _submodule, _submodule_external, _submodule_undocumented + +class _Class: + """Documented underscored class""" + +class _ClassExternal: pass + +class _ClassUndocumented: pass + +class _Enum(enum.Enum): + """Documented underscored enum""" + +class _EnumExternal(enum.Enum): pass + +class _EnumUndocumented(enum.Enum): pass + +def _function(): + """Documented undercored function""" + +def _function_external(): pass + +def _function_undocumented(): pass + +_DATA_IN_MODULE: int = 0 +_DATA_EXTERNAL: int = 1 +_DATA_EXTERNAL_IN_MODULE: int = 2 +_DATA_UNDOCUMENTED: int = 3 + +class Class: + """.. + + :property _property_in_class: In-class documented underscored property. + This won't be picked up by the initial crawl, unfortunately, as the + docstrings are processed much later. + :data _DATA_IN_CLASS: In-class documented underscored data. This won't be + picked up by the initial crawl, unfortunately, as the docstrings are + processed much later. + """ + + def _function(self): + """Documented underscored function""" + + def _function_external(self): pass + + def _function_undocumented(self): pass + + @property + def _property(self): + """Documented underscored property""" + + @property + def _property_in_class(self): pass + + @property + def _property_external(self): pass + + @property + def _property_external_in_class(self): pass + + @property + def _property_undocumented(self): pass + + _DATA_IN_CLASS: int = 4 + _DATA_EXTERNAL: int = 5 + _DATA_EXTERNAL_IN_CLASS: int = 6 + _DATA_UNDOCUMENTED: int = 7 diff --git a/documentation/test_python/inspect_underscored/inspect_underscored/_submodule.py b/documentation/test_python/inspect_underscored/inspect_underscored/_submodule.py new file mode 100644 index 00000000..704df830 --- /dev/null +++ b/documentation/test_python/inspect_underscored/inspect_underscored/_submodule.py @@ -0,0 +1 @@ +"""Documented underscored submodule""" diff --git a/documentation/test_python/inspect_underscored/inspect_underscored/_submodule_external.py b/documentation/test_python/inspect_underscored/inspect_underscored/_submodule_external.py new file mode 100644 index 00000000..e69de29b diff --git a/documentation/test_python/inspect_underscored/inspect_underscored/_submodule_undocumented.py b/documentation/test_python/inspect_underscored/inspect_underscored/_submodule_undocumented.py new file mode 100644 index 00000000..e69de29b diff --git a/documentation/test_python/test_inspect.py b/documentation/test_python/test_inspect.py index 6651b390..c48ca659 100644 --- a/documentation/test_python/test_inspect.py +++ b/documentation/test_python/test_inspect.py @@ -235,3 +235,14 @@ class Attrs(BaseInspectTestCase): self.assertEqual(*self.actual_expected_contents('inspect_attrs.MyClass.html')) self.assertEqual(*self.actual_expected_contents('inspect_attrs.MyClassAutoAttribs.html')) self.assertEqual(*self.actual_expected_contents('inspect_attrs.MySlotClass.html')) + +class Underscored(BaseInspectTestCase): + def test(self): + self.run_python({ + 'PLUGINS': ['m.sphinx'], + 'INPUT_DOCS': ['docs.rst'], + 'M_SPHINX_PARSE_DOCSTRINGS': True + }) + + self.assertEqual(*self.actual_expected_contents('inspect_underscored.html')) + self.assertEqual(*self.actual_expected_contents('inspect_underscored.Class.html')) diff --git a/documentation/test_python/test_page.py b/documentation/test_python/test_page.py index 41422605..f033a2f8 100644 --- a/documentation/test_python/test_page.py +++ b/documentation/test_python/test_page.py @@ -96,5 +96,7 @@ class Plugins(BaseTestCase): # No code, thus no docstrings processed self.assertEqual(fancyline.docstring_call_count, 0) - self.assertEqual(fancyline.pre_page_call_count, 4) + # Once for each page, but nonce for render_docs() as that shouldn't + # generate any output anyway + self.assertEqual(fancyline.pre_page_call_count, 3) self.assertEqual(fancyline.post_run_call_count, 1) -- 2.30.2