documentation/python: implement documenting function params and return.

Did not expect I would need to patch Docutils for this because nobody had
this use case since its initial commit in 2002. How the heck is Sphinx
doing this, are they replacing Docutils internals too?!

diff --git a/doc/documentation/python.rst b/doc/documentation/python.rst
index 276fb8ad190d603f2575631297026c4fcb165b21..4b2306e4af0a716abeb1ec8f6a3dfc5e6d0f8363 100644 (file)
--- a/doc/documentation/python.rst
+++ b/doc/documentation/python.rst
@@ -1046,6 +1046,8 @@ Property                            Description
                                     arguments). Set to :py:`False` when
                                     wrapping on multiple lines would only
                                     occupy too much vertical space.
+:py:`function.has_param_details`    If the function parameters are documented
+:py:`function.return_value`         Return value documentation. Can be empty.
 :py:`function.has_details`          If there is enough content for the full
                                     description block [2]_
 :py:`function.is_classmethod`       Set to :py:`True` if the function is
@@ -1070,6 +1072,7 @@ Property                    Description
 :py:`param.kind`            Parameter kind, a string equivalent to one of the
                             `inspect.Parameter.kind <https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind>`_
                             values
+:py:`param.content`         Detailed documentation, if any
 =========================== ===================================================
 
 In some cases (for example in case of native APIs), the parameters can't be

diff --git a/doc/plugins/sphinx.rst b/doc/plugins/sphinx.rst
index 7dd5c16afcb1a689cd2467911a2da1f697eda45e..0b65ed0c9b2e7577749f8bf9b5f1c22a7fece84c 100644 (file)
--- a/doc/plugins/sphinx.rst
+++ b/doc/plugins/sphinx.rst
@@ -68,11 +68,13 @@ List the plugin in your :py:`PLUGINS`.
 The :rst:`.. py:module::`, :rst:`.. py:class::`, :rst:`.. py:enum::`,
 :rst:`.. py:function::`, :rst:`.. py:property::` and :rst:`.. py:data::`
 directives provide a way to supply module, class, enum, function / method,
-property and data documentation content. Directive option is the name to
-document, directive contents are the actual contents; in addition the
-:py:`:summary:` option can override the docstring extracted using inspection.
-No restrictions are made on the contents, it's possible to make use of any
-additional plugins in the markup. Example:
+property and data documentation content.
+
+Directive option is the name to document, directive contents are the actual
+contents; in addition all the directives have the :py:`:summary:` option that
+can override the docstring extracted using inspection. No restrictions are made
+on the contents, it's also possible to make use of any additional plugins in
+the markup. Example:
 
 .. code:: rst
 
@@ -105,3 +107,23 @@ actual rendered docs.
     exist (i.e., accessible via inspection) in given module. If given name
     doesn't exist, a warning will be printed during processing and the
     documentation ignored.
+
+The :rst:`.. py:function::` directive supports additional options ---
+:py:`:param <name>:` for documenting parameters and :py:`:return:` for
+documenting the return value. It's allowed to have either none or all
+parameters documented (the ``self`` parameter can be omitted), having them
+documented only partially or documenting parameters that are not present in the
+function signature will cause a warning. Example:
+
+.. code:: rst
+
+    .. py:function:: mymodule.MyContainer.add
+        :param key:                 Key to add
+        :param value:               Corresponding value
+        :param overwrite_existing:  Overwrite existing value if already present
+            in the container
+        :return:                    The inserted tuple or the existing
+            key/value pair in case ``overwrite_existing`` is not set
+
+        Add a key/value pair to the container, optionally overwriting the
+        previous value.

diff --git a/documentation/python.py b/documentation/python.py
index 169b2fa08f41a4baaef62c93c6e91c0d60563b77..9ce61e6b13f19a4e670a18e7f5883943795c3935 100755 (executable)
--- a/documentation/python.py
+++ b/documentation/python.py
@@ -27,6 +27,7 @@
 import argparse
 import copy
 import docutils
+import docutils.utils
 import enum
 import urllib.parse
 import hashlib
@@ -1125,6 +1126,7 @@ def extract_function_doc(state: State, parent, entry: Empty) -> List[Any]:
 
             overloads += [out]
 
+        # TODO: assign docs and particular param docs to overloads
         return overloads
 
     # Sane introspection path for non-pybind11 code
@@ -1172,6 +1174,33 @@ def extract_function_doc(state: State, parent, entry: Empty) -> List[Any]:
                 param.kind = str(i.kind)
                 out.params += [param]
 
+            # Get docs for each param and for the return value
+            path_str = '.'.join(entry.path)
+            if path_str in state.function_docs:
+                # Having no parameters documented is okay, having self
+                # undocumented as well. But having the rest documented only
+                # partially isn't okay.
+                if state.function_docs[path_str]['params']:
+                    param_docs = state.function_docs[path_str]['params']
+                    used_params = set()
+                    for param in out.params:
+                        if param.name not in param_docs:
+                            if param.name != 'self':
+                                logging.warning("%s() parameter %s is not documented", path_str, param.name)
+                            continue
+                        param.content = render_inline_rst(state, param_docs[param.name])
+                        used_params.add(param.name)
+                        out.has_param_details = True
+                        out.has_details = True
+                    # Having unused param docs isn't okay either
+                    for name, _ in param_docs.items():
+                        if name not in used_params:
+                            logging.warning("%s() documents parameter %s, which isn't in the signature", path_str, name)
+
+                if state.function_docs[path_str]['return']:
+                    out.return_value = render_inline_rst(state, state.function_docs[path_str]['return'])
+                    out.has_details = True
+
         # In CPython, some builtin functions (such as math.log) do not provide
         # metadata about their arguments. Source:
         # https://docs.python.org/3/library/inspect.html#inspect.signature
@@ -1589,6 +1618,62 @@ class _SaneInlineHtmlTranslator(m.htmlsanity.SaneHtmlTranslator):
 def render_inline_rst(state: State, source):
     return publish_rst(state, source, translator_class=_SaneInlineHtmlTranslator).writer.parts.get('body').rstrip()
 
+# Copy of docutils.utils.extract_options which doesn't throw BadOptionError on
+# multi-word field names but instead turns the body into a tuple containing the
+# extra arguments as a prefix and the original data as a suffix. The original
+# restriction goes back to a nondescript "updated" commit from 2002, with no
+# change of this behavior since:
+# https://github.com/docutils-mirror/docutils/commit/508483835d95632efb5dd6b69c444a956d0fb7df
+def _docutils_extract_options(field_list):
+    option_list = []
+    for field in field_list:
+        field_name_parts = field[0].astext().split()
+        name = str(field_name_parts[0].lower())
+        body = field[1]
+        if len(body) == 0:
+            data = None
+        elif len(body) > 1 or not isinstance(body[0], docutils.nodes.paragraph) \
+              or len(body[0]) != 1 or not isinstance(body[0][0], docutils.nodes.Text):
+            raise docutils.utils.BadOptionDataError(
+                  'extension option field body may contain\n'
+                  'a single paragraph only (option "%s")' % name)
+        else:
+            data = body[0][0].astext()
+        if len(field_name_parts) > 1:
+            # Supporting just one argument, don't need more right now (and
+            # allowing any number would make checks on the directive side too
+            # complicated)
+            if len(field_name_parts) != 2: raise docutils.utils.BadOptionError(
+                'extension option field name may contain either one or two words')
+            data = tuple(field_name_parts[1:] + [data])
+        option_list.append((name, data))
+    return option_list
+
+# ... and allowing duplicate options as well. This restriction goes back to the
+# initial commit in 2002. Here for duplicate options we expect the converter to
+# give us a list and we merge those lists; if not, we throw
+# DuplicateOptionError as in the original code.
+def _docutils_assemble_option_dict(option_list, options_spec):
+    options = {}
+    for name, value in option_list:
+        convertor = options_spec[name]  # raises KeyError if unknown
+        if convertor is None:
+            raise KeyError(name)        # or if explicitly disabled
+        try:
+            converted = convertor(value)
+        except (ValueError, TypeError) as detail:
+            raise detail.__class__('(option: "%s"; value: %r)\n%s'
+                                   % (name, value, ' '.join(detail.args)))
+        if name in options:
+            if isinstance(converted, list):
+                assert isinstance(options[name], list) and not isinstance(options[name], tuple)
+                options[name] += converted
+            else:
+                raise docutils.utils.DuplicateOptionError('duplicate non-list option "%s"' % name)
+        else:
+            options[name] = converted
+    return options
+
 def render_doc(state: State, filename):
     logging.debug("parsing docs from %s", filename)
 
@@ -1596,8 +1681,19 @@ def render_doc(state: State, filename):
     # these functions are not generating any pages
 
     # Render the file. The directives should take care of everything, so just
-    # discard the output afterwards.
-    with open(filename, 'r') as f: publish_rst(state, f.read())
+    # discard the output afterwards. Some directives (such as py:function) have
+    # multi-word field names and can be duplicated, so we have to patch the
+    # option extractor to allow that. See _docutils_extract_options and
+    # _docutils_assemble_option_dict above for details.
+    with open(filename, 'r') as f:
+        prev_extract_options = docutils.utils.extract_options
+        prev_assemble_option_dict = docutils.utils.assemble_option_dict
+        docutils.utils.extract_options = _docutils_extract_options
+        docutils.utils.assemble_option_dict = _docutils_assemble_option_dict
+
+        publish_rst(state, f.read())
+        docutils.utils.extract_options = prev_extract_options
+        docutils.utils.assemble_option_dict = prev_assemble_option_dict
 
 def render_page(state: State, path, input_filename, env):
     filename, url = state.config['URL_FORMATTER'](EntryType.PAGE, path)

diff --git a/documentation/templates/python/details-function.html b/documentation/templates/python/details-function.html
index b78ae71fc4fc2910a652ed4ed3810874f89d9ee9..150686462262d46bec9f4a5b620984d8c024d224 100644 (file)
--- a/documentation/templates/python/details-function.html
+++ b/documentation/templates/python/details-function.html
@@ -14,10 +14,12 @@
               </thead>
               <tbody>
                 {% for param in function.params %}
+                {% if loop.index != 1 or param.name != 'self' or param.content %}
                 <tr>
                   <td{% if loop.index == 1 %} style="width: 1%"{% endif %}>{{ param.name }}</td>
-                  <td>{{ param.description }}</td>
+                  <td>{{ param.content }}</td>
                 </tr>
+                {% endif %}
                 {% endfor %}
               </tbody>
               {% endif %}

diff --git a/documentation/test_python/content/content.Class.html b/documentation/test_python/content/content.Class.html
index 66eb582286f520eacd74e4a34272fef086f3fa5e..26d9b05661e013710c53f9410b741807f3c1390d 100644 (file)
--- a/documentation/test_python/content/content.Class.html
+++ b/documentation/test_python/content/content.Class.html
@@ -67,6 +67,10 @@ indented.</p>
             </dt>
             <dd>This overwrites the docstring for <code>content.Class.method</code>, but
 doesn't add any detailed block.</dd>
+            <dt>
+              <span class="m-doc-wrap-bumper">def <a href="#method_param_docs" class="m-doc">method_param_docs</a>(</span><span class="m-doc-wrap">self, a, b)</span>
+            </dt>
+            <dd>This method gets its params except self documented</dd>
             <dt>
               <span class="m-doc-wrap-bumper">def <a href="#method_with_details" class="m-doc">method_with_details</a>(</span><span class="m-doc-wrap">self)</span>
             </dt>
@@ -124,6 +128,28 @@ but doesn't add any detailed block.</dd>
             </h3>
             <p>This function is a static method</p>
 <p>The <span class="m-label m-info">staticmethod</span> should be shown here.</p>
+          </div></section>
+          <section class="m-doc-details" id="method_param_docs"><div>
+            <h3>
+              <span class="m-doc-wrap-bumper">def content.<wbr />Class.<wbr /></span><span class="m-doc-wrap"><span class="m-doc-wrap-bumper"><a href="#method_param_docs" class="m-doc-self">method_param_docs</a>(</span><span class="m-doc-wrap">self, a, b)</span></span>
+            </h3>
+            <p>This method gets its params except self documented</p>
+            <table class="m-table m-fullwidth m-flat">
+              <thead>
+                <tr><th colspan="2">Parameters</th></tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td>a</td>
+                  <td>The first parameter</td>
+                </tr>
+                <tr>
+                  <td>b</td>
+                  <td>The second parameter</td>
+                </tr>
+              </tbody>
+            </table>
+<p>The <code>self</code> isn't documented and thus also not included in the list.</p>
           </div></section>
           <section class="m-doc-details" id="method_with_details"><div>
             <h3>

diff --git a/documentation/test_python/content/content.html b/documentation/test_python/content/content.html
index 6fc4bcc87aa20605ab55401c48860e2035d9bcca..d5f405c4a8ad6ef936056ae63241ea8ba7ac9e9a 100644 (file)
--- a/documentation/test_python/content/content.html
+++ b/documentation/test_python/content/content.html
@@ -74,12 +74,6 @@ doesn't add any detailed block.</dd>
         <section id="functions">
           <h2><a href="#functions">Functions</a></h2>
           <dl class="m-doc">
-            <dt>
-              <span class="m-doc-wrap-bumper">def <a href="#annotations" class="m-doc">annotations</a>(</span><span class="m-doc-wrap">a: int,
-              b,
-              c: float) -&gt; str</span>
-            </dt>
-            <dd>No annotations shown for this</dd>
             <dt id="foo">
               <span class="m-doc-wrap-bumper">def <a href="#foo" class="m-doc-self">foo</a>(</span><span class="m-doc-wrap">a, b)</span>
             </dt>
@@ -93,6 +87,16 @@ doesn't add any detailed block.</dd>
               <span class="m-doc-wrap-bumper">def <a href="#function_with_summary" class="m-doc">function_with_summary</a>(</span><span class="m-doc-wrap">)</span>
             </dt>
             <dd>This function has summary from the docstring</dd>
+            <dt>
+              <span class="m-doc-wrap-bumper">def <a href="#param_docs" class="m-doc">param_docs</a>(</span><span class="m-doc-wrap">a: int,
+              b,
+              c: float) -&gt; str</span>
+            </dt>
+            <dd>Detailed param docs and annotations</dd>
+            <dt>
+              <span class="m-doc-wrap-bumper">def <a href="#param_docs_wrong" class="m-doc">param_docs_wrong</a>(</span><span class="m-doc-wrap">a, b)</span>
+            </dt>
+            <dd>Should give warnings</dd>
           </dl>
         </section>
         <section id="data">
@@ -139,15 +143,6 @@ doesn't add any detailed block.</dd>
         </section>
         <section>
           <h2>Function documentation</h2>
-          <section class="m-doc-details" id="annotations"><div>
-            <h3>
-              <span class="m-doc-wrap-bumper">def content.<wbr /></span><span class="m-doc-wrap"><span class="m-doc-wrap-bumper"><a href="#annotations" class="m-doc-self">annotations</a>(</span><span class="m-doc-wrap">a: int,
-              b,
-              c: float) -&gt; str</span></span>
-            </h3>
-            <p>No annotations shown for this</p>
-<p>Type annotations in detailed docs.</p>
-          </div></section>
           <section class="m-doc-details" id="foo_with_details"><div>
             <h3>
               <span class="m-doc-wrap-bumper">def content.<wbr /></span><span class="m-doc-wrap"><span class="m-doc-wrap-bumper"><a href="#foo_with_details" class="m-doc-self">foo_with_details</a>(</span><span class="m-doc-wrap">a, b)</span></span>
@@ -163,6 +158,62 @@ Detailed docs for this function</div>
             <p>This function has summary from the docstring</p>
 <p>This function has external details but summary from the docstring.</p>
           </div></section>
+          <section class="m-doc-details" id="param_docs"><div>
+            <h3>
+              <span class="m-doc-wrap-bumper">def content.<wbr /></span><span class="m-doc-wrap"><span class="m-doc-wrap-bumper"><a href="#param_docs" class="m-doc-self">param_docs</a>(</span><span class="m-doc-wrap">a: int,
+              b,
+              c: float) -&gt; str</span></span>
+            </h3>
+            <p>Detailed param docs and annotations</p>
+            <table class="m-table m-fullwidth m-flat">
+              <thead>
+                <tr><th colspan="2">Parameters</th></tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td style="width: 1%">a</td>
+                  <td>First parameter</td>
+                </tr>
+                <tr>
+                  <td>b</td>
+                  <td>The second one</td>
+                </tr>
+                <tr>
+                  <td>c</td>
+                  <td>And a <code>float</code></td>
+                </tr>
+              </tbody>
+              <tfoot>
+                <tr>
+                  <th>Returns</th>
+                  <td>String, of course, it's all <em>stringly</em> typed</td>
+                </tr>
+              </tfoot>
+            </table>
+<p>Type annotations and param list in detailed docs.</p>
+          </div></section>
+          <section class="m-doc-details" id="param_docs_wrong"><div>
+            <h3>
+              <span class="m-doc-wrap-bumper">def content.<wbr /></span><span class="m-doc-wrap"><span class="m-doc-wrap-bumper"><a href="#param_docs_wrong" class="m-doc-self">param_docs_wrong</a>(</span><span class="m-doc-wrap">a, b)</span></span>
+            </h3>
+            <p>Should give warnings</p>
+            <table class="m-table m-fullwidth m-flat">
+              <thead>
+                <tr><th colspan="2">Parameters</th></tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td style="width: 1%">a</td>
+                  <td>First</td>
+                </tr>
+                <tr>
+                  <td>b</td>
+                  <td></td>
+                </tr>
+              </tbody>
+            </table>
+<p>The <code>b</code> is not documented, while <code>c</code> isn't in the signature.</p>
+          </div></section>
         </section>
         <section>
           <h2>Data documentation</h2>

diff --git a/documentation/test_python/content/content/__init__.py b/documentation/test_python/content/content/__init__.py
index f48c850aed331ca8d2859dfe093ac30e272f74c9..be50c5371fad42e0fcd64abd7222ae1671b861ff 100644 (file)
--- a/documentation/test_python/content/content/__init__.py
+++ b/documentation/test_python/content/content/__init__.py
@@ -23,6 +23,9 @@ class Class:
     def method_with_details(self):
         pass
 
+    def method_param_docs(self, a, b):
+        """This method gets its params except self documented"""
+
     @property
     def a_property(self):
         """This summary is not shown either"""
@@ -60,8 +63,11 @@ def foo_with_details(a, b):
 def function_with_summary():
     """This function has summary from the docstring"""
 
-def annotations(a: int, b, c: float) -> str:
-    """No annotations shown for this"""
+def param_docs(a: int, b, c: float) -> str:
+    """Detailed param docs and annotations"""
+
+def param_docs_wrong(a, b):
+    """Should give warnings"""
 
 CONSTANT: float = 3.14
 

diff --git a/documentation/test_python/content/docs.rst b/documentation/test_python/content/docs.rst
index bef6a8e802da654d77790552581cacb9265ff7bc..5bf105a5ef226a915d7444d05a1d210b9faad3f2 100644 (file)
--- a/documentation/test_python/content/docs.rst
+++ b/documentation/test_python/content/docs.rst
@@ -39,6 +39,12 @@
 
     This one has a detailed block without any summary.
 
+.. py:function:: content.Class.method_param_docs
+    :param a: The first parameter
+    :param b: The second parameter
+
+    The ``self`` isn't documented and thus also not included in the list.
+
 .. py:property:: content.Class.a_property
     :summary: This overwrites the docstring for ``content.Class.a_property``,
         but doesn't add any detailed block.
@@ -83,9 +89,19 @@
 
     This function has external details but summary from the docstring.
 
-.. py:function:: content.annotations
+.. py:function:: content.param_docs
+    :param a: First parameter
+    :param b: The second one
+    :param c: And a ``float``
+    :return: String, of course, it's all *stringly* typed
+
+    Type annotations and param list in detailed docs.
+
+.. py:function:: content.param_docs_wrong
+    :param a: First
+    :param c: Third
 
-    Type annotations in detailed docs.
+    The ``b`` is not documented, while ``c`` isn't in the signature.
 
 .. py:data:: content.CONSTANT
     :summary: This is finally a docstring for ``content.CONSTANT``

diff --git a/plugins/m/sphinx.py b/plugins/m/sphinx.py
index cc11538ef454594d2d47c1938e97c1115eb14a10..56a594e77af8fc2d2ad18cf279d0f41f50947a9b 100644 (file)
--- a/plugins/m/sphinx.py
+++ b/plugins/m/sphinx.py
@@ -71,15 +71,31 @@ class PyEnum(rst.Directive):
         }
         return []
 
+# List option (allowing multiple arguments). See _docutils_assemble_option_dict
+# in python.py for details.
+def directives_unchanged_list(argument):
+    return [directives.unchanged(argument)]
+
 class PyFunction(rst.Directive):
     final_argument_whitespace = True
     has_content = True
     required_arguments = 1
-    option_spec = {'summary': directives.unchanged}
+    option_spec = {'summary': directives.unchanged,
+                   'param': directives_unchanged_list,
+                   'return': directives.unchanged}
 
     def run(self):
+        # Check that params are parsed properly, turn them into a dict. This
+        # will blow up if the param name is not specified.
+        params = {}
+        for name, content in self.options.get('param', []):
+            if name in params: raise KeyError(f"duplicate param {name}")
+            params[name] = content
+
         function_doc_output[self.arguments[0]] = {
             'summary': self.options.get('summary', ''),
+            'params': params,
+            'return': self.options.get('return'),
             'content': '\n'.join(self.content)
         }
         return []