From 597ebe99163192fb6daf2e6a45f87eb0095a3d23 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 25 Aug 2019 12:27:05 +0200 Subject: [PATCH] m.sphinx: ability to create intersphinx inventories as well. --- doc/plugins/sphinx.rst | 155 ++++++++++++++---- documentation/python.py | 1 + documentation/test_python/CMakeLists.txt | 6 + .../inspect_create_intersphinx/__init__.py | 26 +++ .../inspect_create_intersphinx/pybind.cpp | 16 ++ .../inspect_create_intersphinx/page.rst | 4 + documentation/test_python/test_inspect.py | 38 +++++ plugins/m/sphinx.py | 27 ++- 8 files changed, 243 insertions(+), 30 deletions(-) create mode 100644 documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/__init__.py create mode 100644 documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/pybind.cpp create mode 100644 documentation/test_python/inspect_create_intersphinx/page.rst diff --git a/doc/plugins/sphinx.rst b/doc/plugins/sphinx.rst index 85be2631..abc6c373 100644 --- a/doc/plugins/sphinx.rst +++ b/doc/plugins/sphinx.rst @@ -53,52 +53,62 @@ using external files in a way similar to `Sphinx `_ Download the `m/sphinx.py <{filename}/plugins.rst>`_ file, put it including the ``m/`` directory into one of your :py:`PLUGIN_PATHS` and add ``m.sphinx`` -package to your :py:`PLUGINS` in ``pelicanconf.py``. The plugin uses Sphinx -inventory files to get a list of linkable symbols and you need to provide -list of tuples containing tag file path, URL prefix, an optional list of -implicitly prepended paths and an optional list of CSS classes for each link in -:py:`M_SPHINX_INVENTORIES`. Every Sphinx-generated documentation contains an -``objects.inv`` file in its root directory (and the root directory is the URL -prefix as well), for example for Python 3 it's located at -https://docs.python.org/3/objects.inv. Download the files and specify path to -them and the URL they were downloaded from, for example: +package to your :py:`PLUGINS` in ``pelicanconf.py``. The +:py:`M_SPHINX_INVENTORIES` option is described in the +`Links to external Sphinx documentation`_ section below. .. code:: python PLUGINS += ['m.sphinx'] - M_SPHINX_INVENTORIES = [ - ('sphinx/python.inv', 'https://docs.python.org/3/', ['xml.']), - ('sphinx/numpy.inv', 'https://docs.scipy.org/doc/numpy/', [], ['m-flat'])] + M_SPHINX_INVENTORIES = [...] `Python doc theme`_ ------------------- List the plugin in your :py:`PLUGINS`. The :py:`M_SPHINX_INVENTORIES` -configuration option is interpreted the same way as in case of the `Pelican`_ -plugin. +configuration option is described in the `Links to external Sphinx documentation`_ +section below, additionally it supports also the inverse --- saving internal +symbols to a file to be linked from elsewhere, see +`Creating an Intersphinx inventory file`_ for a description of the +:py:`M_SPHINX_INVENTORY_OUTPUT` option. .. code:: py PLUGINS += ['m.sphinx'] M_SPHINX_INVENTORIES = [...] + M_SPHINX_INVENTORY_OUTPUT = 'objects.inv' `Links to external Sphinx documentation`_ ========================================= -Use the :rst:`:ref:` interpreted text role for linking to symbols defined in -:py:`M_SPHINX_INVENTORIES`. In order to save you some typing, the leading -name(s) mentioned there can be omitted when linking to given symbol. - -Link text is equal to link target unless the target provides its own title -(such as documentation pages), function links have ``()`` appended to make it -clear it's a function. It's possible to specify custom link title using the -:rst:`:ref:`link title ``` syntax. If a symbol can't be found, a -warning is printed to output and link target is rendered in a monospace font -(or, if custom link title is specified, just the title is rendered, as normal -text). You can append ``#anchor`` to ``link-target`` to link to anchors that -are not present in the inventory file, the same works for query parameters -starting with ``?``. Adding custom CSS classes can be done by deriving the role -and adding the :rst:`:class:` option. +Sphinx provides a so-called Intersphinx files to make names from one +documentation available for linking from elsewhere. The plugin supports the +(current) version 2 of those inventory files, version 1 is not supported. You +need to provide a list of tuples containing tag file path, URL prefix, an +optional list of implicitly prepended paths and an optional list of CSS classes +for each link in :py:`M_SPHINX_INVENTORIES`. Every Sphinx-generated +documentation contains an ``objects.inv`` file in its root directory (and the +root directory is the URL prefix as well), for example for Python 3 it's +located at https://docs.python.org/3/objects.inv. Download the files and +specify path to them and the URL they were downloaded from, for example: + +.. code:: py + + M_SPHINX_INVENTORIES = [ + ('sphinx/python.inv', 'https://docs.python.org/3/', ['xml.']), + ('sphinx/numpy.inv', 'https://docs.scipy.org/doc/numpy/', [], ['m-flat'])] + +Use the :rst:`:ref:` interpreted text role for linking to those symbols. Link +text is equal to link target unless the target provides its own title (such as +documentation pages), function links have ``()`` appended to make it clear it's +a function. It's possible to specify custom link title using the :rst:`:ref:` +link title ``` syntax. If a symbol can't be found, a warning is +printed to output and link target is rendered in a monospace font (or, if +custom link title is specified, just the title is rendered, as normal text). +You can append ``#anchor`` to ``link-target`` to link to anchors that are not +present in the inventory file, the same works for query parameters starting +with ``?``. Adding custom CSS classes can be done by deriving the role and +adding the :rst:`:class:` option. Since there's many possible targets and there can be conflicting names, sometimes it's desirable to disambiguate. If you suffix the link target with @@ -150,6 +160,95 @@ linked to from all signatures. by the `m.dox <{filename}/plugins/links.rst#doxygen-documentation>`_ plugin. +`Creating an Intersphinx inventory file`_ +========================================= + +In the Python doc theme, the :py:`M_SPHINX_INVENTORY_OUTPUT` option can be used +to produce an Intersphinx inventory file --- basically an inverse of +:py:`M_SPHINX_INVENTORIES`. Set it to a filename and the plugin will fill it +with all names present in the documentation. Commonly, Sphinx projects expect +this file to be named ``objects.inv`` and located in the documentation root, so +doing the following will ensure it can be easily used: + +.. code:: py + + M_SPHINX_INVENTORY_OUTPUT = 'objects.inv' + +.. block-info:: Inventory file format + + The format is unfortunately not well-documented in Sphinx itself and this + plugin additionally makes some extensions to it, so the following is a + description of the file structure as used by m.css. File header is a few + textual lines as shown below, while everything after is zlib-compressed. + The plugin creates the inventory file in the (current) version 2 and at the + moment hardcodes project name to ``X`` and version to ``0``:: + + # Sphinx inventory version 2 + # Project: X + # Version: 0 + # The remainder of this file is compressed using zlib. + + When decompressing the rest, the contents are again textual, each line + being one entry:: + + mymodule.MyClass py:class 2 mymodule.MyClass.html - + mymodule.foo py:function 2 mymodule.html#foo - + my-page std:doc 2 my-page.html A documentation page + + .. class:: m-table + + =========== =============================================================== + Field Description + =========== =============================================================== + name Name of the module, class, function, page... Basically the link + target used by :rst:`:ref:`. + type Type. Files created by the ``m.sphinx`` plugins always use only + the following types; Sphinx-created files may have arbitrary + other types such as ``c:function``. This type is what can be + used in :rst:`:ref:` to further disambiguate the target. + + - ``py:module`` for modules + - ``py:class`` for classes + - ``py:function`` :label-warning:`m.css-specific` for + functions, but currently also methods, class methods and + static methods. Sphinx uses ``py:classmethod``, + ``py:staticmethod`` and ``py:method`` instead. + - ``py:attribute`` for properties + - ``py:enum`` :label-warning:`m.css-specific` for enums. + Sphinx treats those the same as ``py:class``. + - ``py:enumvalue`` :label-warning:`m.css-specific` for enum + values. Sphinx treats those the same as ``py:data``. + - ``py:data`` for data + - ``std:doc`` for pages + ``2`` A `mysterious number `_. + `Sphinx implementation `_ + denotes this as ``prio`` but doesn't use it in any way. + url Full url of the page. There's a minor space-saving + optimization --- if the URL ends with ``$``, it should be + composed as :py:`location = location[:-1] + name`. The plugin + can recognize this feature but doesn't make use of it when + writing the file. + title Link title. If set to ``-``, :py:`name` should be used as a + link title instead. + =========== =============================================================== + + For debugging purposes, the ``sphinx.py`` plugin script can decode and + print inventory files passed to it on the command line. See ``--help`` for + more options. + + .. code:: shell-session + + $ ./m/sphinx.py python.inv + # Sphinx inventory version 2 + # Project: Python + # Version: 3.7 + # The remainder of this file is compressed using zlib. + CO_FUTURE_DIVISION c:var 1 c-api/veryhigh.html#c.$ - + PYMEM_DOMAIN_MEM c:var 1 c-api/memory.html#c.$ - + PYMEM_DOMAIN_OBJ c:var 1 c-api/memory.html#c.$ - + ... + + `Module, class, enum, function, property and data docs`_ ======================================================== diff --git a/documentation/python.py b/documentation/python.py index b45f791e..a696772e 100755 --- a/documentation/python.py +++ b/documentation/python.py @@ -1971,6 +1971,7 @@ def run(basedir, config, *, templates=default_templates, search_add_lookahead_ba # Go through all crawled names and render modules, classes and pages. A # side effect of the render is entry.summary (and entry.name for pages) # being filled. + # TODO: page name need to be added earlier for intersphinx! for entry in state.name_map.values(): # If there is no object, the entry is an external reference. Skip # those. Can't do `not entry.object` because that gives ValueError diff --git a/documentation/test_python/CMakeLists.txt b/documentation/test_python/CMakeLists.txt index 5dc0a690..298c40f8 100644 --- a/documentation/test_python/CMakeLists.txt +++ b/documentation/test_python/CMakeLists.txt @@ -32,6 +32,12 @@ foreach(target pybind_signatures pybind_enums pybind_submodules pybind_type_link set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${target}) endforeach() +# Need a special location for this one +pybind11_add_module(pybind_inspect_create_intersphinx inspect_create_intersphinx/inspect_create_intersphinx/pybind.cpp) +set_target_properties(pybind_inspect_create_intersphinx PROPERTIES + OUTPUT_NAME pybind + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/inspect_create_intersphinx/inspect_create_intersphinx) + # Need a special location for this one pybind11_add_module(pybind_link_formatting link_formatting/link_formatting/pybind.cpp) set_target_properties(pybind_link_formatting PROPERTIES diff --git a/documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/__init__.py b/documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/__init__.py new file mode 100644 index 00000000..4574114f --- /dev/null +++ b/documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/__init__.py @@ -0,0 +1,26 @@ +import enum + +from . import pybind + +def function(): pass + +class Class: + @classmethod + def class_method(cls): pass + + @staticmethod + def static_method(): pass + + def method(self): pass + + @property + def a_property(self): pass + + CLASS_DATA = 3 + +class Enum(enum.Enum): + ENUM_VALUE = 1 + +MODULE_DATA = {} + +def _private_function(): pass diff --git a/documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/pybind.cpp b/documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/pybind.cpp new file mode 100644 index 00000000..71e7d9bd --- /dev/null +++ b/documentation/test_python/inspect_create_intersphinx/inspect_create_intersphinx/pybind.cpp @@ -0,0 +1,16 @@ +#include + +namespace py = pybind11; + +namespace { + +void overloadedFunction(int, float) {} +void overloadedFunction(int) {} + +} + +PYBIND11_MODULE(pybind, m) { + m + .def("overloaded_function", static_cast(&overloadedFunction)) + .def("overloaded_function", static_cast(&overloadedFunction)); +} diff --git a/documentation/test_python/inspect_create_intersphinx/page.rst b/documentation/test_python/inspect_create_intersphinx/page.rst new file mode 100644 index 00000000..1e953fc7 --- /dev/null +++ b/documentation/test_python/inspect_create_intersphinx/page.rst @@ -0,0 +1,4 @@ +A documentation page +#################### + +Page content. diff --git a/documentation/test_python/test_inspect.py b/documentation/test_python/test_inspect.py index 92b8f737..a4b71c99 100644 --- a/documentation/test_python/test_inspect.py +++ b/documentation/test_python/test_inspect.py @@ -33,6 +33,9 @@ from distutils.version import LooseVersion from python import default_templates from . import BaseInspectTestCase +sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../plugins')) +import m.sphinx + class String(BaseInspectTestCase): def test(self): self.run_python({ @@ -177,3 +180,38 @@ class TypeLinks(BaseInspectTestCase): self.assertEqual(*self.actual_expected_contents('inspect_type_links.second.Foo.html')) self.assertEqual(*self.actual_expected_contents('inspect_type_links.second.FooSlots.html')) self.assertEqual(*self.actual_expected_contents('inspect_type_links.second.FooSlotsInvalid.html')) + +class CreateIntersphinx(BaseInspectTestCase): + def test(self): + self.run_python({ + 'PLUGINS': ['m.sphinx'], + 'INPUT_PAGES': ['page.rst'], + 'M_SPHINX_INVENTORIES': [ + # Nothing from here should be added to the output + ('../../../doc/documentation/python.inv', 'https://docs.python.org/3/', [], ['m-doc-external'])], + 'M_SPHINX_INVENTORY_OUTPUT': 'things.inv', + 'PYBIND11_COMPATIBILITY': True + }) + + with open(os.path.join(self.path, 'output/things.inv'), 'rb') as f: + self.assertEqual(m.sphinx.pretty_print_intersphinx_inventory(f), """ +# Sphinx inventory version 2 +# Project: X +# Version: 0 +# The remainder of this file is compressed using zlib. +inspect_create_intersphinx.Class.a_property py:attribute 2 inspect_create_intersphinx.Class.html#a_property - +inspect_create_intersphinx.Class py:class 2 inspect_create_intersphinx.Class.html - +inspect_create_intersphinx.Class.CLASS_DATA py:data 2 inspect_create_intersphinx.Class.html#CLASS_DATA - +inspect_create_intersphinx.MODULE_DATA py:data 2 inspect_create_intersphinx.html#MODULE_DATA - +inspect_create_intersphinx.Enum py:enum 2 inspect_create_intersphinx.html#Enum - +inspect_create_intersphinx.Enum.ENUM_VALUE py:enumvalue 2 inspect_create_intersphinx.html#Enum-ENUM_VALUE - +inspect_create_intersphinx.Class.class_method py:function 2 inspect_create_intersphinx.Class.html#class_method - +inspect_create_intersphinx.Class.method py:function 2 inspect_create_intersphinx.Class.html#method - +inspect_create_intersphinx.Class.static_method py:function 2 inspect_create_intersphinx.Class.html#static_method - +inspect_create_intersphinx.function py:function 2 inspect_create_intersphinx.html#function - +inspect_create_intersphinx.pybind.overloaded_function py:function 2 inspect_create_intersphinx.pybind.html#overloaded_function - +inspect_create_intersphinx py:module 2 inspect_create_intersphinx.html - +inspect_create_intersphinx.pybind py:module 2 inspect_create_intersphinx.pybind.html - +page std:doc 2 page.html - +""".lstrip()) + # Yes, above it should say A documentation page, but it doesn't diff --git a/plugins/m/sphinx.py b/plugins/m/sphinx.py index 88b784ef..90bfc834 100755 --- a/plugins/m/sphinx.py +++ b/plugins/m/sphinx.py @@ -46,6 +46,7 @@ enum_doc_output = None function_doc_output = None property_doc_output = None data_doc_output = None +inventory_filename = None class PyModule(rst.Directive): final_argument_whitespace = True @@ -324,7 +325,7 @@ def merge_inventories(name_map, **kwargs): type_string = 'py:data' elif entry.type == EntryType.PAGE: type_string = 'std:doc' - else: + else: # pragma: no cover # TODO: what to do with these? allow linking to them? disambiguate # or prefix the names somehow? assert entry.type == EntryType.SPECIAL, entry.type @@ -362,14 +363,36 @@ def merge_inventories(name_map, **kwargs): assert path not in data data[path] = value + # Save the internal inventory, if requested. Again basically a copy of + # sphinx.util.inventory.InventoryFile.dump(). + if inventory_filename: + with open(os.path.join(inventory_filename), 'wb') as f: + # Header + # TODO: user-defined project/version + f.write(b'# Sphinx inventory version 2\n' + b'# Project: X\n' + b'# Version: 0\n' + b'# The remainder of this file is compressed using zlib.\n') + + # Body. Sorting so it's in a reproducible order for testing. + compressor = zlib.compressobj(9) + for type_, data in sorted(internal_inventory.items()): + for path, value in data.items(): + url, title, css_classes = value + # The type has to contain a colon. Wtf is the 2? + assert ':' in type_ + f.write(compressor.compress(f'{path} {type_} 2 {url} {title}\n'.encode('utf-8'))) + f.write(compressor.flush()) + def register_mcss(mcss_settings, module_doc_contents, class_doc_contents, enum_doc_contents, function_doc_contents, property_doc_contents, data_doc_contents, hooks_post_crawl, hooks_pre_page, **kwargs): - global module_doc_output, class_doc_output, enum_doc_output, function_doc_output, property_doc_output, data_doc_output + global module_doc_output, class_doc_output, enum_doc_output, function_doc_output, property_doc_output, data_doc_output, inventory_filename module_doc_output = module_doc_contents class_doc_output = class_doc_contents enum_doc_output = enum_doc_contents function_doc_output = function_doc_contents property_doc_output = property_doc_contents data_doc_output = data_doc_contents + inventory_filename = os.path.join(mcss_settings['OUTPUT'], mcss_settings['M_SPHINX_INVENTORY_OUTPUT']) if 'M_SPHINX_INVENTORY_OUTPUT' in mcss_settings else None parse_intersphinx_inventories(input=mcss_settings['INPUT'], inventories=mcss_settings.get('M_SPHINX_INVENTORIES', [])) -- 2.30.2