From: Vladimír Vondruš Date: Tue, 10 Sep 2019 20:06:27 +0000 (+0200) Subject: documentation/python: extend URL_FORMATTER also to static data. X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~cjwatson/git?a=commitdiff_plain;h=9a4402312e0cb4cae62bdc18bd9c28b0f2e653d9;p=blog.git documentation/python: extend URL_FORMATTER also to static data. --- diff --git a/doc/documentation/python.rst b/doc/documentation/python.rst index 7fbe9d3b..93f8068b 100644 --- a/doc/documentation/python.rst +++ b/doc/documentation/python.rst @@ -229,7 +229,7 @@ Variable Description and the rendered HTML does not contain search-related UI or support. If not set, :py:`False` is used. -:py:`SEARCH_DOWNLOAD_BINARY: bool` Download search data as a binary to save +:py:`SEARCH_DOWNLOAD_BINARY` Download search data as a binary to save bandwidth and initial processing time. If not set, :py:`False` is used. See `Search options`_ for more information. @@ -351,7 +351,10 @@ Base85-encoded representation of the search binary and loading that asynchronously as a plain JavaScript file. This results in the search data being 25% larger, but since this is for serving from a local filesystem, it's not considered a problem. If your docs are accessed through a server (or you -don't need Chrome support), enable the :py:`SEARCH_DOWNLOAD_BINARY` option. +don't need Chrome support), set the :py:`SEARCH_DOWNLOAD_BINARY` option to +:py:`True`. The search data are by default fetched from the current directory +on the webserver, if you want to supply a different location, set it to a +string and provide a `custom URL formatter <#custom-url-formatters>`_. The site can provide search engine metadata using the `OpenSearch `_ specification. On supported browsers this means you can add the search field to @@ -389,23 +392,26 @@ search to a subdomain: The :py:`URL_FORMATTER` option allows you to control how *all* filenames and generated URLs look like. It takes an entry type and a "path" as a list of strings (so for example :py:`my.module.Class` is represented as -:py:`['my', 'module', 'Class']`), returning a tuple a filename and an URL. +:py:`['my', 'module', 'Class']`), returning a tuple of a filename and an URL. Those can be the same, but also different (for example a file getting saved into ``my/module/Class/index.html`` but the actual URL being ``https://docs.my.module/Class/``). The default implementation looks like this, producing both filenames and URLs in the form of ``my.module.Class.html``: -.. code:: py - - def default_url_formatter(type: EntryType, path: List[str]) -> Tuple[str, str]: - url = '.'.join(path) + '.html' - return url, url +.. include:: ../../../documentation/python.py + :code: py + :start-after: # [default-url-formatter] + :end-before: # [/default-url-formatter] The ``type`` is an enum, if you don't want to fiddle with imports, compare -:py:`str(type)` against a string, which is one of :py:`'PAGE'`, :py:`'MODULE'`, -:py:`'CLASS'` or :py:`'SPECIAL'`. The :py:`'SPECIAL'` is for index pages and in -that case the ``path`` has always just one item, one of :py:`'pages'`, -:py:`'modules'` or :py:`'classes'`. +:py:`type.name` against a string, which is one of :py:`'PAGE'`, :py:`'MODULE'`, +:py:`'CLASS'`, :py:`'SPECIAL'` or :py:`'STATIC'`. The :py:`'SPECIAL'` is for +index pages and in that case the ``path`` has always just one item, one of +:py:`'pages'`, :py:`'modules'` or :py:`'classes'`. The :py:`'STATIC'` is for +static data such as images or CSS files and the ``path`` is absolute input +filename including the extension and except for search data (which are +generated on-the-fly) it always exists. If the static path is an URL, the URL +formatter is not called. The :py:`ID_FORMATTER` handles formatting of anchors on a page. Again it takes an entry type (which in this case is always one of :py:`'ENUM'`, @@ -1078,14 +1084,15 @@ Filename Use Each template gets passed all configuration values from the `Configuration`_ table as-is, together with a :py:`URL` variable with URL of given output file. -In addition to builtin Jinja2 filters, the ``basename_or_url`` filter returns -either a basename of file path, if the path is relative; or a full URL, if the -argument is an absolute URL. It's useful in cases like this: +In addition to builtin Jinja2 filters, the ``format_url`` filter returns either +a path formatted according to `custom URL formatters`_, if the path is relative; +or a full URL, if the argument is an absolute URL. It's useful in cases like +this: .. code:: html+jinja {% for css in HTML_EXTRA_STYLESHEET %} - + {% endfor %} The actual page contents are provided in a :py:`page` object, which has the diff --git a/documentation/_search.py b/documentation/_search.py index 7d6e2656..cd8b6438 100644 --- a/documentation/_search.py +++ b/documentation/_search.py @@ -33,6 +33,7 @@ from typing import List, Tuple # Version 0 was without the type map searchdata_format_version = 1 +search_filename = f'search-v{searchdata_format_version}.js' searchdata_filename = f'searchdata-v{searchdata_format_version}.bin' searchdata_filename_b85 = f'searchdata-v{searchdata_format_version}.js' diff --git a/documentation/doxygen.py b/documentation/doxygen.py index 10992f5d..0e39cccf 100755 --- a/documentation/doxygen.py +++ b/documentation/doxygen.py @@ -47,7 +47,7 @@ from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers import TextLexer, BashSessionLexer, get_lexer_by_name, find_lexer_class_for_filename -from _search import CssClass, ResultFlag, ResultMap, Trie, serialize_search_data, base85encode_search_data, searchdata_filename, searchdata_filename_b85, searchdata_format_version +from _search import CssClass, ResultFlag, ResultMap, Trie, serialize_search_data, base85encode_search_data, search_filename, searchdata_filename, searchdata_filename_b85, searchdata_format_version sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plugins')) import dot2svg @@ -3596,6 +3596,10 @@ def run(doxyfile, *, templates=default_templates, wildcard=default_wildcard, ind # Skip absolute URLs if urllib.parse.urlparse(i).netloc: continue + # The search.js is special, we encode the version information into its + # filename + file_out = search_filename if i == 'search.js' else i + # If file is found relative to the Doxyfile, use that if os.path.exists(os.path.join(state.basedir, i)): i = os.path.join(state.basedir, i) @@ -3605,7 +3609,7 @@ def run(doxyfile, *, templates=default_templates, wildcard=default_wildcard, ind i = os.path.join(os.path.dirname(os.path.realpath(__file__)), i) logging.debug("copying {} to output".format(i)) - shutil.copy(i, os.path.join(html_output, os.path.basename(i))) + shutil.copy(i, os.path.join(html_output, os.path.basename(file_out))) # Save updated math cache file if state.doxyfile['M_MATH_CACHE_FILE']: diff --git a/documentation/python.py b/documentation/python.py index 36cd04f9..b6331f52 100755 --- a/documentation/python.py +++ b/documentation/python.py @@ -53,7 +53,7 @@ from docutils.transforms import Transform import jinja2 -from _search import CssClass, ResultFlag, ResultMap, Trie, serialize_search_data, base85encode_search_data, searchdata_format_version, searchdata_filename, searchdata_filename_b85 +from _search import CssClass, ResultFlag, ResultMap, Trie, serialize_search_data, base85encode_search_data, searchdata_format_version, search_filename, searchdata_filename, searchdata_filename_b85 sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plugins')) import m.htmlsanity @@ -77,6 +77,8 @@ class EntryType(Enum): # Types not exposed to search are below. Deliberately set to large values # so their accidental use triggers assertions when building search data. + # Statically linked data, such as images. Passed only to the URL_FORMATTER. + STATIC = 98 # One of files from special_pages. Doesn't make sense to include in the # search. SPECIAL = 99 @@ -97,11 +99,22 @@ search_type_map = [ (CssClass.DEFAULT, "data") ] +# TODO: what about nested pages, how to format? +# [default-url-formatter] def default_url_formatter(type: EntryType, path: List[str]) -> Tuple[str, str]: - # TODO: what about nested pages, how to format? + if type == EntryType.STATIC: + url = os.path.basename(path[0]) + + # Encode version information into the search driver + if url == 'search.js': + url = 'search-v{}.js'.format(searchdata_format_version) + + return url, url + url = '.'.join(path) + '.html' - assert '/' not in url # TODO + assert '/' not in url return url, url +# [/default-url-formatter] def default_id_formatter(type: EntryType, path: List[str]) -> str: # Encode pybind11 function overloads into the anchor (hash them, like Rust @@ -1998,9 +2011,11 @@ class ExtractImages(Transform): default_priority = 991 # There is no simple way to have stateful transforms (the publisher always - # gets just the class, not the instance) so we have to use this + # gets just the class, not the instance) so we have to make all data + # awfully global. UGH. # TODO: maybe the pending nodes could solve this? - external_data = set() + _url_formatter = None + _external_data = set() def __init__(self, document, startnode): Transform.__init__(self, document, startnode=startnode) @@ -2013,16 +2028,21 @@ class ExtractImages(Transform): # TODO: is there a non-private access to current document source # path? - ExtractImages._external_data.add(os.path.join(os.path.dirname(self.document.settings._source), image['uri']) if isinstance(self.document.settings._source, str) else image['uri']) + absolute_uri = os.path.join(os.path.dirname(self.document.settings._source), image['uri']) if isinstance(self.document.settings._source, str) else image['uri'] + ExtractImages._external_data.add(absolute_uri) - # Patch the URL to be just the filename - image['uri'] = os.path.basename(image['uri']) + # Patch the URL according to the URL formatter + image['uri'] = ExtractImages._url_formatter(EntryType.STATIC, [absolute_uri])[1] class DocumentationWriter(m.htmlsanity.SaneHtmlWriter): def get_transforms(self): return m.htmlsanity.SaneHtmlWriter.get_transforms(self) + [ExtractImages] def publish_rst(state: State, source, *, source_path=None, translator_class=m.htmlsanity.SaneHtmlTranslator): + # Make the URL formatter known to the image extractor so it can use it for + # patching the URLs + ExtractImages._url_formatter = state.config['URL_FORMATTER'] + pub = docutils.core.Publisher( writer=DocumentationWriter(), source_class=docutils.io.StringInput, @@ -2326,10 +2346,18 @@ def run(basedir, config, *, templates=default_templates, search_add_lookahead_ba env = jinja2.Environment( loader=jinja2.FileSystemLoader(templates), trim_blocks=True, lstrip_blocks=True, enable_async=True) - # Filter to return file basename or the full URL, if absolute - def basename_or_url(path): + # Filter to return formatted URL or the full URL, if already absolute + def format_url(path): if urllib.parse.urlparse(path).netloc: return path - return os.path.basename(path) + + # If file is found relative to the conf file, use that + if os.path.exists(os.path.join(config['INPUT'], path)): + path = os.path.join(config['INPUT'], path) + # Otherwise use path relative to script directory + else: + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), path) + + return config['URL_FORMATTER'](EntryType.STATIC, [path])[1] # Filter to return URL for given symbol. If the path is a string, first try # to treat it as an URL -- either it needs to have the scheme or at least # one slash for relative links (in contrast, Python names don't have @@ -2342,7 +2370,7 @@ def run(basedir, config, *, templates=default_templates, search_add_lookahead_ba entry = state.name_map['.'.join(path)] return entry.url - env.filters['basename_or_url'] = basename_or_url + env.filters['format_url'] = format_url env.filters['path_to_url'] = path_to_url env.filters['urljoin'] = urljoin @@ -2527,11 +2555,16 @@ def run(basedir, config, *, templates=default_templates, search_add_lookahead_ba data = build_search_data(state, add_lookahead_barriers=search_add_lookahead_barriers, merge_subtrees=search_merge_subtrees, merge_prefixes=search_merge_prefixes) + # Joining twice, first before passing those to the URL formatter and + # second after. If SEARCH_DOWNLOAD_BINARY is a string, use that as a + # filename. + # TODO: any chance we could write the file *before* it gets ever passed + # to URL formatters so we can add cache buster hashes to its URL? if state.config['SEARCH_DOWNLOAD_BINARY']: - with open(os.path.join(config['OUTPUT'], searchdata_filename), 'wb') as f: + with open(os.path.join(config['OUTPUT'], config['URL_FORMATTER'](EntryType.STATIC, [os.path.join(config['OUTPUT'], state.config['SEARCH_DOWNLOAD_BINARY'] if isinstance(state.config['SEARCH_DOWNLOAD_BINARY'], str) else searchdata_filename)])[0]), 'wb') as f: f.write(data) else: - with open(os.path.join(config['OUTPUT'], searchdata_filename_b85), 'wb') as f: + with open(os.path.join(config['OUTPUT'], config['URL_FORMATTER'](EntryType.STATIC, [os.path.join(config['OUTPUT'], searchdata_filename_b85)])[0]), 'wb') as f: f.write(base85encode_search_data(data)) # OpenSearch metadata, in case we have the base URL @@ -2563,7 +2596,7 @@ def run(basedir, config, *, templates=default_templates, search_add_lookahead_ba i = os.path.join(os.path.dirname(os.path.realpath(__file__)), i) logging.debug("copying %s to output", i) - shutil.copy(i, os.path.join(config['OUTPUT'], os.path.basename(i))) + shutil.copy(i, os.path.join(config['OUTPUT'], config['URL_FORMATTER'](EntryType.STATIC, [i])[0])) # Call all registered finalization hooks for hook in state.hooks_post_run: hook() diff --git a/documentation/search.js b/documentation/search.js index 67e8e703..82b9fd42 100644 --- a/documentation/search.js +++ b/documentation/search.js @@ -117,11 +117,11 @@ var Search = { return true; }, - download: /* istanbul ignore next */ function(urlBase) { + download: /* istanbul ignore next */ function(url) { var req = window.XDomainRequest ? new XDomainRequest() : new XMLHttpRequest(); if(!req) return; - req.open("GET", urlBase + "searchdata-v" + this.formatVersion + ".bin", true); + req.open("GET", url, true); req.responseType = 'arraybuffer'; req.onreadystatechange = function() { if(req.readyState != 4) return; diff --git a/documentation/templates/doxygen/base.html b/documentation/templates/doxygen/base.html index 2421f08d..69d77f63 100644 --- a/documentation/templates/doxygen/base.html +++ b/documentation/templates/doxygen/base.html @@ -137,10 +137,10 @@ - + {% if M_SEARCH_DOWNLOAD_BINARY %} {% else %} diff --git a/documentation/templates/python/base.html b/documentation/templates/python/base.html index ecde3feb..09857cb8 100644 --- a/documentation/templates/python/base.html +++ b/documentation/templates/python/base.html @@ -4,10 +4,10 @@ {% block title %}{{ PROJECT_TITLE }}{% if PROJECT_SUBTITLE %} {{ PROJECT_SUBTITLE }}{% endif %}{% endblock %} {% for css in STYLESHEETS %} - + {% endfor %} {% if FAVICON %} - + {% endif %} {% if not SEARCH_DISABLED and SEARCH_BASE_URL %} @@ -129,13 +129,13 @@ - + {% if SEARCH_DOWNLOAD_BINARY %} {% else %} - + {% endif %} {% endif %} {% if FINE_PRINT %} diff --git a/documentation/test_doxygen/layout/pages.html b/documentation/test_doxygen/layout/pages.html index 663c778a..c94e94ee 100644 --- a/documentation/test_doxygen/layout/pages.html +++ b/documentation/test_doxygen/layout/pages.html @@ -111,7 +111,7 @@ - +