From: Vladimír Vondruš Date: Tue, 23 Jan 2018 09:35:16 +0000 (+0100) Subject: doxygen: initial client-side search implementation. X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~cjwatson/git?a=commitdiff_plain;h=fffe7cf354d9f7e41acad26c41963f30e4e95219;p=blog.git doxygen: initial client-side search implementation. --- diff --git a/artwork/README.rst b/artwork/README.rst index 9a324779..566c0db9 100644 --- a/artwork/README.rst +++ b/artwork/README.rst @@ -9,3 +9,9 @@ Export as 16x16 PNG and convert to an ``*.ico``: .. code:: sh convert favicon.png favicon.ico + +magnifier.svg +============= + +Doxygen search icon. Export as "Optimized SVG" and copy into the ``base.html`` +template. diff --git a/artwork/magnifier.svg b/artwork/magnifier.svg new file mode 100644 index 00000000..77b24639 --- /dev/null +++ b/artwork/magnifier.svg @@ -0,0 +1,55 @@ + + + + Svg Vector Icons : http://www.onlinewebfonts.com/icon image/svg+xml \ No newline at end of file diff --git a/css/m-dark+doxygen.compiled.css b/css/m-dark+doxygen.compiled.css index 82c1a4f6..4213a074 100644 --- a/css/m-dark+doxygen.compiled.css +++ b/css/m-dark+doxygen.compiled.css @@ -1989,3 +1989,99 @@ article section.m-dox-details:target > div { border-left-width: 0.25rem; border-left-color: #a5c9ea; } +a.m-dox-search-icon { + padding-left: 1rem; + padding-right: 1rem; +} +a.m-dox-search-icon svg { + height: 0.9rem; + fill: #ffffff; +} +body > header > nav #m-navbar-collapse a.m-dox-search-icon svg { + vertical-align: -5%; +} +a.m-dox-search-icon:focus svg, a.m-dox-search-icon:hover svg, a.m-dox-search-icon:active svg { + fill: #a5c9ea; +} +.m-dox-search { + display: none; + z-index: 10; + position: fixed; + left: 0; + right: 0; + top: 0; + bottom: 0; + background-color: rgba(34, 39, 46, 0.75); +} +.m-dox-search:target { + display: block; +} +.m-dox-search > a { + display: block; + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; +} +.m-dox-search-header { + margin-top: 2.5rem; + padding: 0.5rem 1rem; + height: 2rem; +} +.m-dox-search-header > div:first-child { + float: right; +} +.m-dox-search-content { + background-color: #22272e; + border-radius: 0.2rem; + padding: 1rem; +} +.m-dox-search input { + width: 100%; + height: 3rem; + font-size: 1.2rem; + border-width: 0; + color: #dcdcdc; + background-color: #34424d; + border-radius: 0.2rem; + margin-bottom: 1rem; + padding: 0 1rem; +} +.m-dox-search #search-notfound { + display: none; +} +.m-dox-search ul#search-results { + list-style-type: none; + padding-left: 0; + max-height: calc(100vh - 12.5rem); + overflow-y: auto; + display: none; +} +.m-dox-search ul#search-results li a { + display: block; + padding-left: 1rem; + padding-right: 1rem; + text-decoration: none; + width: 100%; + line-height: 1.5rem; + color: #dcdcdc; +} +.m-dox-search ul#search-results li a > div { + white-space: nowrap; + overflow: hidden; + direction: rtl; +} +.m-dox-search ul#search-results li#search-current a { + background-color: #34424d; +} +.m-dox-search-typed { + color: #5b9dd9; +} +.m-dox-search input[type="search"] { -webkit-appearance: textfield; } +.m-dox-search input[type="search"]::-webkit-search-decoration, +.m-dox-search input[type="search"]::-webkit-search-cancel-button, +.m-dox-search input[type="search"]::-webkit-search-results-button, +.m-dox-search input[type="search"]::-webkit-search-results-decoration { + display: none; +} diff --git a/css/m-dark.doxygen.compiled.css b/css/m-dark.doxygen.compiled.css index b52000f0..5183f313 100644 --- a/css/m-dark.doxygen.compiled.css +++ b/css/m-dark.doxygen.compiled.css @@ -175,3 +175,99 @@ article section.m-dox-details:target > div { border-left-width: 0.25rem; border-left-color: #a5c9ea; } +a.m-dox-search-icon { + padding-left: 1rem; + padding-right: 1rem; +} +a.m-dox-search-icon svg { + height: 0.9rem; + fill: #ffffff; +} +body > header > nav #m-navbar-collapse a.m-dox-search-icon svg { + vertical-align: -5%; +} +a.m-dox-search-icon:focus svg, a.m-dox-search-icon:hover svg, a.m-dox-search-icon:active svg { + fill: #a5c9ea; +} +.m-dox-search { + display: none; + z-index: 10; + position: fixed; + left: 0; + right: 0; + top: 0; + bottom: 0; + background-color: rgba(34, 39, 46, 0.75); +} +.m-dox-search:target { + display: block; +} +.m-dox-search > a { + display: block; + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; +} +.m-dox-search-header { + margin-top: 2.5rem; + padding: 0.5rem 1rem; + height: 2rem; +} +.m-dox-search-header > div:first-child { + float: right; +} +.m-dox-search-content { + background-color: #22272e; + border-radius: 0.2rem; + padding: 1rem; +} +.m-dox-search input { + width: 100%; + height: 3rem; + font-size: 1.2rem; + border-width: 0; + color: #dcdcdc; + background-color: #34424d; + border-radius: 0.2rem; + margin-bottom: 1rem; + padding: 0 1rem; +} +.m-dox-search #search-notfound { + display: none; +} +.m-dox-search ul#search-results { + list-style-type: none; + padding-left: 0; + max-height: calc(100vh - 12.5rem); + overflow-y: auto; + display: none; +} +.m-dox-search ul#search-results li a { + display: block; + padding-left: 1rem; + padding-right: 1rem; + text-decoration: none; + width: 100%; + line-height: 1.5rem; + color: #dcdcdc; +} +.m-dox-search ul#search-results li a > div { + white-space: nowrap; + overflow: hidden; + direction: rtl; +} +.m-dox-search ul#search-results li#search-current a { + background-color: #34424d; +} +.m-dox-search-typed { + color: #5b9dd9; +} +.m-dox-search input[type="search"] { -webkit-appearance: textfield; } +.m-dox-search input[type="search"]::-webkit-search-decoration, +.m-dox-search input[type="search"]::-webkit-search-cancel-button, +.m-dox-search input[type="search"]::-webkit-search-results-button, +.m-dox-search input[type="search"]::-webkit-search-results-decoration { + display: none; +} diff --git a/css/m-doxygen.css b/css/m-doxygen.css index 05a6bc28..d83db15f 100644 --- a/css/m-doxygen.css +++ b/css/m-doxygen.css @@ -179,4 +179,117 @@ article section.m-dox-details:target > div { border-left-color: var(--article-heading-color); } +a.m-dox-search-icon { + padding-left: 1rem; + padding-right: 1rem; +} +a.m-dox-search-icon svg { + height: 0.9rem; + fill: var(--header-link-color); +} +body > header > nav #m-navbar-collapse a.m-dox-search-icon svg { + vertical-align: -5%; +} +a.m-dox-search-icon:focus svg, a.m-dox-search-icon:hover svg, a.m-dox-search-icon:active svg { + fill: var(--header-link-active-color); +} +.m-dox-search { + display: none; + z-index: 10; + position: fixed; + left: 0; + right: 0; + top: 0; + bottom: 0; + background-color: var(--header-background-color-landing); +} +.m-dox-search:target { + display: block; +} +.m-dox-search > a { + display: block; + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; +} +.m-dox-search-header { + margin-top: 2.5rem; + padding: 0.5rem 1rem; + height: 2rem; +} +.m-dox-search-header > div:first-child { + float: right; +} +.m-dox-search-content { + background-color: var(--header-background-color); + border-radius: var(--border-radius); + padding: 1rem; +} +.m-dox-search input { + width: 100%; + height: 3rem; + font-size: 1.2rem; + border-width: 0; + color: var(--color); + background-color: var(--default-filled-background-color); + border-radius: var(--border-radius); + margin-bottom: 1rem; + padding: 0 1rem; /* putting it on all sides cuts text off in FF */ +} +.m-dox-search #search-notfound { + display: none; +} +.m-dox-search ul#search-results { + list-style-type: none; + padding-left: 0; + /* Size breakdown: + 2.5 margin of .m-dox-search-header from top + 2 height of .m-dox-search-header + 1 padding around .m-dox-search-header (twice 0.5rem) + 1 padding of .m-dox-search-content from top + 3 height of the input field + 1 margin under input + 1 padding of .m-dox-search-content from bottom + 1 margin under .m-dox-search-content + ------ + 12.5 total */ + max-height: calc(100vh - 12.5rem); + overflow-y: auto; + display: none; +} +.m-dox-search ul#search-results li a { + display: block; + padding-left: 1rem; + padding-right: 1rem; + text-decoration: none; + width: 100%; + line-height: 1.5rem; + color: var(--color); +} +.m-dox-search ul#search-results li a > div { + white-space: nowrap; + overflow: hidden; + /* This is here in order to cut the text off at the left side. Besides this + there's special patching needed for punctuation characters, see search.js + for details. */ + direction: rtl; +} +.m-dox-search ul#search-results li#search-current a { + background-color: var(--default-filled-background-color); +} +.m-dox-search-typed { + color: var(--link-color); +} + +/* WELL THANK YOU WEBKIT! FOR SURE I WANTED ALL THAT SHIT HERE! */ +.m-dox-search input[type="search"] { -webkit-appearance: textfield; } +.m-dox-search input[type="search"]::-webkit-search-decoration, +.m-dox-search input[type="search"]::-webkit-search-cancel-button, +.m-dox-search input[type="search"]::-webkit-search-results-button, +.m-dox-search input[type="search"]::-webkit-search-results-decoration { + display: none; +} + /* kate: indent-width 2; */ diff --git a/css/m-light+doxygen.compiled.css b/css/m-light+doxygen.compiled.css index 91174f5d..198d0e20 100644 --- a/css/m-light+doxygen.compiled.css +++ b/css/m-light+doxygen.compiled.css @@ -1925,3 +1925,99 @@ article section.m-dox-details:target > div { border-left-width: 0.25rem; border-left-color: #cb4b16; } +a.m-dox-search-icon { + padding-left: 1rem; + padding-right: 1rem; +} +a.m-dox-search-icon svg { + height: 0.9rem; + fill: #000000; +} +body > header > nav #m-navbar-collapse a.m-dox-search-icon svg { + vertical-align: -5%; +} +a.m-dox-search-icon:focus svg, a.m-dox-search-icon:hover svg, a.m-dox-search-icon:active svg { + fill: #cb4b16; +} +.m-dox-search { + display: none; + z-index: 10; + position: fixed; + left: 0; + right: 0; + top: 0; + bottom: 0; + background-color: rgba(255, 255, 255, 0.75); +} +.m-dox-search:target { + display: block; +} +.m-dox-search > a { + display: block; + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; +} +.m-dox-search-header { + margin-top: 2.5rem; + padding: 0.5rem 1rem; + height: 2rem; +} +.m-dox-search-header > div:first-child { + float: right; +} +.m-dox-search-content { + background-color: #ffffff; + border-radius: 0.2rem; + padding: 1rem; +} +.m-dox-search input { + width: 100%; + height: 3rem; + font-size: 1.2rem; + border-width: 0; + color: #000000; + background-color: #fbf0ec; + border-radius: 0.2rem; + margin-bottom: 1rem; + padding: 0 1rem; +} +.m-dox-search #search-notfound { + display: none; +} +.m-dox-search ul#search-results { + list-style-type: none; + padding-left: 0; + max-height: calc(100vh - 12.5rem); + overflow-y: auto; + display: none; +} +.m-dox-search ul#search-results li a { + display: block; + padding-left: 1rem; + padding-right: 1rem; + text-decoration: none; + width: 100%; + line-height: 1.5rem; + color: #000000; +} +.m-dox-search ul#search-results li a > div { + white-space: nowrap; + overflow: hidden; + direction: rtl; +} +.m-dox-search ul#search-results li#search-current a { + background-color: #fbf0ec; +} +.m-dox-search-typed { + color: #ea7944; +} +.m-dox-search input[type="search"] { -webkit-appearance: textfield; } +.m-dox-search input[type="search"]::-webkit-search-decoration, +.m-dox-search input[type="search"]::-webkit-search-cancel-button, +.m-dox-search input[type="search"]::-webkit-search-results-button, +.m-dox-search input[type="search"]::-webkit-search-results-decoration { + display: none; +} diff --git a/css/m-light.doxygen.compiled.css b/css/m-light.doxygen.compiled.css index ed0050e5..98676224 100644 --- a/css/m-light.doxygen.compiled.css +++ b/css/m-light.doxygen.compiled.css @@ -175,3 +175,99 @@ article section.m-dox-details:target > div { border-left-width: 0.25rem; border-left-color: #cb4b16; } +a.m-dox-search-icon { + padding-left: 1rem; + padding-right: 1rem; +} +a.m-dox-search-icon svg { + height: 0.9rem; + fill: #000000; +} +body > header > nav #m-navbar-collapse a.m-dox-search-icon svg { + vertical-align: -5%; +} +a.m-dox-search-icon:focus svg, a.m-dox-search-icon:hover svg, a.m-dox-search-icon:active svg { + fill: #cb4b16; +} +.m-dox-search { + display: none; + z-index: 10; + position: fixed; + left: 0; + right: 0; + top: 0; + bottom: 0; + background-color: rgba(255, 255, 255, 0.75); +} +.m-dox-search:target { + display: block; +} +.m-dox-search > a { + display: block; + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; +} +.m-dox-search-header { + margin-top: 2.5rem; + padding: 0.5rem 1rem; + height: 2rem; +} +.m-dox-search-header > div:first-child { + float: right; +} +.m-dox-search-content { + background-color: #ffffff; + border-radius: 0.2rem; + padding: 1rem; +} +.m-dox-search input { + width: 100%; + height: 3rem; + font-size: 1.2rem; + border-width: 0; + color: #000000; + background-color: #fbf0ec; + border-radius: 0.2rem; + margin-bottom: 1rem; + padding: 0 1rem; +} +.m-dox-search #search-notfound { + display: none; +} +.m-dox-search ul#search-results { + list-style-type: none; + padding-left: 0; + max-height: calc(100vh - 12.5rem); + overflow-y: auto; + display: none; +} +.m-dox-search ul#search-results li a { + display: block; + padding-left: 1rem; + padding-right: 1rem; + text-decoration: none; + width: 100%; + line-height: 1.5rem; + color: #000000; +} +.m-dox-search ul#search-results li a > div { + white-space: nowrap; + overflow: hidden; + direction: rtl; +} +.m-dox-search ul#search-results li#search-current a { + background-color: #fbf0ec; +} +.m-dox-search-typed { + color: #ea7944; +} +.m-dox-search input[type="search"] { -webkit-appearance: textfield; } +.m-dox-search input[type="search"]::-webkit-search-decoration, +.m-dox-search input[type="search"]::-webkit-search-cancel-button, +.m-dox-search input[type="search"]::-webkit-search-results-button, +.m-dox-search input[type="search"]::-webkit-search-results-decoration { + display: none; +} diff --git a/doc/doxygen.rst b/doc/doxygen.rst index b41e5218..7f90b90a 100644 --- a/doc/doxygen.rst +++ b/doc/doxygen.rst @@ -38,6 +38,8 @@ Doxygen theme :language: ini .. role:: jinja(code) :language: jinja +.. role:: js(code) + :language: js .. role:: py(code) :language: py .. role:: sh(code) @@ -201,8 +203,6 @@ amount of generated content for no added value. `Not yet implemented features`_ ------------------------------- -- Code search. I want to provide something that's actually usable to replace - the terribly slow stock client-side search, but I'm not there yet. - Clickable symbols in code snippets. Doxygen has quite a lot of false positives while a lot of symbols stay unmatched. I need to find a way around that. @@ -280,6 +280,25 @@ Variable Description :ini:`M_EXPAND_INNER_TYPES` Whether to expand inner types (e.g. a class inside a class) in the symbol tree. If not set, ``NO`` is used. +:ini:`M_SEARCH_DISABLED` Disable search functionality. If this + option is set, no search data is compiled + and the rendered HTML does not contain any + search-related UI or support. If not set, + ``NO`` is used. +:ini:`M_SEARCH_DOWNLOAD_BINARY` Download search data as a binary to save + bandwidth and initial processing time. If + not set, ``NO`` is used. See `Search`_ for + more information. +:ini:`M_SEARCH_HELP` HTML code to display as help text on empty + search popup. If not set, a default message + is used. Has effect only if + :ini:`M_SEARCH_DISABLED` is not ``YES``. +:ini:`M_SEARCH_EXTERNAL_URL` URL for external search. The ``{query}`` + placeholder is replaced with urlencoded + search string. If not set, no external + search is offered. See `Search`_ for more + information. Has effect only if + :ini:`M_SEARCH_DISABLED` is not ``YES``. =================================== ======================================= Note that namespace, directory and page lists are always fully expanded as @@ -376,6 +395,34 @@ This will put links to namespaces Foo, Bar and Utils as a sub-items of a top-level *Namespaces* item and links to two subdirectories as sub-items of the *Files* item. +`Search`_ +--------- + +Symbol search is implemented using JavaScript Typed Arrays and does not need +any server-side functionality to perform well --- the client automatically +downloads a tightly packed binary containing search data and performs search +directly on it. + +However, due to `restrictions of Chromium-based browsers `_, +it's not possible to download data using :js:`XMLHttpRequest` when served from +local file-system. Because of that, the search defaults to producing a +Base85-encoded representation of the search binary and loading that +asynchronously as a plain JavaScript file. This results in the search data +being 25% larger, but since this is for serving from a local filesystem, it's +not considered a problem. If your docs are accessed through a server (or you +don't need Chrome support), enable the :ini:`M_SEARCH_DOWNLOAD_BINARY` option. + +If :ini:`M_SEARCH_EXTERNAL_URL` is specified, full-text search using an +external search engine is offered if nothing is found for given string or if +the user has JavaScript disabled. It's recommended to restrict the search to +a particular domain or add additional keywords to the search query to filter +out irrelevant results. Example, using Google search engine and restricting +the search to a subdomain: + +.. code:: ini + + M_SEARCH_EXTERNAL_URL = "https://google.com/search?q=site:doc.magnum.graphics+{query}" + `Command-line options`_ ======================= @@ -856,6 +903,8 @@ Property Description template file from above :py:`compound.id` Unique compound identifier, usually corresponding to output file name +:py:`compound.url` Compound URL (or where this file will + be saved) :py:`compound.name` Compound name :py:`compound.templates` Template specification. Set only for classes. See `Template properties`_ for @@ -964,8 +1013,6 @@ Property Description structs and unions; on templated classes contains also the list of template parameter names. -:py:`compound.save_as` Filename including extension where the - result will be saved ======================================= ======================================= `Navigation properties`_ diff --git a/doxygen/.gitignore b/doxygen/.gitignore index 68f19739..ccb3caeb 100644 --- a/doxygen/.gitignore +++ b/doxygen/.gitignore @@ -1,2 +1,4 @@ test/*/html/ test/*/xml/ +test/js-test-data/ +coverage/ diff --git a/doxygen/dox2html5.py b/doxygen/dox2html5.py index 9294184b..20bfb11d 100755 --- a/doxygen/dox2html5.py +++ b/doxygen/dox2html5.py @@ -26,6 +26,7 @@ import xml.etree.ElementTree as ET import argparse +import base64 import sys import re import html @@ -177,10 +178,13 @@ class State: def __init__(self): self.basedir = '' self.compounds: Dict[str, Any] = {} + self.search: List[Any] = [] self.examples: List[Any] = [] self.doxyfile: Dict[str, str] = {} self.images: List[str] = [] self.current = '' + self.current_prefix = [] + self.current_url = '' def slugify(text: str) -> str: # Maybe some Unicode normalization would be nice here? @@ -1085,11 +1089,18 @@ def parse_enum(state: State, element: ET.Element): if ''.join(enumvalue.find('briefdescription').itertext()).strip(): logging.warning("{}: ignoring brief description of enum value {}::{}".format(state.current, enum.name, value.name)) value.description = parse_desc(state, enumvalue.find('detaileddescription')) - if value.description: enum.has_value_details = True + if value.description: + enum.has_value_details = True + if not state.doxyfile['M_SEARCH_DISABLED']: + state.search += [(state.current_url + '#' + value.id, state.current_prefix + [enum.name], value.name)] enum.values += [value] enum.has_details = enum.description or enum.has_value_details - return enum if enum.brief or enum.has_details or enum.has_value_details else None + if enum.brief or enum.has_details or enum.has_value_details: + if not state.doxyfile['M_SEARCH_DISABLED']: + state.search += [(state.current_url + '#' + enum.id, state.current_prefix, enum.name)] + return enum + return None def parse_template_params(state: State, element: ET.Element, description): if element is None: return False, None @@ -1143,7 +1154,10 @@ def parse_typedef(state: State, element: ET.Element): typedef.has_template_details, typedef.templates = parse_template_params(state, element.find('templateparamlist'), templates) typedef.has_details = typedef.description or typedef.has_template_details - return typedef if typedef.brief or typedef.has_details else None + if typedef.brief or typedef.has_details: + state.search += [(state.current_url + '#' + typedef.id, state.current_prefix, typedef.name)] + return typedef + return None def parse_func(state: State, element: ET.Element): assert element.tag == 'memberdef' and element.attrib['kind'] == 'function' @@ -1234,7 +1248,11 @@ def parse_func(state: State, element: ET.Element): if params: logging.warning("{}: function parameter description doesn't match parameter names: {}".format(state.current, repr(params))) func.has_details = func.description or func.has_template_details or func.has_param_details or func.return_value - return func if func.brief or func.has_details else None + if func.brief or func.has_details: + if not state.doxyfile['M_SEARCH_DISABLED']: + state.search += [(state.current_url + '#' + func.id, state.current_prefix, func.name + '()')] + return func + return None def parse_var(state: State, element: ET.Element): assert element.tag == 'memberdef' and element.attrib['kind'] == 'variable' @@ -1255,7 +1273,11 @@ def parse_var(state: State, element: ET.Element): var.description = parse_var_desc(state, element) var.has_details = not not var.description - return var if var.brief or var.has_details else None + if var.brief or var.has_details: + if not state.doxyfile['M_SEARCH_DISABLED']: + state.search += [(state.current_url + '#' + var.id, state.current_prefix, var.name)] + return var + return None def parse_define(state: State, element: ET.Element): assert element.tag == 'memberdef' and element.attrib['kind'] == 'define' @@ -1284,7 +1306,11 @@ def parse_define(state: State, element: ET.Element): if params: logging.warning("{}: define parameter description doesn't match parameter names: {}".format(state.current, repr(params))) define.has_details = define.description or define.return_value - return define if define.brief or define.has_details else None + if define.brief or define.has_details: + if not state.doxyfile['M_SEARCH_DISABLED']: + state.search += [(state.current_url + '#' + define.id, [], define.name + ('' if define.params is None else '()'))] + return define + return None def extract_metadata(state: State, xml): logging.debug("Extracting metadata from {}".format(os.path.basename(xml))) @@ -1423,6 +1449,61 @@ def postprocess_state(state: State): if state.doxyfile['M_FAVICON']: state.doxyfile['M_FAVICON'] = (state.doxyfile['M_FAVICON'], mimetypes.guess_type(state.doxyfile['M_FAVICON'])[0]) +def _build_search_data(state: State, prefix, id: str, trie: Trie, map: ResultMap): + compound = state.compounds[id] + if not compound.brief and not compound.has_details: return 0 + + # Add current item name to prefix list + prefixed_name = prefix + [compound.leaf_name] + + # Calculate fully-qualified name + if compound.kind in ['namespace', 'struct', 'class', 'union']: + joiner = '::' + elif compound.kind in ['file', 'dir']: + joiner = '/' + else: + joiner = '' + + # If just a leaf name, add it once + if not joiner: + # TODO: escape elsewhere so i don't have to unescape here + name = html.unescape(compound.leaf_name) + trie.insert(name.lower(), map.add(name, compound.url)) + + # Otherwise add it multiple times with all possible prefixes + else: + # TODO: escape elsewhere so i don't have to unescape here + index = map.add(html.unescape(joiner.join(prefixed_name)), compound.url) + for i in range(len(prefixed_name)): + trie.insert(html.unescape(joiner.join(prefixed_name[i:])).lower(), index) + + for i in compound.children: + if i in state.compounds: + _build_search_data(state, prefixed_name, i, trie, map) + +def build_search_data(state: State) -> bytearray: + trie = Trie() + map = ResultMap() + + for id, compound in state.compounds.items(): + if compound.parent: continue # start from the root + _build_search_data(state, [], id, trie, map) + + for url, prefix, name in state.search: + # Add current item name to prefix list + prefixed_name = prefix + [name] + + # TODO: escape elsewhere so i don't have to unescape here + index = map.add(html.unescape('::'.join(prefixed_name)), url) + for i in range(len(prefixed_name)): + trie.insert(html.unescape('::'.join(prefixed_name[i:])).lower(), index) + + return serialize_search_data(trie, map) + +def base85encode_search_data(data: bytearray) -> bytearray: + return (b"/* Generated by http://mcss.mosra.cz/doxygen/. Do not edit. */\n" + + b"Search.load('" + base64.b85encode(data, True) + b"');\n") + def parse_xml(state: State, xml: str): # Reset counter for unique math formulas m.math.counter = 0 @@ -1461,6 +1542,8 @@ def parse_xml(state: State, xml: str): # Compound name is page filename, so we have to use title there. The same # is for groups. compound.name = compounddef.find('title').text if compound.kind in ['page', 'group'] else compounddef.find('compoundname').text + # Compound URL is ID, except for index page + compound.url = (compounddef.find('compoundname').text if compound.kind == 'page' else compound.id) + '.html' compound.has_template_details = False compound.templates = None compound.brief = parse_desc(state, compounddef.find('briefdescription')) @@ -1512,6 +1595,14 @@ def parse_xml(state: State, xml: str): for i in reversed(path_reverse): compound.breadcrumb += [(state.compounds[i].leaf_name, state.compounds[i].url)] + # Save current prefix for search + state.current_prefix = [name for name, _ in compound.breadcrumb] + else: + state.current_prefix = [] + + # Save current compound URL for search data building + state.current_url = compound.url + if compound.kind == 'page': # Drop TOC for pages, if not requested if compounddef.find('tableofcontents') is None: @@ -1969,11 +2060,6 @@ def parse_xml(state: State, xml: str): parsed = Empty() parsed.version = root.attrib['version'] - - # Decide about save as filename. Pages mess this up, because index page has - # "indexpage" as a name so we have to use the compound name instead - parsed.save_as = (compounddef.find('compoundname').text if compound.kind == 'page' else compound.id) + '.html' - parsed.compound = compound return parsed @@ -2117,7 +2203,11 @@ def parse_doxyfile(state: State, doxyfile, config = None): 'M_FAVICON': [], 'M_LINKS_NAVBAR1': ['pages', 'namespaces'], 'M_LINKS_NAVBAR2': ['annotated', 'files'], - 'M_PAGE_FINE_PRINT': ['[default]'] + 'M_PAGE_FINE_PRINT': ['[default]'], + 'M_SEARCH_DISABLED': ['NO'], + 'M_SEARCH_DOWNLOAD_BINARY': ['NO'], + 'M_SEARCH_HELP': ['Search for symbols, headers, pages or example source files. You can omit any prefix from the symbol or file path.'], + 'M_SEARCH_EXTERNAL_URL': [''] } def parse_value(var): @@ -2194,7 +2284,9 @@ def parse_doxyfile(state: State, doxyfile, config = None): 'M_PAGE_HEADER', 'M_PAGE_FINE_PRINT', 'M_THEME_COLOR', - 'M_FAVICON']: + 'M_FAVICON', + 'M_SEARCH_HELP', + 'M_SEARCH_EXTERNAL_URL']: if i in config: state.doxyfile[i] = ' '.join(config[i]) # Int values that we want @@ -2203,7 +2295,9 @@ def parse_doxyfile(state: State, doxyfile, config = None): if i in config: state.doxyfile[i] = int(' '.join(config[i])) # Boolean values that we want - for i in ['M_EXPAND_INNER_TYPES']: + for i in ['M_EXPAND_INNER_TYPES', + 'M_SEARCH_DISABLED', + 'M_SEARCH_DOWNLOAD_BINARY']: if i in config: state.doxyfile[i] = ' '.join(config[i]) == 'YES' # List values that we want. Drop empty lines. @@ -2277,10 +2371,10 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_ template = env.get_template('{}.html'.format(parsed.compound.kind)) rendered = template.render(compound=parsed.compound, DOXYGEN_VERSION=parsed.version, - FILENAME=parsed.save_as, + FILENAME=parsed.compound.url, **state.doxyfile) - output = os.path.join(html_output, parsed.save_as) + output = os.path.join(html_output, parsed.compound.url) with open(output, 'w') as f: f.write(rendered) @@ -2302,8 +2396,18 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_ with open(output, 'w') as f: f.write(rendered) + if not state.doxyfile['M_SEARCH_DISABLED']: + data = build_search_data(state) + + if state.doxyfile['M_SEARCH_DOWNLOAD_BINARY']: + with open(os.path.join(html_output, "searchdata.bin"), 'wb') as f: + f.write(data) + else: + with open(os.path.join(html_output, "searchdata.js"), 'wb') as f: + f.write(base85encode_search_data(data)) + # Copy all referenced files - for i in state.images + state.doxyfile['HTML_EXTRA_STYLESHEET'] + state.doxyfile['HTML_EXTRA_FILES']: + for i in state.images + state.doxyfile['HTML_EXTRA_STYLESHEET'] + state.doxyfile['HTML_EXTRA_FILES'] + ([] if state.doxyfile['M_SEARCH_DISABLED'] else ['search.js']): # Skip absolute URLs if urllib.parse.urlparse(i).netloc: continue diff --git a/doxygen/search.js b/doxygen/search.js new file mode 100644 index 00000000..22e8b5cf --- /dev/null +++ b/doxygen/search.js @@ -0,0 +1,405 @@ +/* + This file is part of m.css. + + Copyright © 2017, 2018 Vladimír VondruÅ¡ + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +"use strict"; /* it summons the Cthulhu in a proper way, they say */ + +var Search = { + trie: null, + map: null, + symbolCount: 0, + maxResults: 0, + + /* Always contains at least the root node offset and then one node offset + per entered character */ + searchString: '', + searchStack: [], + + init: function(buffer, maxResults) { + let view = new DataView(buffer); + + /* The file is too short to contain at least the headers */ + if(view.byteLength < 18) { + console.error("Search data too short"); + return false; + } + + if(view.getUint8(0) != 'M'.charCodeAt(0) || + view.getUint8(1) != 'C'.charCodeAt(0) || + view.getUint8(2) != 'S'.charCodeAt(0)) { + console.error("Invalid search data signature"); + return false; + } + + if(view.getUint8(3) != 0) { + console.error("Invalid search data version"); + return false; + } + + /* Separate the data into the trie and the result map */ + let mapOffset = view.getUint32(4, true); + this.trie = new DataView(buffer, 8, mapOffset - 8); + this.map = new DataView(buffer, mapOffset); + + /* Set initial properties */ + this.symbolCount = (this.map.getUint32(0, true) & 0x00ffffff)/4 - 1; + this.maxResults = maxResults ? maxResults : 100; + this.searchString = ''; + this.searchStack = [this.trie.getUint32(0, true)]; + + /* istanbul ignore if */ + if(typeof document !== 'undefined') { + document.getElementById('search-symbolcount').innerHTML = + this.symbolCount + ' symbols (' + + Math.round(buffer.byteLength/102.4)/10 + "kB)"; + document.getElementById('search-input').disabled = false; + document.getElementById('search-input').placeholder = "Type something here …"; + document.getElementById('search-input').focus(); + + /* Search for the input value (there might be something already, + for example when going back in the browser) */ + let value = document.getElementById('search-input').value; + if(value.length) Search.renderResults(value, Search.search(value)); + } + + return true; + }, + + download: /* istanbul ignore next */ function(url) { + var req = window.XDomainRequest ? new XDomainRequest() : new XMLHttpRequest(); + if(!req) return; + + req.open("GET", url, true); + req.responseType = 'arraybuffer'; + req.onreadystatechange = function() { + if(req.readyState != 4) return; + + Search.init(req.response); + } + req.send(); + }, + + base85decode: function(base85string) { + function charValue(char) { + if(char >= 48 && char < 58) /* 0-9 -> 0-9 */ + return char - 48 + 0; + if(char >= 65 && char < 91) /* A-Z -> 10-35 */ + return char - 65 + 10; + if(char >= 97 && char < 123) /* a-z -> 36-61 */ + return char - 97 + 36; + if(char == 33) /* ! -> 62 */ + return 62; + /* skipping 34 (') */ + if(char >= 35 && char < 39) /* #-& -> 63-66 */ + return char - 35 + 63; + /* skipping 39 (") */ + if(char >= 40 && char < 44) /* (-+ -> 67-70 */ + return char - 40 + 67; + /* skipping 44 (,) */ + if(char == 45) /* - -> 71 */ + return 71; + if(char >= 59 && char < 65) /* ;-@ -> 72-77 */ + return char - 59 + 72; + if(char >= 94 && char < 97) /* ^-` -> 78-80 */ + return char - 94 + 78; + if(char >= 123 && char < 127) /* {-~ -> 81-84 */ + return char - 123 + 81; + + return 0; /* Interpret padding values as zeros */ + } + + /* Pad the string for easier decode later. We don't read past the file + end, so it doesn't matter what garbage is there. */ + if(base85string.length % 5) { + console.log("Expected properly padded base85 data"); + return; + } + + let buffer = new ArrayBuffer(base85string.length*4/5); + let data8 = new DataView(buffer); + for(let i = 0; i < base85string.length; i += 5) { + let char1 = charValue(base85string.charCodeAt(i + 0)); + let char2 = charValue(base85string.charCodeAt(i + 1)); + let char3 = charValue(base85string.charCodeAt(i + 2)); + let char4 = charValue(base85string.charCodeAt(i + 3)); + let char5 = charValue(base85string.charCodeAt(i + 4)); + + data8.setUint32(i*4/5, char5 + + char4*85 + + char3*85*85 + + char2*85*85*85 + + char1*85*85*85*85, false); /* BE, yes */ + } + + return buffer; + }, + + load: function(base85string) { + return this.init(this.base85decode(base85string)); + }, + + search: function(searchString) { + /* Normalize the search string first */ + searchString = searchString.toLowerCase().trim(); + + /* TODO: maybe i could make use of InputEvent.data and others here */ + + /* Find longest common prefix of previous and current value so we don't + need to needlessly search again */ + let max = Math.min(searchString.length, this.searchString.length); + let commonPrefix = 0; + for(; commonPrefix != max; ++commonPrefix) + if(searchString[commonPrefix] != this.searchString[commonPrefix]) break; + + /* Drop items off the stack if it has has more than is needed for the + common prefix (it needs to have at least one item, though) */ + if(commonPrefix + 1 < this.searchStack.length) + this.searchStack.splice(commonPrefix + 1, this.searchStack.length - commonPrefix - 1); + + /* Add new characters from the search string */ + let foundPrefix = commonPrefix; + for(; foundPrefix != searchString.length; ++foundPrefix) { + /* Calculate offset and count of children */ + let offset = this.searchStack[this.searchStack.length - 1]; + let nodeSize = this.trie.getUint8(offset)*2; + let relChildOffset = 2 + this.trie.getUint8(offset + 1)*2; + let childCount = (nodeSize - relChildOffset)/4; + + /* Go through all children and find the next offset */ + let childOffset = offset + relChildOffset; + let found = false; + for(let j = 0; j != childCount; ++j) { + if(String.fromCharCode(this.trie.getUint8(childOffset + j*4 + 3)) != searchString[foundPrefix]) + continue; + + this.searchStack.push(this.trie.getUint32(childOffset + j*4, true) & 0x00ffffff); + found = true; + break; + } + + /* Character not found, exit */ + if(!found) break; + } + + /* Save the whole found prefix for next time */ + this.searchString = searchString.substr(0, foundPrefix); + + /* If the whole thing was not found, return an empty result and offer + external search */ + if(foundPrefix != searchString.length) { + /* istanbul ignore if */ + if(typeof document !== 'undefined') { + let link = document.getElementById('search-external'); + if(link) + link.href = link.dataset.searchEngine.replace('{query}', encodeURIComponent(searchString)); + } + return []; + } + + /* Otherwise recursively gather the results */ + let results = []; + this.gatherResults(this.searchStack[this.searchStack.length - 1], 0, results); + return results; + }, + + gatherResults: function(offset, suffixLength, results) { + let valueCount = this.trie.getUint8(offset + 1); + + /* Populate the results with all values associated with this node */ + for(let i = 0; i != valueCount; ++i) { + let index = this.trie.getUint16(offset + (i + 1)*2, true); + //let flags = this.map.getUint8(index*4 + 3); /* not used yet */ + let resultOffset = this.map.getUint32(index*4, true) & 0x00ffffff; + let nextResultOffset = this.map.getUint32((index + 1)*4, true) & 0x00ffffff; + + let name = ''; + let j = resultOffset; + for(; j != nextResultOffset; ++j) { + let c = this.map.getUint8(j); + + /* End of null-delimited name */ + if(!c) { + ++j; + break; /* null-delimited */ + } + + name += String.fromCharCode(c); /* eheh. IS THIS FAST?! */ + } + + let url = ''; + for(; j != nextResultOffset; ++j) { + url += String.fromCharCode(this.map.getUint8(j)); + } + + results.push({name: name, url: url, suffixLength: suffixLength}); + + /* 'nuff said. */ + /* TODO: remove once proper barriers are in */ + if(results.length >= this.maxResults) return true; + } + + /* Dig deeper. If the child already has enough, return. */ + /* TODO: hmmm. this is helluvalot duplicated code. hmm. */ + let nodeSize = this.trie.getUint8(offset)*2; + let relChildOffset = 2 + this.trie.getUint8(offset + 1)*2; + let childCount = (nodeSize - relChildOffset)/4; + let childOffset = offset + relChildOffset; + for(let j = 0; j != childCount; ++j) + if(this.gatherResults(this.trie.getUint32(childOffset + j*4, true) & 0x00ffffff, suffixLength + 1, results)) + return true; + + /* Still hungry. */ + return false; + }, + + escapeForRtl: function(name) { + /* Besides the obvious escaping of HTML entities we also need + to escape punctuation, because due to the RTL hack to cut + text off on left side the punctuation characters get + reordered (of course). Prepending ‎ works for most + characters, parentheses we need to *soak* in it. But only + the right ones. And that for some reason needs to be also for &. + Huh. https://en.wikipedia.org/wiki/Right-to-left_mark */ + return name.replace(/[\"&<>]/g, function (a) { + return { '"': '"', '&': '&', '<': '<', '>': '>' }[a]; + }).replace(/[:=]/g, '‎$&').replace(/(\)|>|&|\/)/g, '‎$&‎'); + }, + + renderResults: /* istanbul ignore next */ function(value, results) { + /* Normalize the value length so the slicing works properly */ + value = value.trim(); + + if(!value.length) { + document.getElementById('search-help').style.display = 'block'; + document.getElementById('search-results').style.display = 'none'; + document.getElementById('search-notfound').style.display = 'none'; + return; + } + + document.getElementById('search-help').style.display = 'none'; + + if(results.length) { + document.getElementById('search-results').style.display = 'block'; + document.getElementById('search-notfound').style.display = 'none'; + + var list = ''; + for(let i = 0; i != results.length; ++i) { + list += '
' + this.escapeForRtl(results[i].name.substr(0, results[i].name.length - value.length - results[i].suffixLength)) + '' + this.escapeForRtl(results[i].name.substr(results[i].name.length - value.length - results[i].suffixLength, value.length)) + '' + this.escapeForRtl(results[i].name.substr(results[i].name.length - results[i].suffixLength)) + '
'; + } + document.getElementById('search-results').innerHTML = list; + document.getElementById('search-current').scrollIntoView(true); + + } else { + document.getElementById('search-results').style.display = 'none'; + document.getElementById('search-notfound').style.display = 'block'; + } + }, +}; + +/* istanbul ignore next */ +function selectResult(event) { + if(event.currentTarget.parentNode.id == 'search-current') return; + + let current = document.getElementById('search-current'); + current.id = ''; + event.currentTarget.parentNode.id = 'search-current'; +} + +/* istanbul ignore next */ +function showSearch() { + window.location.hash = '#search'; + document.getElementById('search-input').value = ''; + document.getElementById('search-input').focus(); + document.getElementById('search-results').style.display = 'none'; + document.getElementById('search-notfound').style.display = 'none'; + document.getElementById('search-help').style.display = 'block'; + return false; +} + +/* istanbul ignore next */ +function hideSearch() { + window.location.hash = '#!'; + window.history.pushState('', '', window.location.pathname); + return false; +} + +/* Only in case we're running in a browser. Why a simple if(document) doesn't + work is beyond me. */ /* istanbul ignore if */ +if(typeof document !== 'undefined') { + document.getElementById('search-input').oninput = function(event) { + let value = document.getElementById('search-input').value; + Search.renderResults(value, Search.search(value)); + }; + + document.onkeydown = function(event) { + /* Search shown */ + if(window.location.hash == '#search') { + /* Close the search */ + if(event.key == 'Escape') { + hideSearch(); + + /* Select next item */ + } else if(event.key == 'ArrowDown' || (event.key == 'Tab' && !event.shiftKey)) { + let current = document.getElementById('search-current'); + if(current) { + let next = current.nextSibling; + if(next) { + current.id = ''; + next.id = 'search-current'; + next.scrollIntoView(false); + } + } + return false; /* so the keypress doesn't affect input cursor */ + + /* Select prev item */ + } else if(event.key == 'ArrowUp' || (event.key == 'Tab' && event.shiftKey)) { + let current = document.getElementById('search-current'); + if(current) { + let prev = current.previousSibling; + if(prev) { + current.id = ''; + prev.id = 'search-current'; + prev.scrollIntoView(false); + } + } + return false; /* so the keypress doesn't affect input cursor */ + + /* Go to result */ + } else if(event.key == 'Enter') { + document.getElementById('search-current').firstElementChild.click(); + return false; /* so the keypress doesn't affect input cursor */ + } + + /* Search hidden */ + } else { + /* Open the search on the T or Tab key */ + if(event.key.toLowerCase() == 't' || event.key == 'Tab') { + showSearch(); + return false; /* so T doesn't get entered into the box */ + } + } + }; +} + +/* For Node.js testing */ /* istanbul ignore else */ +if(typeof module !== 'undefined') { module.exports = { Search: Search }; } diff --git a/doxygen/templates/base.html b/doxygen/templates/base.html index 46963d0a..2128cc6f 100644 --- a/doxygen/templates/base.html +++ b/doxygen/templates/base.html @@ -21,9 +21,16 @@
{{ PROJECT_NAME }}{% if PROJECT_BRIEF %} {{ PROJECT_BRIEF }}{% endif %} - {% if M_LINKS_NAVBAR1 or M_LINKS_NAVBAR2 %} - - + {% if M_LINKS_NAVBAR1 or M_LINKS_NAVBAR2 or not M_SEARCH_DISABLED %} +
+ {% if not M_SEARCH_DISABLED %} + + + + {% endif %} + + +
+{% if not M_SEARCH_DISABLED %} + + +{% if M_SEARCH_DOWNLOAD_BINARY %} + +{% else %} + +{% endif %} +{% endif %} {% if M_PAGE_FINE_PRINT %}