chiark / gitweb /
documentation: encode format version into search data filename.
authorVladimír Vondruš <mosra@centrum.cz>
Wed, 17 Jul 2019 16:33:41 +0000 (18:33 +0200)
committerVladimír Vondruš <mosra@centrum.cz>
Thu, 18 Jul 2019 11:34:34 +0000 (13:34 +0200)
The browser having a stale data (and a stale script) cached is okay-ish
as things still mostly work, but having a stale data with a script
expecting a new version or vice versa is *bad*, so prevent that by
having the script always download file of the exact version it expects.

This is a bit less useful with docs served from local filesystem (as
the filename is encoded in HTML pages rather than in the script), but
there I don't assume the browser is doing much caching so that should be
okay.

23 files changed:
documentation/_search.py
documentation/doxygen.py
documentation/python.py
documentation/search.js
documentation/templates/doxygen/base.html
documentation/templates/python/base.html
documentation/test_doxygen/layout/pages.html
documentation/test_doxygen/layout_generated_doxyfile/index.html
documentation/test_doxygen/layout_minimal/index.html
documentation/test_doxygen/layout_search_binary/index.html
documentation/test_doxygen/layout_search_opensearch/index.html
documentation/test_doxygen/test_layout.py
documentation/test_doxygen/test_search.py
documentation/test_doxygen/test_undocumented.py
documentation/test_doxygen/undocumented/File_8h.html
documentation/test_doxygen/undocumented/annotated.html
documentation/test_doxygen/undocumented/classClass.html
documentation/test_doxygen/undocumented/dir_4b0d5f8864bf89936129251a2d32609b.html
documentation/test_doxygen/undocumented/files.html
documentation/test_doxygen/undocumented/group__group.html
documentation/test_doxygen/undocumented/namespaceNamespace.html
documentation/test_doxygen/undocumented/structNamespace_1_1ClassInANamespace.html
documentation/test_python/layout/index.html

index 12e31f63716cc5b068ec98e3816e1d8e606d3acf..fb48e7f2e1589ccdca19fa6e98a852ac05c272c3 100644 (file)
@@ -30,6 +30,10 @@ import struct
 from enum import Flag
 from types import SimpleNamespace as Empty
 
+searchdata_format_version = 0
+searchdata_filename = f'searchdata-v{searchdata_format_version}.bin'
+searchdata_filename_b85 = f'searchdata-v{searchdata_format_version}.js'
+
 class ResultFlag(Flag):
     @staticmethod
     def from_type(flag: 'ResultFlag', type) -> 'ResultFlag':
@@ -362,7 +366,7 @@ def serialize_search_data(trie: Trie, map: ResultMap, symbol_count, merge_subtre
     serialized_trie = trie.serialize(merge_subtrees=merge_subtrees)
     serialized_map = map.serialize(merge_prefixes=merge_prefixes)
     # magic header, version, symbol count, offset of result map
-    return search_data_header_struct.pack(b'MCS', 0, symbol_count, len(serialized_trie) + 10) + serialized_trie + serialized_map
+    return search_data_header_struct.pack(b'MCS', searchdata_format_version, symbol_count, len(serialized_trie) + 10) + serialized_trie + serialized_map
 
 def base85encode_search_data(data: bytearray) -> bytearray:
     return (b"/* Generated by https://mcss.mosra.cz/documentation/doxygen/. Do not edit. */\n" +
@@ -490,7 +494,7 @@ def pretty_print_map(serialized: bytes, *, entryTypeClass, colors=False):
 def pretty_print(serialized: bytes, *, entryTypeClass, show_merged=False, show_lookahead_barriers=True, colors=False):
     magic, version, symbol_count, map_offset = search_data_header_struct.unpack_from(serialized)
     assert magic == b'MCS'
-    assert version == 0
+    assert version == searchdata_format_version
 
     pretty_trie, stats = pretty_print_trie(serialized[search_data_header_struct.size:map_offset], show_merged=show_merged, show_lookahead_barriers=show_lookahead_barriers, colors=colors)
     pretty_map = pretty_print_map(serialized[map_offset:], entryTypeClass=entryTypeClass, colors=colors)
index 388befb853800001334e0068577b0d4bdfa1e3c2..9d56e271da268db36e7cc9fac7c7e4ff7806f79f 100755 (executable)
@@ -47,7 +47,7 @@ from pygments import highlight
 from pygments.formatters import HtmlFormatter
 from pygments.lexers import TextLexer, BashSessionLexer, get_lexer_by_name, find_lexer_class_for_filename
 
-from _search import ResultFlag, ResultMap, Trie, serialize_search_data, base85encode_search_data
+from _search import ResultFlag, ResultMap, Trie, serialize_search_data, base85encode_search_data, searchdata_filename, searchdata_filename_b85, searchdata_format_version
 
 sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plugins'))
 import dot2svg
@@ -3498,6 +3498,7 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_
                 rendered = template.render(index=parsed.index,
                     DOXYGEN_VERSION=parsed.version,
                     FILENAME=file,
+                    SEARCHDATA_FORMAT_VERSION=searchdata_format_version,
                     **state.doxyfile)
 
                 output = os.path.join(html_output, file)
@@ -3516,6 +3517,7 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_
             rendered = template.render(compound=parsed.compound,
                 DOXYGEN_VERSION=parsed.version,
                 FILENAME=parsed.compound.url,
+                SEARCHDATA_FORMAT_VERSION=searchdata_format_version,
                 **state.doxyfile)
 
             output = os.path.join(html_output, parsed.compound.url)
@@ -3542,6 +3544,7 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_
         rendered = template.render(compound=compound,
             DOXYGEN_VERSION='0',
             FILENAME='index.html',
+            SEARCHDATA_FORMAT_VERSION=searchdata_format_version,
             **state.doxyfile)
         output = os.path.join(html_output, 'index.html')
         with open(output, 'wb') as f:
@@ -3558,10 +3561,10 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_
         data = build_search_data(state, add_lookahead_barriers=search_add_lookahead_barriers, merge_subtrees=search_merge_subtrees, merge_prefixes=search_merge_prefixes)
 
         if state.doxyfile['M_SEARCH_DOWNLOAD_BINARY']:
-            with open(os.path.join(html_output, "searchdata.bin"), 'wb') as f:
+            with open(os.path.join(html_output, searchdata_filename), 'wb') as f:
                 f.write(data)
         else:
-            with open(os.path.join(html_output, "searchdata.js"), 'wb') as f:
+            with open(os.path.join(html_output, searchdata_filename_b85), 'wb') as f:
                 f.write(base85encode_search_data(data))
 
         # OpenSearch metadata, in case we have the base URL
index 27ad2d1e2775d2244bcd19571e24c6b85cc39a53..e6436e1b8681e476e2e448c1be788acc5a37f697 100755 (executable)
@@ -51,6 +51,8 @@ from docutils.transforms import Transform
 
 import jinja2
 
+from _search import searchdata_format_version
+
 sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plugins'))
 import m.htmlsanity
 
@@ -1236,7 +1238,9 @@ def extract_data_doc(state: State, parent, path: List[str], data):
 
 def render(config, template: str, page, env: jinja2.Environment):
     template = env.get_template(template)
-    rendered = template.render(page=page, URL=page.url, **config)
+    rendered = template.render(page=page,
+        URL=page.url,
+        SEARCHDATA_FORMAT_VERSION=searchdata_format_version, **config)
     with open(os.path.join(config['OUTPUT'], page.filename), 'wb') as f:
         f.write(rendered.encode('utf-8'))
         # Add back a trailing newline so we don't need to bother with
index 69e055f8b002d2af838d5ce72aa75abe26468ecf..423f5e1c9ba90deae98ca48d212867cd76562e94 100644 (file)
@@ -25,6 +25,8 @@
 "use strict"; /* it summons the Cthulhu in a proper way, they say */
 
 var Search = {
+    formatVersion: 0, /* the data filename contains this number too */
+
     trie: null,
     map: null,
     dataSize: 0,
@@ -65,7 +67,7 @@ var Search = {
             return false;
         }
 
-        if(view.getUint8(3) != 0) {
+        if(view.getUint8(3) != this.formatVersion) {
             console.error("Invalid search data version");
             return false;
         }
@@ -112,11 +114,11 @@ var Search = {
         return true;
     },
 
-    download: /* istanbul ignore next */ function(url) {
+    download: /* istanbul ignore next */ function(urlBase) {
         var req = window.XDomainRequest ? new XDomainRequest() : new XMLHttpRequest();
         if(!req) return;
 
-        req.open("GET", url, true);
+        req.open("GET", urlBase + "searchdata-v" + this.formatVersion + ".bin", true);
         req.responseType = 'arraybuffer';
         req.onreadystatechange = function() {
             if(req.readyState != 4) return;
index 03273ead2b533004b7c5f4d3a53051cf2987216f..5ef2a7bce2eaa3d5054fb270eaa8e42236ef5edc 100644 (file)
 <script src="search.js"></script>
 {% if M_SEARCH_DOWNLOAD_BINARY %}
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 {% else %}
-<script src="searchdata.js" async="async"></script>
+<script src="searchdata-v{{ SEARCHDATA_FORMAT_VERSION }}.js" async="async"></script>
 {% endif %}
 {% endif %}
 {% if M_PAGE_FINE_PRINT %}
 {#- sanity checks for variables that should be always defined -#}
 {% if FILENAME is not defined %}{{ FILENAME.is_not_defined_the_script_is_broken }}{% endif %}
 {% if DOXYGEN_VERSION is not defined %}{{ DOXYGEN_VERSION.is_not_defined_the_script_is_broken }}{% endif %}
+{% if SEARCHDATA_FORMAT_VERSION is defined %}{{ SEARCHDATA_FORMAT_VERSION.is_not_defined_the_script_is_broken }}{% endif %}
index a3dd819861062d37e4b45fb4ce0cafad69a87d41..b6ca07c6ad480e02ae5c98cd90c44b8776a837b0 100644 (file)
 <script src="search.js"></script>
 {% if SEARCH_DOWNLOAD_BINARY %}
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 {% else %}
-<script src="searchdata.js" async="async"></script>
+<script src="searchdata-v{{ SEARCHDATA_FORMAT_VERSION }}.js" async="async"></script>
 {% endif %}
 {% endif %}
 {% if FINE_PRINT %}
 </html>
 {#- sanity checks for variables that should be always defined -#}
 {% if URL is not defined %}{{ URL.is_not_defined_the_script_is_broken }}{% endif %}
+{% if SEARCHDATA_FORMAT_VERSION is defined %}{{ SEARCHDATA_FORMAT_VERSION.is_not_defined_the_script_is_broken }}{% endif %}
index 55d0dedc1eec1d8cf9395ad6f647e165fb62fe0c..57019fed6bbb5754640cf232f21d763595c8d59a 100644 (file)
   </div>
 </div>
 <script src="search.js"></script>
-<script src="searchdata.js" async="async"></script>
+<script src="searchdata-v0.js" async="async"></script>
 <footer><nav>
   <div class="m-container">
     <div class="m-row">
index d9d912baecdf9dcbc824765a5ffad3862aa8023c..d099c574ffb80828587c640feda3e8ae4ef6bedc 100644 (file)
@@ -85,7 +85,7 @@
   </div>
 </div>
 <script src="search.js"></script>
-<script src="searchdata.js" async="async"></script>
+<script src="searchdata-v0.js" async="async"></script>
 <footer><nav>
   <div class="m-container">
     <div class="m-row">
index d9d912baecdf9dcbc824765a5ffad3862aa8023c..d099c574ffb80828587c640feda3e8ae4ef6bedc 100644 (file)
@@ -85,7 +85,7 @@
   </div>
 </div>
 <script src="search.js"></script>
-<script src="searchdata.js" async="async"></script>
+<script src="searchdata-v0.js" async="async"></script>
 <footer><nav>
   <div class="m-container">
     <div class="m-row">
index 4a3edf4bf7ff935bec807c2e4b21ed690fcd1cff..37f498246654b20bef5ab893c71b8eecf731427e 100644 (file)
@@ -70,7 +70,7 @@
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index ba961e2839fe00e3d560a65a9a3e4accae81aa82..2d402f3f046e24072c195574def9061ca294050e 100644 (file)
@@ -71,6 +71,6 @@
   </div>
 </div>
 <script src="search.js"></script>
-<script src="searchdata.js" async="async"></script>
+<script src="searchdata-v0.js" async="async"></script>
 </body>
 </html>
index d40605c8cdf460de5c12fb0b7716e87c95a513ac..65599bd10d2ddc681e670a30f52016dadcf31368 100644 (file)
@@ -25,6 +25,7 @@
 import os
 import subprocess
 
+from _search import searchdata_filename, searchdata_filename_b85
 from . import BaseTestCase
 
 class Layout(BaseTestCase):
@@ -36,7 +37,7 @@ class Layout(BaseTestCase):
         self.assertEqual(*self.actual_expected_contents('pages.html'))
         self.assertTrue(os.path.exists(os.path.join(self.path, 'html', 'm-dark+documentation.compiled.css')))
         self.assertTrue(os.path.exists(os.path.join(self.path, 'html', 'search.js')))
-        self.assertTrue(os.path.exists(os.path.join(self.path, 'html', 'searchdata.js')))
+        self.assertTrue(os.path.exists(os.path.join(self.path, 'html', searchdata_filename_b85)))
         self.assertTrue(os.path.exists(os.path.join(self.path, 'html', 'favicon-light.png')))
 
 class GeneratedDoxyfile(BaseTestCase):
@@ -98,7 +99,7 @@ class SearchBinary(BaseTestCase):
     def test(self):
         self.run_doxygen(wildcard='indexpage.xml')
         self.assertEqual(*self.actual_expected_contents('index.html'))
-        self.assertTrue(os.path.exists(os.path.join(self.path, 'html', 'searchdata.bin')))
+        self.assertTrue(os.path.exists(os.path.join(self.path, 'html', searchdata_filename)))
 
 class SearchOpenSearch(BaseTestCase):
     def __init__(self, *args, **kwargs):
index 37a9d5a8e841c4452f4d9da65e2696c417f856e4..69ba9b3df8be7874bd489d2360502e9c2d97530a 100755 (executable)
@@ -31,7 +31,7 @@ import unittest
 from types import SimpleNamespace as Empty
 
 from doxygen import EntryType
-from _search import Trie, ResultMap, ResultFlag, serialize_search_data, pretty_print_trie, pretty_print_map, pretty_print
+from _search import Trie, ResultMap, ResultFlag, serialize_search_data, pretty_print_trie, pretty_print_map, pretty_print, searchdata_filename
 
 from test_doxygen import IntegrationTestCase
 
@@ -243,7 +243,7 @@ class Search(IntegrationTestCase):
     def test(self):
         self.run_doxygen(index_pages=[], wildcard='*.xml')
 
-        with open(os.path.join(self.path, 'html', 'searchdata.bin'), 'rb') as f:
+        with open(os.path.join(self.path, 'html', searchdata_filename), 'rb') as f:
             serialized = f.read()
             search_data_pretty = pretty_print(serialized, entryTypeClass=EntryType)[0]
         #print(search_data_pretty)
@@ -420,7 +420,7 @@ class SearchLongSuffixLength(IntegrationTestCase):
     def test(self):
         self.run_doxygen(index_pages=[], wildcard='*.xml')
 
-        with open(os.path.join(self.path, 'html', 'searchdata.bin'), 'rb') as f:
+        with open(os.path.join(self.path, 'html', searchdata_filename), 'rb') as f:
             serialized = f.read()
             search_data_pretty = pretty_print(serialized, entryTypeClass=EntryType)[0]
         #print(search_data_pretty)
index 07a80fa4f7a9aeb8f517bc5f2712309db2c64bcc..9bc0052adcd594550a1b6208171fdc7b682bd1ef 100644 (file)
@@ -26,7 +26,7 @@
 
 import os
 
-from _search import search_data_header_struct
+from _search import search_data_header_struct, searchdata_filename
 
 from . import IntegrationTestCase
 
@@ -54,7 +54,7 @@ class Test(IntegrationTestCase):
         # Test we have all symbols in search data. It's enough to assert the
         # count, it equal to symbol count in the header file
         # TODO: reuse the search data deserialization API once done
-        with open(os.path.join(self.path, 'html', 'searchdata.bin'), 'rb') as f:
+        with open(os.path.join(self.path, 'html', searchdata_filename), 'rb') as f:
             serialized = f.read()
             magic, version, symbol_count, map_offset = search_data_header_struct.unpack_from(serialized)
             self.assertEqual(symbol_count, 44)
index 8a4e4da21865b885d010db49795792c5d52b7695..3d5f085480f6182a68d07aa80b0d54675938b695 100644 (file)
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index c174cad087d605ea5aed95f55d856bebe57f7491..c2c14e711472eb3c1e36a660c6fb6bbaea35d7b4 100644 (file)
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index dc5a7217b7115da13e9b70910c3a9728cfcf4d87..fd5c5b6c11de0db90bce4fdc196fca6588f4ade2 100644 (file)
@@ -86,7 +86,7 @@
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index 8152377aa1c7b9c09cf4141d4607b49fe19e4ceb..5ca37f1bf8ac64e3e06dd1429301881723239a39 100644 (file)
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index e6935a60a9de01a48309e48136c05536101a6558..c9d4a4cd9e9219abf94f99714ffd6923fac03515 100644 (file)
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index 9165c5b968ec3ecf056c9910ffc6dccc54286eaa..1f56f81741d93f541b86977182f13d05a1dbb98b 100644 (file)
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index 6776b58547682b94d2b60a65484ad1a9ad54e8fa..e8157188f03bc9422f4bdeaa9dbc4e4bebbe2240 100644 (file)
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index defb747f329d213bd9e2efd1a2543a148f2480bf..f81e0127a8db8fbe0e34896189f901684fbdc377 100644 (file)
 </div>
 <script src="search.js"></script>
 <script>
-  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1) + "searchdata.bin");
+  Search.download(window.location.pathname.substr(0, window.location.pathname.lastIndexOf('/') + 1));
 </script>
 </body>
 </html>
index 54a48216724dfce78b40012811efeb147c856cbc..8fe9e0197630ed0a70d444488448efd2af70cdd8 100644 (file)
@@ -91,7 +91,7 @@
   </div>
 </div>
 <script src="search.js"></script>
-<script src="searchdata.js" async="async"></script>
+<script src="searchdata-v0.js" async="async"></script>
 <footer><nav>
   <div class="m-container">
     <div class="m-row">