From: Vladimír Vondruš Date: Mon, 11 Jun 2018 13:32:49 +0000 (+0200) Subject: Implement caching of rendered math output. X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~cjwatson/git?a=commitdiff_plain;h=9c59340a2aa2d65e05c0b222f0a06f5752a954ca;p=blog.git Implement caching of rendered math output. Stores the output of divsvgm in a dict and (de)serializes that in the filesystem to preserve it across runs. Rough speedup: * For the Magnum website, initial generation went down from 14 seconds to 9, with subsequent runs just 2.5 seconds. * For the Magnum docs, generation time went down from 1:33 to just 14 seconds. --- diff --git a/doc/doxygen.rst b/doc/doxygen.rst index d2bb35d0..f910c5bd 100644 --- a/doc/doxygen.rst +++ b/doc/doxygen.rst @@ -317,6 +317,12 @@ Variable Description :ini:`M_EXPAND_INNER_TYPES` Whether to expand inner types (e.g. a class inside a class) in the symbol tree. If not set, ``NO`` is used. +:ini:`M_MATH_CACHE_FILE` File to cache rendered math formulas. If + not set, ``m.math.cache`` file in the + output directory is used. Old cached output + is periodically pruned and new formulas + added to the file. Set it empty to disable + caching. :ini:`M_SEARCH_DISABLED` Disable search functionality. If this option is set, no search data is compiled and the rendered HTML does not contain any diff --git a/doc/plugins/math-and-code.rst b/doc/plugins/math-and-code.rst index 09b4e568..7828a3d9 100644 --- a/doc/plugins/math-and-code.rst +++ b/doc/plugins/math-and-code.rst @@ -63,6 +63,8 @@ assumes presence of `m.htmlsanity <{filename}/plugins/htmlsanity.rst>`_. .. code:: python PLUGINS += ['m.htmlsanity', 'm.math'] + M_MATH_RENDER_AS_CODE = False + M_MATH_CACHE_FILE = 'm.math.cache' In addition you need some LaTeX distribution installed. Use your distribution package manager, for example on Ubuntu: @@ -146,6 +148,12 @@ want to add additional CSS classes, derive a custom role from it. Quaternion-conjugated dual quaternion is :math-info:`\hat q^* = q_0^* + q_\epsilon^*`, while dual-conjugation gives :math:`\overline{\hat q} = q_0 - \epsilon q_\epsilon`. +The :py:`M_MATH_CACHE_FILE` setting (defaulting to ``m.math.cache`` in the +site root directory) describes a file used for caching rendered LaTeX math +formulas for speeding up subsequent runs. Old cached output is periodically +pruned and new formulas added to the file. Set it to :py:`None` to disable +caching. + .. note-info:: LaTeX can be sometimes a real pain to set up. In order to make it possible diff --git a/doxygen/dox2html5.py b/doxygen/dox2html5.py index 2f776acb..0a758f46 100755 --- a/doxygen/dox2html5.py +++ b/doxygen/dox2html5.py @@ -1208,7 +1208,7 @@ def parse_desc_internal(state: State, element: ET.Element, immediate_parent: ET. # Assume that Doxygen wrapped the formula properly to distinguish # between inline, block or special environment - rendered = latex2svg.latex2svg('{}'.format(i.text), params=latex2svgextra.params) + depth, svg = latex2svgextra.fetch_cached_or_render('{}'.format(i.text)) # We should have decided about block/inline above assert formula_block is not None @@ -1216,15 +1216,15 @@ def parse_desc_internal(state: State, element: ET.Element, immediate_parent: ET. has_block_elements = True out.parsed += '

{}

'.format( ' ' + add_css_class if add_css_class else '', - latex2svgextra.patch(i.text, rendered, '')) + latex2svgextra.patch(i.text, svg, '')) else: # CSS classes and styling for proper vertical alignment. Depth is relative # to font size, describes how below the line the text is. Scaling it back # to 12pt font, scaled by 125% as set above in the config. attribs = ' class="m-math{}" style="vertical-align: -{:.1f}pt;"'.format( ' ' + add_inline_css_class if add_inline_css_class else '', - (rendered['depth'] or 0.0)*12*1.25) - out.parsed += latex2svgextra.patch(i.text, rendered, attribs) + (depth or 0.0)*12*1.25) + out.parsed += latex2svgextra.patch(i.text, svg, attribs) # Inline elements elif i.tag == 'linebreak': @@ -2733,6 +2733,7 @@ def parse_doxyfile(state: State, doxyfile, config = None): 'M_FAVICON': ['favicon-dark.png'], 'M_LINKS_NAVBAR1': ['pages', 'namespaces'], 'M_LINKS_NAVBAR2': ['annotated', 'files'], + 'M_MATH_CACHE_FILE': ['m.math.cache'], 'M_PAGE_FINE_PRINT': ['[default]'], 'M_SEARCH_DISABLED': ['NO'], 'M_SEARCH_DOWNLOAD_BINARY': ['NO'], @@ -2835,6 +2836,7 @@ list using ↓ and 'M_PAGE_FINE_PRINT', 'M_THEME_COLOR', 'M_FAVICON', + 'M_MATH_CACHE_FILE', 'M_SEARCH_HELP', 'M_SEARCH_EXTERNAL_URL']: if i in config: state.doxyfile[i] = ' '.join(config[i]) @@ -2879,6 +2881,15 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_ xml_files = [os.path.join(xml_input, f) for f in glob.glob(os.path.join(xml_input, wildcard))] html_output = os.path.join(state.basedir, state.doxyfile['OUTPUT_DIRECTORY'], state.doxyfile['HTML_OUTPUT']) + # If math rendering cache is not disabled, load the previous version. If + # there is no cache, reset the cache to an empty state to avoid + # order-dependent issues when testing + math_cache_file = os.path.join(state.basedir, state.doxyfile['OUTPUT_DIRECTORY'], state.doxyfile['M_MATH_CACHE_FILE']) + if state.doxyfile['M_MATH_CACHE_FILE'] and os.path.exists(math_cache_file): + latex2svgextra.unpickle_cache(math_cache_file) + else: + latex2svgextra.unpickle_cache(None) + if sort_globbed_files: xml_files_metadata.sort() xml_files.sort() @@ -2982,6 +2993,10 @@ def run(doxyfile, templates=default_templates, wildcard=default_wildcard, index_ logging.debug("copying {} to output".format(i)) shutil.copy(i, os.path.join(html_output, os.path.basename(i))) + # Save updated math cache file + if state.doxyfile['M_MATH_CACHE_FILE']: + latex2svgextra.pickle_cache(math_cache_file) + if __name__ == '__main__': # pragma: no cover parser = argparse.ArgumentParser() parser.add_argument('doxyfile', help="where the Doxyfile is") diff --git a/doxygen/test/contents_custom/Doxyfile b/doxygen/test/contents_custom/Doxyfile index a0a7a22b..ecc3d464 100644 --- a/doxygen/test/contents_custom/Doxyfile +++ b/doxygen/test/contents_custom/Doxyfile @@ -11,6 +11,7 @@ M_THEME_COLOR = M_FAVICON = M_LINKS_NAVBAR1 = M_LINKS_NAVBAR2 = +M_MATH_CACHE_FILE = M_SEARCH_DISABLED = YES ALIASES = \ diff --git a/doxygen/test/contents_math/Doxyfile b/doxygen/test/contents_math/Doxyfile index d1b0fce7..59c69bc4 100644 --- a/doxygen/test/contents_math/Doxyfile +++ b/doxygen/test/contents_math/Doxyfile @@ -11,3 +11,4 @@ M_FAVICON = M_LINKS_NAVBAR1 = M_LINKS_NAVBAR2 = M_SEARCH_DISABLED = YES +M_MATH_CACHE_FILE = diff --git a/doxygen/test/contents_math_cached/Doxyfile b/doxygen/test/contents_math_cached/Doxyfile new file mode 100644 index 00000000..933dfe74 --- /dev/null +++ b/doxygen/test/contents_math_cached/Doxyfile @@ -0,0 +1,14 @@ +INPUT = input.dox +QUIET = YES +GENERATE_HTML = NO +GENERATE_LATEX = NO +GENERATE_XML = YES +XML_PROGRAMLISTING = NO + +M_PAGE_FINE_PRINT = +M_THEME_COLOR = +M_FAVICON = +M_LINKS_NAVBAR1 = +M_LINKS_NAVBAR2 = +M_SEARCH_DISABLED = YES +M_MATH_CACHE_FILE = xml/math.cache diff --git a/doxygen/test/contents_math_cached/input.dox b/doxygen/test/contents_math_cached/input.dox new file mode 100644 index 00000000..4a1943ee --- /dev/null +++ b/doxygen/test/contents_math_cached/input.dox @@ -0,0 +1,26 @@ +/** @page math Math + +In order to actually test use of the cache, I need to cheat a bit. Inline +formula which is pi in the source but @f$ \pi @f$ here. Then a block +formula which is a Pythagorean theorem in source but not in the output: + +@f[ + a^2 + b^2 = c^2 +@f] + +*/ + +/* The (uncached) output of below should be the same as the cached output of + above. Be sure to preserve the line breaks as well. */ + +/** @page math-uncached Math + +In order to actually test use of the cache, I need to cheat a bit. Inline +formula which is pi in the source but @f$ \frac{\tau}{2} @f$ here. Then a block +formula which is a Pythagorean theorem in source but not in the output: + +@f[ + a^3 + b^3 \neq c^3 +@f] + +*/ diff --git a/doxygen/test/contents_math_cached/math.html b/doxygen/test/contents_math_cached/math.html new file mode 100644 index 00000000..533ac542 --- /dev/null +++ b/doxygen/test/contents_math_cached/math.html @@ -0,0 +1,70 @@ + + + + + Math | My Project + + + + + +

+ Math +

In order to actually test use of the cache, I need to cheat a bit. Inline formula which is pi in the source but here. Then a block formula which is a Pythagorean theorem in source but not in the output:

+ + diff --git a/doxygen/test/test_contents.py b/doxygen/test/test_contents.py index be1090e9..c916e8d1 100644 --- a/doxygen/test/test_contents.py +++ b/doxygen/test/test_contents.py @@ -23,10 +23,13 @@ # import os +import pickle import shutil import subprocess import unittest +from hashlib import sha1 + from distutils.version import LooseVersion from . import BaseTestCase, IntegrationTestCase, doxygen_version @@ -118,6 +121,111 @@ class Math(IntegrationTestCase): with self.assertRaises(subprocess.CalledProcessError) as context: self.run_dox2html5(wildcard='error.xml') +class MathCached(IntegrationTestCase): + def __init__(self, *args, **kwargs): + super().__init__(__file__, 'math_cached', *args, **kwargs) + + # Actually generated from $ \frac{\tau}{2} $ tho + self.tau_half_hash = sha1("""$ \pi $""".encode('utf-8')).digest() + self.tau_half = """ + +""" + # Actually generated from \[ a^3 + b^3 \neq c^3 \] tho + self.fermat_hash = sha1("""\[ a^2 + b^2 = c^2 \]""".encode('utf-8')).digest() + self.fermat = """ + +""" + + # This is using the cache, so doesn't matter if LaTeX is found or not + def test(self): + math_cache = (0, 5, { + self.tau_half_hash: (5, 0.3448408333333333, self.tau_half), + self.fermat_hash: (5, 0.0, self.fermat), + b'does not exist': (5, 0.0, 'something')}) + with open(os.path.join(self.path, 'xml/math.cache'), 'wb') as f: + pickle.dump(math_cache, f) + + self.run_dox2html5(wildcard='math.xml') + self.assertEqual(*self.actual_expected_contents('math.html')) + + # Expect that after the operation the global cache age is bumped, + # unused entries removed and used entries age bumped as well + with open(os.path.join(self.path, 'xml/math.cache'), 'rb') as f: + math_cache_actual = pickle.load(f) + math_cache_expected = (0, 6, { + self.tau_half_hash: (6, 0.3448408333333333, self.tau_half), + self.fermat_hash: (6, 0.0, self.fermat)}) + self.assertEqual(math_cache_actual, math_cache_expected) + + @unittest.skipUnless(shutil.which('latex'), + "Math rendering requires LaTeX installed") + def test_uncached(self): + # Write some bullshit there, which gets immediately reset + with open(os.path.join(self.path, 'xml/math.cache'), 'wb') as f: + pickle.dump((1337, 0, {"something different"}), f) + + self.run_dox2html5(wildcard='math-uncached.xml') + + with open(os.path.join(self.path, 'math.html')) as f: + expected_contents = f.read().strip() + # The file is the same expect for titles of the formulas. Replace them + # and then compare. + with open(os.path.join(self.path, 'html', 'math-uncached.html')) as f: + actual_contents = f.read().strip().replace('a^3 + b^3 \\neq c^3', 'a^2 + b^2 = c^2').replace('\\frac{\\tau}{2}', '\pi') + + self.assertEqual(actual_contents, expected_contents) + + # Expect that after the operation the global cache is filled + with open(os.path.join(self.path, 'xml/math.cache'), 'rb') as f: + math_cache_actual = pickle.load(f) + math_cache_expected = (0, 0, { + sha1("$ \\frac{\\tau}{2} $".encode('utf-8')).digest(): + (0, 0.3448408333333333, self.tau_half), + sha1("\\[ a^3 + b^3 \\neq c^3 \\]".encode('utf-8')).digest(): + (0, 0.0, self.fermat)}) + self.assertEqual(math_cache_actual, math_cache_expected) + + def test_noop(self): + if os.path.exists(os.path.join(self.path, 'xml/math.cache')): + shutil.rmtree(os.path.join(self.path, 'xml/math.cache')) + + # Processing without any math + self.run_dox2html5(wildcard='indexpage.xml') + + # There should be no file generated + self.assertFalse(os.path.exists(os.path.join(self.path, 'xml/math.cache'))) + class Tagfile(IntegrationTestCase): def __init__(self, *args, **kwargs): super().__init__(__file__, 'tagfile', *args, **kwargs) diff --git a/doxygen/test/test_doxyfile.py b/doxygen/test/test_doxyfile.py index 3a7b4621..e058b083 100644 --- a/doxygen/test/test_doxyfile.py +++ b/doxygen/test/test_doxyfile.py @@ -46,6 +46,7 @@ class Doxyfile(unittest.TestCase): 'M_FILE_TREE_EXPAND_LEVELS': 1, 'M_LINKS_NAVBAR1': ['pages', 'namespaces'], 'M_LINKS_NAVBAR2': ['annotated', 'files'], + 'M_MATH_CACHE_FILE': 'm.math.cache', 'M_PAGE_FINE_PRINT': 'this is "quotes"', 'M_PAGE_HEADER': 'this is "quotes" \'apostrophes\'', 'M_SEARCH_DISABLED': False, diff --git a/pelican-plugins/latex2svgextra.py b/pelican-plugins/latex2svgextra.py index 0386d5fe..f28b5c0b 100644 --- a/pelican-plugins/latex2svgextra.py +++ b/pelican-plugins/latex2svgextra.py @@ -22,7 +22,9 @@ # DEALINGS IN THE SOFTWARE. # +import pickle import re +from hashlib import sha1 import latex2svg @@ -63,11 +65,63 @@ _unique_dst = r"""\g='\geq{counter}-\g'""" # Reset back to zero on start of a new page for reproducible behavior. counter = 0 +# Cache for rendered formulas (source formula sha1 -> (depth, svg data)). The +# counter is not included +_cache_version = 0 +_cache = None + +# Fetch cached formula or render it and add to the cache. The formula has to +# be already wrapped in $, $$ etc. environment. +def fetch_cached_or_render(formula): + global _cache + + # unpickle_cache() should be called first + assert _cache + + hash = sha1(formula.encode('utf-8')).digest() + if not _cache or not hash in _cache[2]: + out = latex2svg.latex2svg(formula, params=params) + _cache[2][hash] = (_cache[1], out['depth'], out['svg']) + else: + _cache[2][hash] = (_cache[1], _cache[2][hash][1], _cache[2][hash][2]) + return (_cache[2][hash][1], _cache[2][hash][2]) + +def unpickle_cache(file): + global _cache + + if file: + with open(file, 'rb') as f: + _cache = pickle.load(f) + else: + _cache = None + + # Reset the cache if not valid or not expected version + if not _cache or _cache[0] != _cache_version: + _cache = (_cache_version, 0, {}) + + # Otherwise bump cache age + else: _cache = (_cache[0], _cache[1] + 1, _cache[2]) + +def pickle_cache(file): + global _cache + + # Don't save any file if there is nothing + if not _cache or not _cache[2]: return + + # Prune entries that were not used + cache_to_save = (_cache_version, _cache[1], {}) + for hash, entry in _cache[2].items(): + if entry[0] != _cache[1]: continue + cache_to_save[2][hash] = entry + + with open(file, 'wb') as f: + pickle.dump(cache_to_save, f) + # Patches the output from dvisvgm # - w/o the XML preamble and needless xmlns attributes # - unique element IDs (see `counter`) # - adds additional `attribs` to the + + Math | A Pelican Blog + + + + + + +

Math

+ +

In order to actually test use of the cache, I need to cheat a bit. Inline +formula which is pi in the source but here. Then a block +formula which is a Pythagorean theorem in source but not in the output:

+ +

+ + diff --git a/pelican-plugins/m/test/math_cached/page.rst b/pelican-plugins/m/test/math_cached/page.rst new file mode 100644 index 00000000..adeda7c6 --- /dev/null +++ b/pelican-plugins/m/test/math_cached/page.rst @@ -0,0 +1,10 @@ +Math +#### + +In order to actually test use of the cache, I need to cheat a bit. Inline +formula which is pi in the source but :math:`\pi` here. Then a block +formula which is a Pythagorean theorem in source but not in the output: + +.. math:: + + a^2 + b^2 = c^2 diff --git a/pelican-plugins/m/test/math_uncached/page.rst b/pelican-plugins/m/test/math_uncached/page.rst new file mode 100644 index 00000000..6a7f42af --- /dev/null +++ b/pelican-plugins/m/test/math_uncached/page.rst @@ -0,0 +1,10 @@ +Math +#### + +In order to actually test use of the cache, I need to cheat a bit. Inline +formula which is pi in the source but :math:`\frac{\tau}{2}` here. Then a block +formula which is a Pythagorean theorem in source but not in the output: + +.. math:: + + a^3 + b^3 \neq c^3 diff --git a/pelican-plugins/m/test/test_math.py b/pelican-plugins/m/test/test_math.py index 8a933685..ba4130a9 100644 --- a/pelican-plugins/m/test/test_math.py +++ b/pelican-plugins/m/test/test_math.py @@ -22,13 +22,17 @@ # DEALINGS IN THE SOFTWARE. # +import os +import pickle import sys import shutil import unittest +from hashlib import sha1 + from distutils.version import LooseVersion -from m.test import PluginTestCase +from . import PluginTestCase class Math(PluginTestCase): def __init__(self, *args, **kwargs): @@ -38,11 +42,15 @@ class Math(PluginTestCase): "The math plugin requires at least Python 3.5 and LaTeX installed") def test(self): self.run_pelican({ - 'PLUGINS': ['m.htmlsanity', 'm.math'] + 'PLUGINS': ['m.htmlsanity', 'm.math'], + 'M_MATH_CACHE_FILE': None }) self.assertEqual(*self.actual_expected_contents('page.html')) + # No file should be generated when math caching is disabled + self.assertFalse(os.path.exists(os.path.join(self.path, 'm.math.cache'))) + def test_code_fallback(self): self.run_pelican({ 'PLUGINS': ['m.htmlsanity', 'm.math'], @@ -50,3 +58,117 @@ class Math(PluginTestCase): }) self.assertEqual(*self.actual_expected_contents('page.html', 'page-code-fallback.html')) + + # No file should be generated when there was nothing to cache + self.assertFalse(os.path.exists(os.path.join(self.path, 'm.math.cache'))) + +# Actually generated from $\frac{\tau}{2}$ tho +tau_half_hash = sha1("""$\pi$""".encode('utf-8')).digest() +tau_half = """ + +""" + +# Actually generated from $$a^3 + b^3 \neq c^3$$ tho +fermat_hash = sha1("""$$a^2 + b^2 = c^2$$""".encode('utf-8')).digest() +fermat = """ + +""" + +class MathCached(PluginTestCase): + def __init__(self, *args, **kwargs): + super().__init__(__file__, 'cached', *args, **kwargs) + + # This is using the cache, so doesn't matter if LaTeX is found or not + def test(self): + cache_file = os.path.join(self.path, 'math.cache') + + math_cache = (0, 5, { + tau_half_hash: (5, 0.3448408333333333, tau_half), + fermat_hash: (5, 0.0, fermat), + b'does not exist': (5, 0.0, 'something')}) + with open(cache_file, 'wb') as f: + pickle.dump(math_cache, f) + + self.run_pelican({ + 'PLUGINS': ['m.htmlsanity', 'm.math'], + 'M_MATH_CACHE_FILE': cache_file + }) + + self.assertEqual(*self.actual_expected_contents('page.html')) + + # Expect that after the operation the global cache age is bumped, + # unused entries removed and used entries age bumped as well + with open(cache_file, 'rb') as f: + math_cache_actual = pickle.load(f) + math_cache_expected = (0, 6, { + tau_half_hash: (6, 0.3448408333333333, tau_half), + fermat_hash: (6, 0.0, fermat)}) + self.assertEqual(math_cache_actual, math_cache_expected) + +class MathUncached(PluginTestCase): + def __init__(self, *args, **kwargs): + super().__init__(__file__, 'uncached', *args, **kwargs) + + @unittest.skipUnless(shutil.which('latex'), + "Math rendering requires LaTeX installed") + def test(self): + cache_file = os.path.join(self.path, 'math.cache') + + # Write some bullshit there, which gets immediately reset + with open(cache_file, 'wb') as f: + pickle.dump((1337, 0, {"something different"}), f) + + self.run_pelican({ + 'PLUGINS': ['m.htmlsanity', 'm.math'], + 'M_MATH_CACHE_FILE': cache_file + }) + + with open(os.path.join(self.path, '../math_cached/page.html')) as f: + expected_contents = f.read().strip() + # The file is the same expect for titles of the formulas. Replace them + # and then compare. + with open(os.path.join(self.path, 'output', 'page.html')) as f: + actual_contents = f.read().strip().replace('a^3 + b^3 \\neq c^3', 'a^2 + b^2 = c^2').replace('\\frac{\\tau}{2}', '\pi') + + self.assertEqual(actual_contents, expected_contents) + + # Expect that after the operation the global cache is filled + with open(cache_file, 'rb') as f: + math_cache_actual = pickle.load(f) + math_cache_expected = (0, 0, { + sha1("$\\frac{\\tau}{2}$".encode('utf-8')).digest(): + (0, 0.3448408333333333, tau_half), + sha1("$$a^3 + b^3 \\neq c^3$$".encode('utf-8')).digest(): + (0, 0.0, fermat)}) + self.assertEqual(math_cache_actual, math_cache_expected)