With all the aliases the autodetected number was quite off.
assert len(output) == offset
return output
-search_data_header_struct = struct.Struct('<3sBI')
+search_data_header_struct = struct.Struct('<3sBHI')
-def serialize_search_data(trie: Trie, map: ResultMap, merge_subtrees=True, merge_prefixes=True) -> bytearray:
+def serialize_search_data(trie: Trie, map: ResultMap, symbol_count, merge_subtrees=True, merge_prefixes=True) -> bytearray:
serialized_trie = trie.serialize(merge_subtrees=merge_subtrees)
serialized_map = map.serialize(merge_prefixes=merge_prefixes)
- # magic header, version, offset of result map
- return search_data_header_struct.pack(b'MCS', 0, len(serialized_trie) + 8) + serialized_trie + serialized_map
+ # magic header, version, symbol count, offset of result map
+ return search_data_header_struct.pack(b'MCS', 0, symbol_count, len(serialized_trie) + 10) + serialized_trie + serialized_map
xref_id_rx = re.compile(r"""(.*)_1(_[a-z-]+[0-9]+)$""")
slugify_nonalnum_rx = re.compile(r"""[^\w\s-]""")
keyword_index = map.add(title, '', alias=index, suffix_length=suffix_length)
trie.insert(search.lower(), keyword_index)
- return serialize_search_data(trie, map, merge_subtrees=merge_subtrees, merge_prefixes=merge_prefixes)
+ return serialize_search_data(trie, map, len(state.search), merge_subtrees=merge_subtrees, merge_prefixes=merge_prefixes)
def base85encode_search_data(data: bytearray) -> bytearray:
return (b"/* Generated by http://mcss.mosra.cz/doxygen/. Do not edit. */\n" +
let view = new DataView(buffer);
/* The file is too short to contain at least the headers */
- if(view.byteLength < 18) {
+ if(view.byteLength < 20) {
console.error("Search data too short");
return false;
}
}
/* Separate the data into the trie and the result map */
- let mapOffset = view.getUint32(4, true);
- this.trie = new DataView(buffer, 8, mapOffset - 8);
+ let mapOffset = view.getUint32(6, true);
+ this.trie = new DataView(buffer, 10, mapOffset - 10);
this.map = new DataView(buffer, mapOffset);
/* Set initial properties */
this.dataSize = buffer.byteLength;
- this.symbolCount = (this.map.getUint32(0, true) & 0x00ffffff)/4 - 1;
+ this.symbolCount = view.getUint16(4, true);
this.maxResults = maxResults ? maxResults : 100;
this.searchString = '';
this.searchStack = [this.trie.getUint32(0, true)];
-O+!-viU9xsZ~*`S0RRC2009I504V?g2mk;m009mF0B!&Q6aWBe0RRI400AHX04V?gBme*?00Alh0B!&QFaQ8)00A}t0BryPJOBVX0RaL4LI8j|00Bq<0CE5UPyhgL00CA20CWHWTmS%L00CkE0A&FH1poj6ZU6u&00D9U04M+fcmM!y00Djg0BHaLga80-00D{s06GBy1OSi#fI0vHmH+@{00Eu=0A~OJqyPYJ00F810AT<F8UO%oXaE3qumAvZ00FiD06GBy006`QfI0vH$^Zap00Ghf0CWQY0RRI41poj6-T(k800HIz04M+f>;M3600P(m0Aca~0BHdL2LJ#80s#PJ009O80BisO3;_UV009yK0B!*Q2mk;99svL;00I~R0AV5l04M+fC;<R;00A-q0Am0GssI3CH~|1<0RRR700Bq=0A&CHOaTCA00B|~0AT<DSOEZV00CYB0Ac_EWB~wm00j5|0BvXh0CqkB0CH*p0COk+01!U_0BKhM04jR`0BMl`04TEn0By+t0Knw{008g+015s8002#4bZ7u>VQpn|aA9L*O<{CsE@*UZYybcf2s%1#X>KTKZgealX>N2W03&T_ZU6uPIyzQmV{~tF0Ap-nb8}5$bZB2OUolo?V{~tFE@*UZYyton20A)zX>KSfAY*TCb94YBZE0=*0025VQekdqWdLJrVRLg$VRUF;F<&uKVQyz-E@*UZYy<!o20A)zX>KSfAY*TCb94YBZE0>$VP|C^!n+_-bz*Q~XJr6$bz*Q~XJsyEbZu+|08(XRbYX61Y-I!h2LJ#9IyzEiV{|Af
\ No newline at end of file
+O+!-v2LOx#003|S000310RR921ONaj009U904M+f4gdgd009&L0BHdL0{{R4AOHX<00ATb04M+fDgXd(00A%n0BHaLHUI!^00BGz06GBy0suk)fI0vHNB{tG00B?{0B-;RRsaBW00CS80Am0FVgLYT0RRO600C|Q04V?gasU7*00DRa0B!&QegFVz00D#m0BryPiU0sQ0RaR6kN|)>00EW&0A&CHo&W%600E=`0B!&QssI3C00SBT0BvXh0Cund0CE5Uwg3P+0RaF2!~lRg00GJX0B8UK(f|N-0{{U40{{g800G_r04V?g<^TXF00Ha(0B!&R*Z=@w@&Ev70RRU8009C40A&CH1_1zU009gE0A~OJ5&-~i0RRX900ABW04V?h7y$rbA^`v>00Afg0CWHWG64W%00OE20AV-*0A&FH1^@s7NC5z400B$^0A~OJQUL&A00CG50B`^SUI74N00CqH0CfNa_y7QHXaE3qJ^=u7Y5@RqC;$KuKL7w}R{#JidjJ4wkpKWFvj6~X$p8Sr<p2Nx@Bjb_{r~^~O<{Cs0B&JzWpi+0V`WWYbZ9PUbZu+^01^l~I&EogC~0nVEFfuabSVHMZE0=*0025VR%K&!Z*l-*Y+-YAO<{CsUol@XR%K&!Z*neZbZu+`02l^3I&EogC@COgZ*FsR03&T_ZU6uPIyzEeZf9ixV{Bn_b4_7%XkRg3F;Zb}XJsyEbZu+|02c;2I&EogC@COgZ*FsR03&T_Zct%oWgx=4AX9Z>aA9X<0CRO>aA9X<E@*UZYy<#OWn*+<Zf9&|1ONvB00KHXQe|UwC@BB{
\ No newline at end of file
-MCS\ 1
\ No newline at end of file
+MCS\ 1
\ No newline at end of file
with open(basedir/'short.bin', 'wb') as f:
f.write(b'')
with open(basedir/'wrong-magic.bin', 'wb') as f:
- f.write(b'MOS\0 ')
+ f.write(b'MOS\0 ')
with open(basedir/'wrong-version.bin', 'wb') as f:
- f.write(b'MCS\1 ')
+ f.write(b'MCS\1 ')
with open(basedir/'empty.bin', 'wb') as f:
- f.write(serialize_search_data(Trie(), ResultMap()))
+ f.write(serialize_search_data(Trie(), ResultMap(), 0))
trie = Trie()
map = ResultMap()
trie.insert("rect()", map.add("Rectangle::Rect()", "", suffix_length=2, alias=range_index))
with open(basedir/'searchdata.bin', 'wb') as f:
- f.write(serialize_search_data(trie, map))
+ f.write(serialize_search_data(trie, map, 7))
with open(basedir/'searchdata.b85', 'wb') as f:
- f.write(base64.b85encode(serialize_search_data(trie, map), True))
+ f.write(base64.b85encode(serialize_search_data(trie, map, 7), True))
trie = Trie()
map = ResultMap()
trie.insert("hárá", map.add("Hárá", "#b", flags=ResultFlag.PAGE))
with open(basedir/'unicode.bin', 'wb') as f:
- f.write(serialize_search_data(trie, map))
+ f.write(serialize_search_data(trie, map, 2))
/* Verify that base85-decoded file is equivalent to the binary */
{
let binary = fs.readFileSync(path.join(__dirname, "js-test-data/searchdata.bin"));
- assert.equal(binary.byteLength, 648);
+ assert.equal(binary.byteLength, 650);
let b85 = fs.readFileSync(path.join(__dirname, "js-test-data/searchdata.b85"), {encoding: 'utf-8'});
assert.deepEqual(new DataView(binary.buffer.slice(binary.byteOffset, binary.byteOffset + binary.byteLength)), new DataView(Search.base85decode(b85), 0, binary.byteLength));
}
{
let buffer = fs.readFileSync(path.join(__dirname, "js-test-data/searchdata.bin"));
assert.ok(Search.init(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength)));
- assert.equal(Search.dataSize, 648);
- assert.equal(Search.symbolCount, 9);
+ assert.equal(Search.dataSize, 650);
+ assert.equal(Search.symbolCount, 7);
assert.equal(Search.maxResults, 100);
/* Blow up */
{
let buffer = fs.readFileSync(path.join(__dirname, "js-test-data/searchdata.bin"));
assert.ok(Search.init(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength), 3));
- assert.equal(Search.dataSize, 648);
- assert.equal(Search.symbolCount, 9);
+ assert.equal(Search.dataSize, 650);
+ assert.equal(Search.symbolCount, 7);
assert.equal(Search.maxResults, 3);
assert.deepEqual(Search.search('m'), [
{ name: 'Math',
{
let b85 = fs.readFileSync(path.join(__dirname, "js-test-data/searchdata.b85"), {encoding: 'utf-8'});
assert.ok(Search.load(b85));
- assert.equal(Search.dataSize, 648); /* some padding on the end, that's okay */
- assert.equal(Search.symbolCount, 9);
+ assert.equal(Search.dataSize, 652); /* some padding on the end, that's okay */
+ assert.equal(Search.symbolCount, 7);
assert.equal(Search.maxResults, 100);
assert.deepEqual(Search.search('min'), [
{ name: 'Math::min(int, int)',
{
let buffer = fs.readFileSync(path.join(__dirname, "js-test-data/unicode.bin"));
assert.ok(Search.init(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength), 3));
- assert.equal(Search.dataSize, 122);
+ assert.equal(Search.dataSize, 124);
assert.equal(Search.symbolCount, 2);
assert.deepEqual(Search.search('h'), [
{ name: Search.toUtf8('Hárá'),
return out
def pretty_print(serialized: bytes, show_merged=False, show_lookahead_barriers=True, colors=False):
- magic, version, map_offset = search_data_header_struct.unpack_from(serialized)
+ magic, version, symbol_count, map_offset = search_data_header_struct.unpack_from(serialized)
assert magic == b'MCS'
assert version == 0
- assert not search_data_header_struct.size % 4
pretty_trie, stats = pretty_print_trie(serialized[search_data_header_struct.size:map_offset], show_merged=show_merged, show_lookahead_barriers=show_lookahead_barriers, colors=colors)
pretty_map = pretty_print_map(serialized[map_offset:], colors=colors)
- return pretty_trie + '\n' + pretty_map, stats
+ return '{} symbols\n'.format(symbol_count) + pretty_trie + '\n' + pretty_map, stats
class TrieSerialization(unittest.TestCase):
def __init__(self, *args, **kwargs):
trie.insert("math::range", index)
trie.insert("range", index)
- serialized = serialize_search_data(trie, map)
+ serialized = serialize_search_data(trie, map, 3)
self.compare(serialized, """
+3 symbols
math [0]
| ::vector [1]
| range [2]
1: ::Vector [prefix=0[:0], type=CLASS] -> classMath_1_1Vector.html
2: ::Range [prefix=0[:0], type=CLASS] -> classMath_1_1Range.html
""")
- self.assertEqual(len(serialized), 239)
+ self.assertEqual(len(serialized), 241)
class Search(IntegrationTestCase):
def __init__(self, *args, **kwargs):
serialized = f.read()
search_data_pretty = pretty_print(serialized)[0]
#print(search_data_pretty)
- self.assertEqual(len(serialized), 4570)
+ self.assertEqual(len(serialized), 4572)
self.assertEqual(search_data_pretty, """
+36 symbols
deprecated_macro [0]
|| | ($
|| | ) [1]
~cjwatson / blog.git / commitdiff