Commit | Line | Data |
---|---|---|
61507e3c RK |
1 | #! /usr/bin/perl -w |
2 | # | |
e5a5a138 RK |
3 | # This file is part of DisOrder. |
4 | # Copyright (C) 2007 Richard Kettlewell | |
5 | # | |
6 | # This program is free software; you can redistribute it and/or modify | |
7 | # it under the terms of the GNU General Public License as published by | |
8 | # the Free Software Foundation; either version 2 of the License, or | |
9 | # (at your option) any later version. | |
10 | # | |
11 | # This program is distributed in the hope that it will be useful, but | |
12 | # WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | # General Public License for more details. | |
15 | # | |
16 | # You should have received a copy of the GNU General Public License | |
17 | # along with this program; if not, write to the Free Software | |
18 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 | |
19 | # USA | |
20 | # | |
21 | # | |
22 | # Generate Unicode support tables | |
23 | # | |
24 | # This script will download data from unicode.org if the required files | |
25 | # aren't in the current directory. | |
26 | # | |
27 | # After modifying this script you should run: | |
28 | # make -C lib rebuild-unicode check | |
29 | # | |
30 | # Things not supported yet: | |
31 | # - SpecialCasing.txt data for case mapping | |
32 | # - Title case offsets | |
33 | # - Some kind of hinting for composition | |
34 | # - Word boundary support | |
35 | # - ... | |
36 | # | |
37 | # NB the generated files DO NOT offer a stable ABI and so are not immediately | |
38 | # suitable for use in a general-purpose library. Things that would need to | |
39 | # be done: | |
40 | # - Hide unidata.h from applications; it will never be ABI- or even API-stable. | |
41 | # - Stablized General_Category values | |
42 | # - Extend the unicode.h API to general utility rather than just what | |
43 | # DisOrder needs. | |
44 | # - ... | |
45 | # | |
61507e3c | 46 | use strict; |
35b651f0 | 47 | use File::Basename; |
61507e3c RK |
48 | |
49 | sub out { | |
50 | print @_ or die "$!\n"; | |
51 | } | |
52 | ||
53 | sub key { | |
54 | my $d = shift; | |
55 | local $_; | |
56 | ||
57 | return join("-", map($d->{$_}, sort keys %$d)); | |
58 | } | |
59 | ||
e5a5a138 RK |
60 | # Size of a subtable |
61 | # | |
62 | # This can be varied to trade off the number of subtables against their size. | |
63 | our $modulus = 128; | |
64 | ||
1a05e381 RK |
65 | # Where to break the table. There is a huge empty section of the Unicode |
66 | # code space and we deal with this by simply leaving it out of the table. | |
67 | # This complicates the lookup function a little but should not affect | |
68 | # performance in the cases we care about. | |
69 | our $break_start = 0x30000; | |
70 | our $break_end = 0xE0000; | |
71 | ||
72 | # Similarly we simply omit the very top of the table and sort it out in the | |
73 | # lookup function. | |
74 | our $break_top = 0xE0200; | |
75 | ||
61507e3c RK |
76 | my %cats = (); # known general categories |
77 | my %data = (); # mapping of codepoints to information | |
61507e3c | 78 | my $max = 0; # maximum codepoint |
e5a5a138 RK |
79 | my $maxccc = 0; # maximum combining class |
80 | my $maxud = 0; | |
81 | my $minud = 0; # max/min upper case offset | |
82 | my $maxld = 0; | |
83 | my $minld = 0; # max/min lower case offset | |
84 | ||
85 | # Make sure we have our desired input files. We explicitly specify a | |
86 | # Unicode standard version to make sure that a given version of DisOrder | |
87 | # supports a given version of Unicode. | |
0b7052da | 88 | sub input { |
35b651f0 RK |
89 | my $path = shift; |
90 | my $lpath = basename($path); | |
91 | if(!-e $lpath) { | |
92 | system("wget http://www.unicode.org/Public/5.0.0/ucd/$path"); | |
93 | chmod(0444, $lpath) or die "$lpath: $!\n"; | |
e5a5a138 | 94 | } |
0b7052da | 95 | open(STDIN, "<$lpath") or die "$lpath: $!\n"; |
bcf9ed7f | 96 | print STDERR "Reading $lpath...\n"; |
e5a5a138 | 97 | } |
61507e3c | 98 | |
e5a5a138 RK |
99 | |
100 | # Read the main data file | |
0b7052da | 101 | input("UnicodeData.txt"); |
1a05e381 | 102 | my ($start, $end); |
61507e3c RK |
103 | while(<>) { |
104 | my @f = split(/;/, $_); | |
105 | my $c = hex($f[0]); # codepoint | |
61507e3c | 106 | my $name = $f[1]; |
1a05e381 RK |
107 | die "$f[0] $name is in the break\n" |
108 | if $c >= $break_start && $c < $break_end; | |
e5a5a138 | 109 | my $gc = $f[2]; # General_Category |
bcf9ed7f | 110 | # Variuos GCs we don't expect to see in UnicodeData.txt |
0a4af692 | 111 | $cats{$gc} = 1; # always record all GCs |
1a05e381 RK |
112 | if($name =~ /first>/i) { |
113 | $start = $c; | |
114 | next; | |
115 | } elsif($name =~ /last>/i) { | |
116 | $end = $c; | |
117 | } else { | |
118 | $start = $end = $c; | |
119 | } | |
bcf9ed7f | 120 | die "unexpected Cn" if $gc eq 'Cn'; |
e5a5a138 RK |
121 | my $ccc = $f[3]; # Canonical_Combining_Class |
122 | my $dm = $f[5]; # Decomposition_Type + Decomposition_Mapping | |
123 | my $sum = hex($f[12]) || $c; # Simple_Uppercase_Mapping | |
124 | my $slm = hex($f[13]) || $c; # Simple_Lowercase_Mapping | |
61507e3c RK |
125 | # recalculate the upper/lower case mappings as offsets |
126 | my $ud = $sum - $c; | |
127 | my $ld = $slm - $c; | |
e5a5a138 RK |
128 | # update bounds on various values |
129 | $maxccc = $ccc if $ccc > $maxccc; # assumed never to be -ve | |
130 | $minud = $ud if $ud < $minud; | |
131 | $maxud = $ud if $ud > $maxud; | |
132 | $minld = $ld if $ld < $minld; | |
133 | $maxld = $ld if $ld > $maxld; | |
1a05e381 | 134 | my $d = { |
61507e3c RK |
135 | "gc" => $gc, |
136 | "ccc" => $ccc, | |
137 | "ud" => $ud, | |
e5a5a138 | 138 | "ld" => $ld, |
1a05e381 | 139 | }; |
e5a5a138 RK |
140 | if($dm ne '') { |
141 | if($dm !~ /</) { | |
142 | # This is a canonical decomposition | |
1a05e381 RK |
143 | $d->{canon} = $dm; |
144 | $d->{compat} = $dm; | |
e5a5a138 RK |
145 | } else { |
146 | # This is only a compatibility decomposition | |
147 | $dm =~ s/^<.*>\s*//; | |
1a05e381 | 148 | $d->{compat} = $dm; |
e5a5a138 RK |
149 | } |
150 | } | |
1a05e381 RK |
151 | if($start != $end) { |
152 | printf STDERR "> range %04X-%04X is %s\n", $start, $end, $d->{gc}; | |
153 | } | |
154 | for($c = $start; $c <= $end; ++$c) { | |
155 | $data{$c} = $d; | |
156 | } | |
61507e3c | 157 | $cats{$gc} = 1; |
1a05e381 | 158 | $max = $end if $end > $max; |
61507e3c RK |
159 | } |
160 | ||
0b7052da RK |
161 | sub read_prop_with_ranges { |
162 | my $path = shift; | |
163 | my $propkey = shift; | |
164 | input($path); | |
165 | while(<>) { | |
166 | chomp; | |
167 | s/\s*\#.*//; | |
168 | next if $_ eq ''; | |
169 | my ($range, $propval) = split(/\s*;\s*/, $_); | |
170 | if($range =~ /(.*)\.\.(.*)/) { | |
171 | for my $c (hex($1) .. hex($2)) { | |
172 | if(exists $data{$c}) { | |
173 | $data{$c}->{$propkey} = $propval; | |
174 | } | |
175 | } | |
176 | } else { | |
177 | my $c = hex($range); | |
35b651f0 | 178 | if(exists $data{$c}) { |
0b7052da | 179 | $data{$c}->{$propkey} = $propval; |
35b651f0 RK |
180 | } |
181 | } | |
0b7052da RK |
182 | } |
183 | } | |
184 | ||
349b7b74 | 185 | # Grapheme_Break etc |
0b7052da RK |
186 | # NB we do this BEFORE filling in blanks so that the Hangul characters |
187 | # don't get filled in; we can compute their properties mechanically. | |
188 | read_prop_with_ranges("auxiliary/GraphemeBreakProperty.txt", "gbreak"); | |
0b7052da | 189 | read_prop_with_ranges("auxiliary/WordBreakProperty.txt", "wbreak"); |
349b7b74 | 190 | read_prop_with_ranges("auxiliary/SentenceBreakProperty.txt", "sbreak"); |
0b7052da | 191 | |
349b7b74 RK |
192 | # Compute the full list and fill in the Extend category properly |
193 | my %gbreak = (); | |
194 | my %wbreak = (); | |
195 | my %sbreak = (); | |
0b7052da | 196 | for my $c (keys %data) { |
349b7b74 RK |
197 | if(!exists $data{$c}->{gbreak}) { |
198 | $data{$c}->{gbreak} = 'Other'; | |
199 | } | |
200 | $gbreak{$data{$c}->{gbreak}} = 1; | |
201 | ||
0b7052da | 202 | if(!exists $data{$c}->{wbreak}) { |
349b7b74 | 203 | if($data{$c}->{gbreak} eq 'Extend') { |
0b7052da RK |
204 | $data{$c}->{wbreak} = 'Extend'; |
205 | } else { | |
206 | $data{$c}->{wbreak} = 'Other'; | |
35b651f0 RK |
207 | } |
208 | } | |
349b7b74 RK |
209 | $wbreak{$data{$c}->{wbreak}} = 1; |
210 | ||
211 | if(!exists $data{$c}->{sbreak}) { | |
212 | if($data{$c}->{gbreak} eq 'Extend') { | |
213 | $data{$c}->{sbreak} = 'Extend'; | |
214 | } else { | |
215 | $data{$c}->{sbreak} = 'Other'; | |
216 | } | |
217 | } | |
218 | $sbreak{$data{$c}->{sbreak}} = 1; | |
35b651f0 RK |
219 | } |
220 | ||
e5a5a138 RK |
221 | # Round up the maximum value to a whole number of subtables |
222 | $max += ($modulus - 1) - ($max % $modulus); | |
61507e3c | 223 | |
bcf9ed7f RK |
224 | # Private use characters |
225 | # We only fill in values below $max, utf32__unidata() | |
226 | my $Co = { | |
227 | "gc" => "Co", | |
228 | "ccc" => 0, | |
229 | "ud" => 0, | |
230 | "ld" => 0 | |
231 | }; | |
232 | for(my $c = 0xE000; $c <= 0xF8FF && $c <= $max; ++$c) { | |
233 | $data{$c} = $Co; | |
234 | } | |
235 | for(my $c = 0xF0000; $c <= 0xFFFFD && $c <= $max; ++$c) { | |
236 | $data{$c} = $Co; | |
237 | } | |
238 | for(my $c = 0x100000; $c <= 0x10FFFD && $c <= $max; ++$c) { | |
239 | $data{$c} = $Co; | |
240 | } | |
241 | ||
242 | # Anything left is not assigned | |
243 | my $Cn = { | |
244 | "gc" => "Cn", # not assigned | |
245 | "ccc" => 0, | |
246 | "ud" => 0, | |
247 | "ld" => 0 | |
248 | }; | |
61507e3c RK |
249 | for(my $c = 0; $c <= $max; ++$c) { |
250 | if(!exists $data{$c}) { | |
bcf9ed7f RK |
251 | $data{$c} = $Cn; |
252 | } | |
253 | if(!exists $data{$c}->{wbreak}) { | |
254 | $data{$c}->{wbreak} = 'Other'; | |
255 | } | |
256 | if(!exists $data{$c}->{gbreak}) { | |
257 | $data{$c}->{gbreak} = 'Other'; | |
258 | } | |
259 | if(!exists $data{$c}->{sbreak}) { | |
260 | $data{$c}->{sbreak} = 'Other'; | |
61507e3c RK |
261 | } |
262 | } | |
263 | $cats{'Cn'} = 1; | |
264 | ||
e5a5a138 | 265 | # Read the casefolding data too |
0b7052da | 266 | input("CaseFolding.txt"); |
e5a5a138 RK |
267 | while(<>) { |
268 | chomp; | |
269 | next if /^\#/ or $_ eq ''; | |
270 | my @f = split(/\s*;\s*/, $_); | |
271 | # Full case folding means use status C and F. | |
272 | # We discard status T, Turkish users may wish to change this. | |
273 | if($f[1] eq 'C' or $f[1] eq 'F') { | |
274 | my $c = hex($f[0]); | |
275 | $data{$c}->{casefold} = $f[2]; | |
276 | # We are particularly interest in combining characters that | |
277 | # case-fold to non-combining characters, or characters that | |
278 | # case-fold to sequences with combining characters in non-initial | |
279 | # positions, as these required decomposiiton before case-folding | |
280 | my @d = map(hex($_), split(/\s+/, $data{$c}->{casefold})); | |
281 | if($data{$c}->{ccc} != 0) { | |
282 | # This is a combining character | |
283 | if($data{$d[0]}->{ccc} == 0) { | |
284 | # The first character of its case-folded form is NOT | |
285 | # a combining character. The field name is the example | |
286 | # explicitly mentioned in the spec. | |
287 | $data{$c}->{ypogegrammeni} = 1; | |
288 | } | |
289 | } else { | |
290 | # This is a non-combining character; inspect the non-initial | |
291 | # code points of the case-folded sequence | |
292 | shift(@d); | |
293 | if(grep($data{$_}->{ccc} != 0, @d)) { | |
294 | # Some non-initial code point in the case-folded for is NOT a | |
295 | # a combining character. | |
296 | $data{$c}->{ypogegrammeni} = 1; | |
297 | } | |
298 | } | |
299 | } | |
300 | } | |
301 | ||
302 | # Generate the header file | |
bcf9ed7f | 303 | print STDERR "Generating unidata.h...\n"; |
61507e3c RK |
304 | open(STDOUT, ">unidata.h") or die "unidata.h: $!\n"; |
305 | ||
e5a5a138 RK |
306 | out("/* Automatically generated file, see scripts/make-unidata */\n", |
307 | "#ifndef UNIDATA_H\n", | |
61507e3c RK |
308 | "#define UNIDATA_H\n"); |
309 | ||
e5a5a138 | 310 | # TODO choose stable values for General_Category |
14523635 | 311 | out("enum unicode_General_Category {\n", |
61507e3c | 312 | join(",\n", |
14523635 | 313 | map(" unicode_General_Category_$_", sort keys %cats)), "\n};\n"); |
e5a5a138 | 314 | |
349b7b74 RK |
315 | out("enum unicode_Grapheme_Break {\n", |
316 | join(",\n", | |
317 | map(" unicode_Grapheme_Break_$_", sort keys %gbreak)), | |
318 | "\n};\n"); | |
319 | out("extern const char *const unicode_Grapheme_Break_names[];\n"); | |
320 | ||
0b7052da RK |
321 | out("enum unicode_Word_Break {\n", |
322 | join(",\n", | |
349b7b74 | 323 | map(" unicode_Word_Break_$_", sort keys %wbreak)), |
0b7052da | 324 | "\n};\n"); |
bb48024f | 325 | out("extern const char *const unicode_Word_Break_names[];\n"); |
0b7052da | 326 | |
349b7b74 RK |
327 | out("enum unicode_Sentence_Break {\n", |
328 | join(",\n", | |
329 | map(" unicode_Sentence_Break_$_", sort keys %sbreak)), | |
330 | "\n};\n"); | |
331 | out("extern const char *const unicode_Sentence_Break_names[];\n"); | |
332 | ||
e5a5a138 | 333 | out("enum unicode_flags {\n", |
0b7052da | 334 | " unicode_normalize_before_casefold = 1\n", |
e5a5a138 RK |
335 | "};\n", |
336 | "\n"); | |
337 | ||
338 | # Choose the narrowest type that will fit the required values | |
339 | sub choosetype { | |
340 | my ($min, $max) = @_; | |
341 | if($min >= 0) { | |
342 | return "char" if $max <= 127; | |
343 | return "unsigned char" if $max <= 255; | |
344 | return "int16_t" if $max < 32767; | |
345 | return "uint16_t" if $max < 65535; | |
346 | return "int32_t"; | |
347 | } else { | |
348 | return "char" if $min >= -127 && $max <= 127; | |
349 | return "int16_t" if $min >= -32767 && $max <= 32767; | |
350 | return "int32_t"; | |
351 | } | |
352 | } | |
353 | ||
61507e3c | 354 | out("struct unidata {\n", |
e5a5a138 RK |
355 | " const uint32_t *compat;\n", |
356 | " const uint32_t *canon;\n", | |
357 | " const uint32_t *casefold;\n", | |
1a05e381 RK |
358 | # " ".choosetype($minud, $maxud)." upper_offset;\n", |
359 | # " ".choosetype($minld, $maxld)." lower_offset;\n", | |
e5a5a138 | 360 | " ".choosetype(0, $maxccc)." ccc;\n", |
14523635 | 361 | " char general_category;\n", |
e5a5a138 | 362 | " uint8_t flags;\n", |
349b7b74 | 363 | " char grapheme_break;\n", |
0b7052da | 364 | " char word_break;\n", |
349b7b74 | 365 | " char sentence_break;\n", |
61507e3c | 366 | "};\n"); |
e5a5a138 RK |
367 | # compat, canon and casefold do have have non-BMP characters, so we |
368 | # can't use a simple 16-bit table. We could use UTF-8 or UTF-16 | |
369 | # though, saving a bit of space (probably not that much...) at the | |
370 | # cost of marginally reduced performance and additional complexity | |
61507e3c RK |
371 | |
372 | out("extern const struct unidata *const unidata[];\n"); | |
373 | ||
374 | out("#define UNICODE_NCHARS ", ($max + 1), "\n"); | |
e5a5a138 | 375 | out("#define UNICODE_MODULUS $modulus\n"); |
1a05e381 RK |
376 | out("#define UNICODE_BREAK_START $break_start\n"); |
377 | out("#define UNICODE_BREAK_END $break_end\n"); | |
378 | out("#define UNICODE_BREAK_TOP $break_top\n"); | |
61507e3c RK |
379 | |
380 | out("#endif\n"); | |
381 | ||
382 | close STDOUT or die "unidata.h: $!\n"; | |
383 | ||
bcf9ed7f | 384 | print STDERR "Generating unidata.c...\n"; |
61507e3c RK |
385 | open(STDOUT, ">unidata.c") or die "unidata.c: $!\n"; |
386 | ||
e5a5a138 RK |
387 | out("/* Automatically generated file, see scripts/make-unidata */\n", |
388 | "#include <config.h>\n", | |
389 | "#include \"types.h\"\n", | |
390 | "#include \"unidata.h\"\n"); | |
391 | ||
349b7b74 | 392 | # Short aliases to keep .c file small |
e5a5a138 | 393 | |
14523635 RK |
394 | out(map(sprintf("#define %s unicode_General_Category_%s\n", $_, $_), |
395 | sort keys %cats)); | |
396 | out(map(sprintf("#define GB%s unicode_Grapheme_Break_%s\n", $_, $_), | |
397 | sort keys %gbreak)); | |
398 | out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_), | |
399 | sort keys %wbreak)); | |
400 | out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_), | |
401 | sort keys %sbreak)); | |
e5a5a138 | 402 | |
349b7b74 RK |
403 | # Names for *_Break properties |
404 | out("const char *const unicode_Grapheme_Break_names[] = {\n", | |
405 | join(",\n", | |
406 | map(" \"$_\"", sort keys %gbreak)), | |
407 | "\n};\n"); | |
bb48024f RK |
408 | out("const char *const unicode_Word_Break_names[] = {\n", |
409 | join(",\n", | |
349b7b74 RK |
410 | map(" \"$_\"", sort keys %wbreak)), |
411 | "\n};\n"); | |
412 | out("const char *const unicode_Sentence_Break_names[] = {\n", | |
413 | join(",\n", | |
414 | map(" \"$_\"", sort keys %sbreak)), | |
bb48024f RK |
415 | "\n};\n"); |
416 | ||
e5a5a138 RK |
417 | # Generate the decomposition mapping tables. We look out for duplicates |
418 | # in order to save space and report this as decompsaved at the end. In | |
419 | # Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes. | |
420 | my $decompnum = 0; | |
421 | my %decompnums = (); | |
422 | my $decompsaved = 0; | |
423 | out("static const uint32_t "); | |
424 | for(my $c = 0; $c <= $max; ++$c) { | |
425 | # If canon is set then compat will be too and will be identical. | |
426 | # If compat is set the canon might be clear. So we use the | |
427 | # compat version and fix up the symbols after. | |
bcf9ed7f | 428 | if(exists $data{$c} && exists $data{$c}->{compat}) { |
e5a5a138 RK |
429 | my $s = join(",", |
430 | (map(hex($_), split(/\s+/, $data{$c}->{compat})), 0)); | |
431 | if(!exists $decompnums{$s}) { | |
432 | out(",\n") if $decompnum != 0; | |
433 | out("cd$decompnum\[]={$s}"); | |
434 | $decompnums{$s} = $decompnum++; | |
435 | } else { | |
436 | ++$decompsaved; | |
437 | } | |
438 | $data{$c}->{compatsym} = "cd$decompnums{$s}"; | |
439 | if(exists $data{$c}->{canon}) { | |
440 | $data{$c}->{canonsym} = "cd$decompnums{$s}"; | |
441 | } | |
442 | } | |
443 | } | |
444 | out(";\n"); | |
445 | ||
446 | # ...and the case folding table. Again we compress equal entries to save | |
447 | # space. In Unicode 5.0.0 this saves 51 entries or at least 408 bytes. | |
448 | # This doesns't seem as worthwhile as the decomposition mapping saving above. | |
449 | my $cfnum = 0; | |
450 | my %cfnums = (); | |
451 | my $cfsaved = 0; | |
452 | out("static const uint32_t "); | |
453 | for(my $c = 0; $c <= $max; ++$c) { | |
bcf9ed7f | 454 | if(exists $data{$c} && exists $data{$c}->{casefold}) { |
e5a5a138 RK |
455 | my $s = join(",", |
456 | (map(hex($_), split(/\s+/, $data{$c}->{casefold})), 0)); | |
457 | if(!exists $cfnums{$s}) { | |
458 | out(",\n") if $cfnum != 0; | |
459 | out("cf$cfnum\[]={$s}"); | |
460 | $cfnums{$s} = $cfnum++; | |
461 | } else { | |
462 | ++$cfsaved; | |
463 | } | |
464 | $data{$c}->{cfsym} = "cf$cfnums{$s}"; | |
465 | } | |
466 | } | |
467 | out(";\n"); | |
468 | ||
469 | # Visit all the $modulus-character blocks in turn and generate the | |
470 | # required subtables. As above we spot duplicates to save space. In | |
471 | # Unicode 5.0.0 with $modulus=128 and current table data this saves | |
472 | # 1372 subtables or at least three and a half megabytes on 32-bit | |
473 | # platforms. | |
61507e3c | 474 | |
61507e3c RK |
475 | my %subtable = (); # base->subtable number |
476 | my %subtableno = (); # subtable number -> content | |
477 | my $subtablecounter = 0; # counter for subtable numbers | |
e5a5a138 RK |
478 | my $subtablessaved = 0; # number of tables saved |
479 | for(my $base = 0; $base <= $max; $base += $modulus) { | |
1a05e381 RK |
480 | next if $base >= $break_start && $base < $break_end; |
481 | next if $base >= $break_top; | |
61507e3c | 482 | my @t; |
e5a5a138 | 483 | for(my $c = $base; $c < $base + $modulus; ++$c) { |
61507e3c | 484 | my $d = $data{$c}; |
e5a5a138 RK |
485 | my $canonsym = ($data{$c}->{canonsym} or "0"); |
486 | my $compatsym = ($data{$c}->{compatsym} or "0"); | |
487 | my $cfsym = ($data{$c}->{cfsym} or "0"); | |
35b651f0 RK |
488 | my @flags = (); |
489 | if($data{$c}->{ypogegrammeni}) { | |
490 | push(@flags, "unicode_normalize_before_casefold"); | |
491 | } | |
35b651f0 | 492 | my $flags = @flags ? join("|", @flags) : 0; |
e5a5a138 RK |
493 | push(@t, "{". |
494 | join(",", | |
495 | $compatsym, | |
496 | $canonsym, | |
497 | $cfsym, | |
1a05e381 RK |
498 | # $d->{ud}, |
499 | # $d->{ld}, | |
0b7052da RK |
500 | $d->{ccc}, |
501 | $d->{gc}, | |
e5a5a138 | 502 | $flags, |
349b7b74 RK |
503 | "GB$d->{gbreak}", |
504 | "WB$d->{wbreak}", | |
505 | "SB$d->{sbreak}", | |
e5a5a138 | 506 | )."}"); |
61507e3c RK |
507 | } |
508 | my $t = join(",\n", @t); | |
509 | if(!exists $subtable{$t}) { | |
e5a5a138 | 510 | out("static const struct unidata st$subtablecounter\[] = {\n", |
61507e3c RK |
511 | "$t\n", |
512 | "};\n"); | |
513 | $subtable{$t} = $subtablecounter++; | |
e5a5a138 RK |
514 | } else { |
515 | ++$subtablessaved; | |
61507e3c RK |
516 | } |
517 | $subtableno{$base} = $subtable{$t}; | |
518 | } | |
519 | ||
bcf9ed7f | 520 | out("const struct unidata *const unidata[]={\n"); |
e5a5a138 | 521 | for(my $base = 0; $base <= $max; $base += $modulus) { |
1a05e381 RK |
522 | next if $base >= $break_start && $base < $break_end; |
523 | next if $base >= $break_top; | |
0a4af692 | 524 | #out("st$subtableno{$base} /* ".sprintf("%04x", $base)." */,\n"); |
e5a5a138 | 525 | out("st$subtableno{$base},\n"); |
61507e3c RK |
526 | } |
527 | out("};\n"); | |
528 | ||
529 | close STDOUT or die "unidata.c: $!\n"; | |
530 | ||
bcf9ed7f RK |
531 | printf STDERR "max=%04X\n", $max; |
532 | print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n"; | |
e5a5a138 | 533 | print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n"; |