Commit | Line | Data |
---|---|---|
61507e3c RK |
1 | #! /usr/bin/perl -w |
2 | # | |
e5a5a138 RK |
3 | # This file is part of DisOrder. |
4 | # Copyright (C) 2007 Richard Kettlewell | |
5 | # | |
e7eb3a27 | 6 | # This program is free software: you can redistribute it and/or modify |
e5a5a138 | 7 | # it under the terms of the GNU General Public License as published by |
e7eb3a27 | 8 | # the Free Software Foundation, either version 3 of the License, or |
e5a5a138 | 9 | # (at your option) any later version. |
e7eb3a27 RK |
10 | # |
11 | # This program is distributed in the hope that it will be useful, | |
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | # GNU General Public License for more details. | |
15 | # | |
e5a5a138 | 16 | # You should have received a copy of the GNU General Public License |
e7eb3a27 | 17 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
e5a5a138 RK |
18 | # |
19 | # | |
20 | # Generate Unicode support tables | |
21 | # | |
22 | # This script will download data from unicode.org if the required files | |
23 | # aren't in the current directory. | |
24 | # | |
25 | # After modifying this script you should run: | |
26 | # make -C lib rebuild-unicode check | |
27 | # | |
28 | # Things not supported yet: | |
29 | # - SpecialCasing.txt data for case mapping | |
30 | # - Title case offsets | |
31 | # - Some kind of hinting for composition | |
e5a5a138 RK |
32 | # - ... |
33 | # | |
34 | # NB the generated files DO NOT offer a stable ABI and so are not immediately | |
35 | # suitable for use in a general-purpose library. Things that would need to | |
36 | # be done: | |
37 | # - Hide unidata.h from applications; it will never be ABI- or even API-stable. | |
38 | # - Stablized General_Category values | |
39 | # - Extend the unicode.h API to general utility rather than just what | |
40 | # DisOrder needs. | |
41 | # - ... | |
42 | # | |
61507e3c | 43 | use strict; |
35b651f0 | 44 | use File::Basename; |
61507e3c RK |
45 | |
46 | sub out { | |
47 | print @_ or die "$!\n"; | |
48 | } | |
49 | ||
50 | sub key { | |
51 | my $d = shift; | |
52 | local $_; | |
53 | ||
54 | return join("-", map($d->{$_}, sort keys %$d)); | |
55 | } | |
56 | ||
e5a5a138 RK |
57 | # Size of a subtable |
58 | # | |
59 | # This can be varied to trade off the number of subtables against their size. | |
c2e01e0a RK |
60 | # 16 gave the smallest results last time I checked (on a Mac with a 32-bit |
61 | # build). | |
62 | our $modulus = 16; | |
63 | ||
64 | if(@ARGV) { | |
65 | $modulus = shift; | |
66 | } | |
e5a5a138 | 67 | |
1a05e381 RK |
68 | # Where to break the table. There is a huge empty section of the Unicode |
69 | # code space and we deal with this by simply leaving it out of the table. | |
70 | # This complicates the lookup function a little but should not affect | |
71 | # performance in the cases we care about. | |
72 | our $break_start = 0x30000; | |
73 | our $break_end = 0xE0000; | |
74 | ||
75 | # Similarly we simply omit the very top of the table and sort it out in the | |
76 | # lookup function. | |
77 | our $break_top = 0xE0200; | |
78 | ||
61507e3c RK |
79 | my %cats = (); # known general categories |
80 | my %data = (); # mapping of codepoints to information | |
61507e3c | 81 | my $max = 0; # maximum codepoint |
e5a5a138 RK |
82 | my $maxccc = 0; # maximum combining class |
83 | my $maxud = 0; | |
84 | my $minud = 0; # max/min upper case offset | |
85 | my $maxld = 0; | |
86 | my $minld = 0; # max/min lower case offset | |
87 | ||
88 | # Make sure we have our desired input files. We explicitly specify a | |
89 | # Unicode standard version to make sure that a given version of DisOrder | |
90 | # supports a given version of Unicode. | |
0b7052da | 91 | sub input { |
35b651f0 RK |
92 | my $path = shift; |
93 | my $lpath = basename($path); | |
94 | if(!-e $lpath) { | |
a2a528f2 | 95 | system("wget http://www.unicode.org/Public/5.1.0/ucd/$path"); |
35b651f0 | 96 | chmod(0444, $lpath) or die "$lpath: $!\n"; |
e5a5a138 | 97 | } |
0b7052da | 98 | open(STDIN, "<$lpath") or die "$lpath: $!\n"; |
bcf9ed7f | 99 | print STDERR "Reading $lpath...\n"; |
e5a5a138 | 100 | } |
61507e3c | 101 | |
e5a5a138 RK |
102 | |
103 | # Read the main data file | |
0b7052da | 104 | input("UnicodeData.txt"); |
1a05e381 | 105 | my ($start, $end); |
16506c9d RK |
106 | my $maxcompat = 0; |
107 | my $maxcanon = 0; | |
108 | my $hangul_syllable_decomps = 0; | |
109 | my $hangul_choseong_decomps = 0; | |
61507e3c RK |
110 | while(<>) { |
111 | my @f = split(/;/, $_); | |
112 | my $c = hex($f[0]); # codepoint | |
61507e3c | 113 | my $name = $f[1]; |
1a05e381 RK |
114 | die "$f[0] $name is in the break\n" |
115 | if $c >= $break_start && $c < $break_end; | |
e5a5a138 | 116 | my $gc = $f[2]; # General_Category |
bcf9ed7f | 117 | # Variuos GCs we don't expect to see in UnicodeData.txt |
0a4af692 | 118 | $cats{$gc} = 1; # always record all GCs |
1a05e381 RK |
119 | if($name =~ /first>/i) { |
120 | $start = $c; | |
121 | next; | |
122 | } elsif($name =~ /last>/i) { | |
123 | $end = $c; | |
124 | } else { | |
125 | $start = $end = $c; | |
126 | } | |
bcf9ed7f | 127 | die "unexpected Cn" if $gc eq 'Cn'; |
e5a5a138 RK |
128 | my $ccc = $f[3]; # Canonical_Combining_Class |
129 | my $dm = $f[5]; # Decomposition_Type + Decomposition_Mapping | |
130 | my $sum = hex($f[12]) || $c; # Simple_Uppercase_Mapping | |
131 | my $slm = hex($f[13]) || $c; # Simple_Lowercase_Mapping | |
61507e3c RK |
132 | # recalculate the upper/lower case mappings as offsets |
133 | my $ud = $sum - $c; | |
134 | my $ld = $slm - $c; | |
e5a5a138 RK |
135 | # update bounds on various values |
136 | $maxccc = $ccc if $ccc > $maxccc; # assumed never to be -ve | |
137 | $minud = $ud if $ud < $minud; | |
138 | $maxud = $ud if $ud > $maxud; | |
139 | $minld = $ld if $ld < $minld; | |
140 | $maxld = $ld if $ld > $maxld; | |
1a05e381 | 141 | if($start != $end) { |
0e843521 | 142 | printf STDERR "> range %04X-%04X is %s\n", $start, $end, $gc; |
1a05e381 RK |
143 | } |
144 | for($c = $start; $c <= $end; ++$c) { | |
0e843521 RK |
145 | my $d = { |
146 | "gc" => $gc, | |
147 | "ccc" => $ccc, | |
148 | "ud" => $ud, | |
149 | "ld" => $ld, | |
150 | }; | |
151 | if($dm ne '') { | |
16506c9d | 152 | my $maxref; |
f98fcddb RK |
153 | if($dm =~ /</) { |
154 | # This is a compatibility decomposition | |
0e843521 | 155 | $dm =~ s/^<.*>\s*//; |
f98fcddb | 156 | $d->{compat} = 1; |
16506c9d RK |
157 | $maxref = \$maxcompat; |
158 | } else { | |
159 | $maxref = \$maxcanon; | |
0e843521 | 160 | } |
99695df9 | 161 | $d->{decomp} = [map(hex($_), split(/\s+/, $dm))]; |
16506c9d RK |
162 | my $len = scalar @{$d->{decomp}}; |
163 | $$maxref = $len if $len > $$maxref; | |
164 | if(!$d->{compat}) { | |
165 | if(${$d->{decomp}}[0] >= 0xAC00 && ${$d->{decomp}}[0] <= 0xD7A3) { | |
166 | ++$hangul_syllable_decomps; | |
167 | } | |
168 | if(${$d->{decomp}}[0] >= 0x1100 && ${$d->{decomp}}[0] <= 0x115F) { | |
169 | ++$hangul_choseong_decomps; | |
170 | } | |
171 | } | |
0e843521 | 172 | } |
1a05e381 RK |
173 | $data{$c} = $d; |
174 | } | |
61507e3c | 175 | $cats{$gc} = 1; |
1a05e381 | 176 | $max = $end if $end > $max; |
61507e3c RK |
177 | } |
178 | ||
0b7052da RK |
179 | sub read_prop_with_ranges { |
180 | my $path = shift; | |
181 | my $propkey = shift; | |
182 | input($path); | |
183 | while(<>) { | |
184 | chomp; | |
185 | s/\s*\#.*//; | |
186 | next if $_ eq ''; | |
187 | my ($range, $propval) = split(/\s*;\s*/, $_); | |
188 | if($range =~ /(.*)\.\.(.*)/) { | |
189 | for my $c (hex($1) .. hex($2)) { | |
0e843521 | 190 | $data{$c}->{$propkey} = $propval; |
0b7052da RK |
191 | } |
192 | } else { | |
193 | my $c = hex($range); | |
0e843521 | 194 | $data{$c}->{$propkey} = $propval; |
35b651f0 | 195 | } |
0b7052da RK |
196 | } |
197 | } | |
198 | ||
349b7b74 | 199 | # Grapheme_Break etc |
0b7052da | 200 | read_prop_with_ranges("auxiliary/GraphemeBreakProperty.txt", "gbreak"); |
0b7052da | 201 | read_prop_with_ranges("auxiliary/WordBreakProperty.txt", "wbreak"); |
349b7b74 | 202 | read_prop_with_ranges("auxiliary/SentenceBreakProperty.txt", "sbreak"); |
0b7052da | 203 | |
349b7b74 RK |
204 | # Compute the full list and fill in the Extend category properly |
205 | my %gbreak = (); | |
206 | my %wbreak = (); | |
207 | my %sbreak = (); | |
0b7052da | 208 | for my $c (keys %data) { |
349b7b74 RK |
209 | if(!exists $data{$c}->{gbreak}) { |
210 | $data{$c}->{gbreak} = 'Other'; | |
211 | } | |
212 | $gbreak{$data{$c}->{gbreak}} = 1; | |
213 | ||
0b7052da | 214 | if(!exists $data{$c}->{wbreak}) { |
349b7b74 | 215 | if($data{$c}->{gbreak} eq 'Extend') { |
0b7052da RK |
216 | $data{$c}->{wbreak} = 'Extend'; |
217 | } else { | |
218 | $data{$c}->{wbreak} = 'Other'; | |
35b651f0 RK |
219 | } |
220 | } | |
349b7b74 RK |
221 | $wbreak{$data{$c}->{wbreak}} = 1; |
222 | ||
223 | if(!exists $data{$c}->{sbreak}) { | |
224 | if($data{$c}->{gbreak} eq 'Extend') { | |
225 | $data{$c}->{sbreak} = 'Extend'; | |
226 | } else { | |
227 | $data{$c}->{sbreak} = 'Other'; | |
228 | } | |
229 | } | |
230 | $sbreak{$data{$c}->{sbreak}} = 1; | |
35b651f0 RK |
231 | } |
232 | ||
99695df9 RK |
233 | # Various derived properties |
234 | input("DerivedNormalizationProps.txt"); | |
235 | while(<>) { | |
236 | chomp; | |
237 | s/\s*\#.*//; | |
238 | next if $_ eq ''; | |
239 | my @f = split(/\s*;\s*/, $_); | |
240 | if(@f == 2) { | |
241 | push(@f, 1); | |
242 | } | |
243 | my ($range, $propkey, $propval) = @f; | |
244 | if($range =~ /(.*)\.\.(.*)/) { | |
245 | for my $c (hex($1) .. hex($2)) { | |
246 | $data{$c}->{$propkey} = $propval | |
247 | } | |
248 | } else { | |
249 | my $c = hex($range); | |
250 | $data{$c}->{$propkey} = $propval | |
251 | } | |
252 | } | |
253 | ||
e5a5a138 RK |
254 | # Round up the maximum value to a whole number of subtables |
255 | $max += ($modulus - 1) - ($max % $modulus); | |
61507e3c | 256 | |
bcf9ed7f RK |
257 | # Private use characters |
258 | # We only fill in values below $max, utf32__unidata() | |
259 | my $Co = { | |
260 | "gc" => "Co", | |
261 | "ccc" => 0, | |
262 | "ud" => 0, | |
263 | "ld" => 0 | |
264 | }; | |
265 | for(my $c = 0xE000; $c <= 0xF8FF && $c <= $max; ++$c) { | |
266 | $data{$c} = $Co; | |
267 | } | |
268 | for(my $c = 0xF0000; $c <= 0xFFFFD && $c <= $max; ++$c) { | |
269 | $data{$c} = $Co; | |
270 | } | |
271 | for(my $c = 0x100000; $c <= 0x10FFFD && $c <= $max; ++$c) { | |
272 | $data{$c} = $Co; | |
273 | } | |
274 | ||
275 | # Anything left is not assigned | |
276 | my $Cn = { | |
277 | "gc" => "Cn", # not assigned | |
278 | "ccc" => 0, | |
279 | "ud" => 0, | |
280 | "ld" => 0 | |
281 | }; | |
61507e3c RK |
282 | for(my $c = 0; $c <= $max; ++$c) { |
283 | if(!exists $data{$c}) { | |
bcf9ed7f RK |
284 | $data{$c} = $Cn; |
285 | } | |
286 | if(!exists $data{$c}->{wbreak}) { | |
287 | $data{$c}->{wbreak} = 'Other'; | |
288 | } | |
289 | if(!exists $data{$c}->{gbreak}) { | |
290 | $data{$c}->{gbreak} = 'Other'; | |
291 | } | |
292 | if(!exists $data{$c}->{sbreak}) { | |
293 | $data{$c}->{sbreak} = 'Other'; | |
61507e3c RK |
294 | } |
295 | } | |
296 | $cats{'Cn'} = 1; | |
297 | ||
e5a5a138 | 298 | # Read the casefolding data too |
0b7052da | 299 | input("CaseFolding.txt"); |
e5a5a138 RK |
300 | while(<>) { |
301 | chomp; | |
302 | next if /^\#/ or $_ eq ''; | |
303 | my @f = split(/\s*;\s*/, $_); | |
304 | # Full case folding means use status C and F. | |
305 | # We discard status T, Turkish users may wish to change this. | |
306 | if($f[1] eq 'C' or $f[1] eq 'F') { | |
307 | my $c = hex($f[0]); | |
308 | $data{$c}->{casefold} = $f[2]; | |
309 | # We are particularly interest in combining characters that | |
310 | # case-fold to non-combining characters, or characters that | |
311 | # case-fold to sequences with combining characters in non-initial | |
312 | # positions, as these required decomposiiton before case-folding | |
313 | my @d = map(hex($_), split(/\s+/, $data{$c}->{casefold})); | |
314 | if($data{$c}->{ccc} != 0) { | |
315 | # This is a combining character | |
316 | if($data{$d[0]}->{ccc} == 0) { | |
317 | # The first character of its case-folded form is NOT | |
318 | # a combining character. The field name is the example | |
319 | # explicitly mentioned in the spec. | |
320 | $data{$c}->{ypogegrammeni} = 1; | |
321 | } | |
322 | } else { | |
323 | # This is a non-combining character; inspect the non-initial | |
324 | # code points of the case-folded sequence | |
325 | shift(@d); | |
326 | if(grep($data{$_}->{ccc} != 0, @d)) { | |
327 | # Some non-initial code point in the case-folded for is NOT a | |
328 | # a combining character. | |
329 | $data{$c}->{ypogegrammeni} = 1; | |
330 | } | |
331 | } | |
332 | } | |
333 | } | |
334 | ||
335 | # Generate the header file | |
bcf9ed7f | 336 | print STDERR "Generating unidata.h...\n"; |
61507e3c RK |
337 | open(STDOUT, ">unidata.h") or die "unidata.h: $!\n"; |
338 | ||
132a5a4a RK |
339 | out("/** \@file lib/unidata.h\n", |
340 | " * \@brief Unicode tables\n", | |
341 | " *\n", | |
342 | " * Automatically generated file, see scripts/make-unidata\n", | |
343 | " *\n", | |
344 | " * DO NOT EDIT.\n", | |
345 | " */\n", | |
e5a5a138 | 346 | "#ifndef UNIDATA_H\n", |
61507e3c RK |
347 | "#define UNIDATA_H\n"); |
348 | ||
e5a5a138 | 349 | # TODO choose stable values for General_Category |
14523635 | 350 | out("enum unicode_General_Category {\n", |
61507e3c | 351 | join(",\n", |
14523635 | 352 | map(" unicode_General_Category_$_", sort keys %cats)), "\n};\n"); |
e5a5a138 | 353 | |
349b7b74 RK |
354 | out("enum unicode_Grapheme_Break {\n", |
355 | join(",\n", | |
356 | map(" unicode_Grapheme_Break_$_", sort keys %gbreak)), | |
357 | "\n};\n"); | |
358 | out("extern const char *const unicode_Grapheme_Break_names[];\n"); | |
359 | ||
0b7052da RK |
360 | out("enum unicode_Word_Break {\n", |
361 | join(",\n", | |
349b7b74 | 362 | map(" unicode_Word_Break_$_", sort keys %wbreak)), |
0b7052da | 363 | "\n};\n"); |
bb48024f | 364 | out("extern const char *const unicode_Word_Break_names[];\n"); |
0b7052da | 365 | |
349b7b74 RK |
366 | out("enum unicode_Sentence_Break {\n", |
367 | join(",\n", | |
368 | map(" unicode_Sentence_Break_$_", sort keys %sbreak)), | |
369 | "\n};\n"); | |
370 | out("extern const char *const unicode_Sentence_Break_names[];\n"); | |
371 | ||
e5a5a138 | 372 | out("enum unicode_flags {\n", |
f98fcddb RK |
373 | " unicode_normalize_before_casefold = 1,\n", |
374 | " unicode_compatibility_decomposition = 2\n", | |
e5a5a138 RK |
375 | "};\n", |
376 | "\n"); | |
377 | ||
378 | # Choose the narrowest type that will fit the required values | |
379 | sub choosetype { | |
380 | my ($min, $max) = @_; | |
381 | if($min >= 0) { | |
382 | return "char" if $max <= 127; | |
383 | return "unsigned char" if $max <= 255; | |
384 | return "int16_t" if $max < 32767; | |
385 | return "uint16_t" if $max < 65535; | |
386 | return "int32_t"; | |
387 | } else { | |
388 | return "char" if $min >= -127 && $max <= 127; | |
389 | return "int16_t" if $min >= -32767 && $max <= 32767; | |
390 | return "int32_t"; | |
391 | } | |
392 | } | |
393 | ||
61507e3c | 394 | out("struct unidata {\n", |
99695df9 RK |
395 | # decomposition (canonical or compatibility; |
396 | # unicode_compatibility_decomposition distinguishes) or NULL | |
f98fcddb | 397 | " const uint32_t *decomp;\n", |
99695df9 RK |
398 | |
399 | # case-folded string or NULL | |
e5a5a138 | 400 | " const uint32_t *casefold;\n", |
99695df9 RK |
401 | |
402 | # composed characters that start with this code point. This only | |
403 | # includes primary composites, i.e. the decomposition mapping is | |
404 | # canonical and this code point is not in the exclusion table. | |
405 | " const uint32_t *composed;\n", | |
406 | ||
1a05e381 RK |
407 | # " ".choosetype($minud, $maxud)." upper_offset;\n", |
408 | # " ".choosetype($minld, $maxld)." lower_offset;\n", | |
99695df9 RK |
409 | |
410 | # canonical combining class | |
e5a5a138 | 411 | " ".choosetype(0, $maxccc)." ccc;\n", |
14523635 | 412 | " char general_category;\n", |
99695df9 RK |
413 | |
414 | # see unicode_flags enum | |
e5a5a138 | 415 | " uint8_t flags;\n", |
349b7b74 | 416 | " char grapheme_break;\n", |
0b7052da | 417 | " char word_break;\n", |
349b7b74 | 418 | " char sentence_break;\n", |
61507e3c | 419 | "};\n"); |
f98fcddb | 420 | # decomp and casefold do have have non-BMP characters, so we |
e5a5a138 RK |
421 | # can't use a simple 16-bit table. We could use UTF-8 or UTF-16 |
422 | # though, saving a bit of space (probably not that much...) at the | |
423 | # cost of marginally reduced performance and additional complexity | |
61507e3c RK |
424 | |
425 | out("extern const struct unidata *const unidata[];\n"); | |
426 | ||
18cda350 RK |
427 | out("extern const struct unicode_utf8_row {\n", |
428 | " uint8_t count;\n", | |
429 | " uint8_t min2, max2;\n", | |
430 | "} unicode_utf8_valid[];\n"); | |
431 | ||
61507e3c | 432 | out("#define UNICODE_NCHARS ", ($max + 1), "\n"); |
e5a5a138 | 433 | out("#define UNICODE_MODULUS $modulus\n"); |
1a05e381 RK |
434 | out("#define UNICODE_BREAK_START $break_start\n"); |
435 | out("#define UNICODE_BREAK_END $break_end\n"); | |
436 | out("#define UNICODE_BREAK_TOP $break_top\n"); | |
61507e3c RK |
437 | |
438 | out("#endif\n"); | |
439 | ||
440 | close STDOUT or die "unidata.h: $!\n"; | |
441 | ||
bcf9ed7f | 442 | print STDERR "Generating unidata.c...\n"; |
61507e3c RK |
443 | open(STDOUT, ">unidata.c") or die "unidata.c: $!\n"; |
444 | ||
132a5a4a RK |
445 | out("/** \@file lib/unidata.c\n", |
446 | " * \@brief Unicode tables\n", | |
447 | " *\n", | |
448 | " * Automatically generated file, see scripts/make-unidata\n", | |
449 | " *\n", | |
450 | " * DO NOT EDIT.\n", | |
451 | " */\n", | |
05b75f8d | 452 | "#include \"common.h\"\n", |
e5a5a138 RK |
453 | "#include \"unidata.h\"\n"); |
454 | ||
349b7b74 | 455 | # Short aliases to keep .c file small |
e5a5a138 | 456 | |
14523635 RK |
457 | out(map(sprintf("#define %s unicode_General_Category_%s\n", $_, $_), |
458 | sort keys %cats)); | |
459 | out(map(sprintf("#define GB%s unicode_Grapheme_Break_%s\n", $_, $_), | |
460 | sort keys %gbreak)); | |
461 | out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_), | |
462 | sort keys %wbreak)); | |
463 | out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_), | |
464 | sort keys %sbreak)); | |
99695df9 RK |
465 | out("#define NBC unicode_normalize_before_casefold\n"); |
466 | out("#define CD unicode_compatibility_decomposition\n"); | |
e5a5a138 | 467 | |
349b7b74 RK |
468 | # Names for *_Break properties |
469 | out("const char *const unicode_Grapheme_Break_names[] = {\n", | |
470 | join(",\n", | |
471 | map(" \"$_\"", sort keys %gbreak)), | |
472 | "\n};\n"); | |
bb48024f RK |
473 | out("const char *const unicode_Word_Break_names[] = {\n", |
474 | join(",\n", | |
349b7b74 RK |
475 | map(" \"$_\"", sort keys %wbreak)), |
476 | "\n};\n"); | |
477 | out("const char *const unicode_Sentence_Break_names[] = {\n", | |
478 | join(",\n", | |
479 | map(" \"$_\"", sort keys %sbreak)), | |
bb48024f RK |
480 | "\n};\n"); |
481 | ||
99695df9 RK |
482 | our $ddnum = 0; |
483 | our $ddsaved = 0; | |
484 | our %ddnums = (); | |
485 | my $ddfirst = 1; | |
486 | out("static const uint32_t "); | |
487 | sub dedupe { | |
488 | my $s = join(",", @_); | |
489 | if(!exists $ddnums{$s}) { | |
490 | if($ddfirst) { | |
491 | $ddfirst = 0; | |
492 | } else { | |
493 | out(",\n"); | |
494 | } | |
495 | out("dd$ddnum\[]={$s}"); | |
496 | $ddnums{$s} = $ddnum++; | |
497 | } else { | |
498 | ++$ddsaved; | |
499 | } | |
500 | return "dd$ddnums{$s}"; | |
501 | } | |
502 | ||
16506c9d | 503 | # Generate the decomposition mapping tables. |
99695df9 | 504 | print STDERR "> decomposition mappings\n"; |
e5a5a138 | 505 | for(my $c = 0; $c <= $max; ++$c) { |
f98fcddb | 506 | if(exists $data{$c} && exists $data{$c}->{decomp}) { |
99695df9 RK |
507 | $data{$c}->{decompsym} = dedupe(@{$data{$c}->{decomp}}, 0); |
508 | } | |
509 | } | |
510 | ||
511 | print STDERR "> composition mappings\n"; | |
512 | # First we must generate the mapping of each code point to possible | |
16506c9d | 513 | # compositions. |
99695df9 RK |
514 | for(my $c = 0; $c <= $max; ++$c) { |
515 | if(exists $data{$c} | |
516 | && exists $data{$c}->{decomp} | |
517 | && !exists $data{$c}->{compat} | |
518 | && !$data{$c}->{Full_Composition_Exclusion}) { | |
519 | # $c has a non-excluded canonical decomposition, i.e. it is | |
520 | # a primary composite. Find the first code point of the decomposition | |
521 | my $first = ${$data{$c}->{decomp}}[0]; | |
522 | if(!exists $data{$first}->{compose}) { | |
16506c9d | 523 | $data{$first}->{compose} = [$c]; |
e5a5a138 | 524 | } else { |
16506c9d | 525 | push(@{$data{$first}->{compose}}, $c); |
e5a5a138 | 526 | } |
e5a5a138 RK |
527 | } |
528 | } | |
16506c9d | 529 | # Then we can generate the tables. |
99695df9 RK |
530 | for(my $c = 0; $c <= $max; ++$c) { |
531 | if(exists $data{$c} && exists $data{$c}->{compose}) { | |
532 | $data{$c}->{compsym} = dedupe(@{$data{$c}->{compose}}, 0); | |
533 | } | |
534 | } | |
e5a5a138 | 535 | |
16506c9d | 536 | # The case folding table. |
99695df9 | 537 | print STDERR "> case-fold mappings\n"; |
e5a5a138 | 538 | for(my $c = 0; $c <= $max; ++$c) { |
bcf9ed7f | 539 | if(exists $data{$c} && exists $data{$c}->{casefold}) { |
99695df9 RK |
540 | $data{$c}->{cfsym} = dedupe(map(hex($_), split(/\s+/, |
541 | $data{$c}->{casefold})), | |
542 | 0); | |
e5a5a138 RK |
543 | } |
544 | } | |
99695df9 RK |
545 | |
546 | # End of de-dupable arrays | |
e5a5a138 RK |
547 | out(";\n"); |
548 | ||
549 | # Visit all the $modulus-character blocks in turn and generate the | |
550 | # required subtables. As above we spot duplicates to save space. In | |
551 | # Unicode 5.0.0 with $modulus=128 and current table data this saves | |
552 | # 1372 subtables or at least three and a half megabytes on 32-bit | |
553 | # platforms. | |
99695df9 | 554 | print STDERR "> subtables\n"; |
61507e3c RK |
555 | my %subtable = (); # base->subtable number |
556 | my %subtableno = (); # subtable number -> content | |
557 | my $subtablecounter = 0; # counter for subtable numbers | |
e5a5a138 RK |
558 | my $subtablessaved = 0; # number of tables saved |
559 | for(my $base = 0; $base <= $max; $base += $modulus) { | |
1a05e381 RK |
560 | next if $base >= $break_start && $base < $break_end; |
561 | next if $base >= $break_top; | |
61507e3c | 562 | my @t; |
e5a5a138 | 563 | for(my $c = $base; $c < $base + $modulus; ++$c) { |
61507e3c | 564 | my $d = $data{$c}; |
f98fcddb | 565 | my $decompsym = ($data{$c}->{decompsym} or "0"); |
e5a5a138 | 566 | my $cfsym = ($data{$c}->{cfsym} or "0"); |
99695df9 | 567 | my $compsym = ($data{$c}->{compsym} or "0"); |
35b651f0 RK |
568 | my @flags = (); |
569 | if($data{$c}->{ypogegrammeni}) { | |
99695df9 | 570 | push(@flags, "NBC"); |
35b651f0 | 571 | } |
f98fcddb | 572 | if($data{$c}->{compat}) { |
99695df9 | 573 | push(@flags, "CD"); |
f98fcddb | 574 | } |
35b651f0 | 575 | my $flags = @flags ? join("|", @flags) : 0; |
e5a5a138 RK |
576 | push(@t, "{". |
577 | join(",", | |
f98fcddb | 578 | $decompsym, |
e5a5a138 | 579 | $cfsym, |
99695df9 | 580 | $compsym, |
1a05e381 RK |
581 | # $d->{ud}, |
582 | # $d->{ld}, | |
0b7052da RK |
583 | $d->{ccc}, |
584 | $d->{gc}, | |
e5a5a138 | 585 | $flags, |
349b7b74 RK |
586 | "GB$d->{gbreak}", |
587 | "WB$d->{wbreak}", | |
588 | "SB$d->{sbreak}", | |
e5a5a138 | 589 | )."}"); |
61507e3c RK |
590 | } |
591 | my $t = join(",\n", @t); | |
592 | if(!exists $subtable{$t}) { | |
0e843521 | 593 | out(sprintf("/* %04X-%04X */\n", $base, $base + $modulus - 1)); |
e5a5a138 | 594 | out("static const struct unidata st$subtablecounter\[] = {\n", |
61507e3c RK |
595 | "$t\n", |
596 | "};\n"); | |
597 | $subtable{$t} = $subtablecounter++; | |
e5a5a138 RK |
598 | } else { |
599 | ++$subtablessaved; | |
61507e3c RK |
600 | } |
601 | $subtableno{$base} = $subtable{$t}; | |
602 | } | |
603 | ||
99695df9 | 604 | print STDERR "> main table\n"; |
bcf9ed7f | 605 | out("const struct unidata *const unidata[]={\n"); |
e5a5a138 | 606 | for(my $base = 0; $base <= $max; $base += $modulus) { |
1a05e381 RK |
607 | next if $base >= $break_start && $base < $break_end; |
608 | next if $base >= $break_top; | |
0a4af692 | 609 | #out("st$subtableno{$base} /* ".sprintf("%04x", $base)." */,\n"); |
e5a5a138 | 610 | out("st$subtableno{$base},\n"); |
61507e3c RK |
611 | } |
612 | out("};\n"); | |
613 | ||
99695df9 | 614 | print STDERR "> UTF-8 table\n"; |
18cda350 RK |
615 | out("const struct unicode_utf8_row unicode_utf8_valid[] = {\n"); |
616 | for(my $c = 0; $c <= 0x7F; ++$c) { | |
617 | out(" { 1, 0, 0 }, /* $c */\n"); | |
618 | } | |
619 | for(my $c = 0x80; $c < 0xC2; ++$c) { | |
620 | out(" { 0, 0, 0 }, /* $c */\n"); | |
621 | } | |
622 | for(my $c = 0xC2; $c <= 0xDF; ++$c) { | |
623 | out(" { 2, 0x80, 0xBF }, /* $c */\n"); | |
624 | } | |
625 | for(my $c = 0xE0; $c <= 0xE0; ++$c) { | |
626 | out(" { 3, 0xA0, 0xBF }, /* $c */\n"); | |
627 | } | |
628 | for(my $c = 0xE1; $c <= 0xEC; ++$c) { | |
629 | out(" { 3, 0x80, 0xBF }, /* $c */\n"); | |
630 | } | |
631 | for(my $c = 0xED; $c <= 0xED; ++$c) { | |
632 | out(" { 3, 0x80, 0x9F }, /* $c */\n"); | |
633 | } | |
634 | for(my $c = 0xEE; $c <= 0xEF; ++$c) { | |
635 | out(" { 3, 0x80, 0xBF }, /* $c */\n"); | |
636 | } | |
637 | for(my $c = 0xF0; $c <= 0xF0; ++$c) { | |
638 | out(" { 4, 0x90, 0xBF }, /* $c */\n"); | |
639 | } | |
640 | for(my $c = 0xF1; $c <= 0xF3; ++$c) { | |
641 | out(" { 4, 0x80, 0xBF }, /* $c */\n"); | |
642 | } | |
643 | for(my $c = 0xF4; $c <= 0xF4; ++$c) { | |
644 | out(" { 4, 0x80, 0x8F }, /* $c */\n"); | |
645 | } | |
646 | for(my $c = 0xF5; $c <= 0xFF; ++$c) { | |
647 | out(" { 0, 0, 0 }, /* $c */\n"); | |
648 | } | |
649 | out("};\n"); | |
650 | ||
61507e3c RK |
651 | close STDOUT or die "unidata.c: $!\n"; |
652 | ||
99695df9 | 653 | print STDERR "Done.\n\n"; |
c2e01e0a | 654 | printf STDERR "modulus=%d\n", $modulus; |
bcf9ed7f RK |
655 | printf STDERR "max=%04X\n", $max; |
656 | print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n"; | |
99695df9 | 657 | print STDERR "ddsaved=$ddsaved\n"; |
16506c9d RK |
658 | print STDERR "maxcompat=$maxcompat maxcanon=$maxcanon\n"; |
659 | print STDERR "$hangul_syllable_decomps canonical decompositions to Hangul syllables\n"; | |
660 | print STDERR "$hangul_choseong_decomps canonical decompositions to Hangul Choseong\n"; | |
661 | ||
662 | die "We assumed that canonical decompositions were never more than 2 long!\n" | |
663 | if $maxcanon > 2; | |
664 | ||
665 | die "We assumed no canonical decompositions to Hangul syllables/Choseong!\n" | |
666 | if $hangul_syllable_decomps || $hangul_choseong_decomps; |