3 # This file is part of DisOrder.
4 # Copyright (C) 2007 Richard Kettlewell
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
22 # Generate Unicode support tables
24 # This script will download data from unicode.org if the required files
25 # aren't in the current directory.
27 # After modifying this script you should run:
28 # make -C lib rebuild-unicode check
30 # Things not supported yet:
31 # - SpecialCasing.txt data for case mapping
32 # - Title case offsets
33 # - Some kind of hinting for composition
34 # - Word boundary support
37 # NB the generated files DO NOT offer a stable ABI and so are not immediately
38 # suitable for use in a general-purpose library. Things that would need to
40 # - Hide unidata.h from applications; it will never be ABI- or even API-stable.
41 # - Stablized General_Category values
42 # - Extend the unicode.h API to general utility rather than just what
50 print @_ or die "$!\n";
57 return join("-", map($d->{$_}, sort keys %$d));
62 # This can be varied to trade off the number of subtables against their size.
65 my %cats = (); # known general categories
66 my %data = (); # mapping of codepoints to information
67 my $max = 0; # maximum codepoint
68 my $maxccc = 0; # maximum combining class
70 my $minud = 0; # max/min upper case offset
72 my $minld = 0; # max/min lower case offset
74 # Make sure we have our desired input files. We explicitly specify a
75 # Unicode standard version to make sure that a given version of DisOrder
76 # supports a given version of Unicode.
79 my $lpath = basename($path);
81 system("wget http://www.unicode.org/Public/5.0.0/ucd/$path");
82 chmod(0444, $lpath) or die "$lpath: $!\n";
84 open(STDIN, "<$lpath") or die "$lpath: $!\n";
85 print STDERR "Reading $lpath...\n";
89 # Read the main data file
90 input("UnicodeData.txt");
92 my @f = split(/;/, $_);
93 my $c = hex($f[0]); # codepoint
94 next if $c >= 0xE0000; # ignore various high-numbered stuff
95 # TODO justify this exclusion!
97 my $gc = $f[2]; # General_Category
98 # Variuos GCs we don't expect to see in UnicodeData.txt
99 $cats{$gc} = 1; # always record all GCs
100 next if $name =~ /(first|last)>/i; # ignore placeholders
101 die "unexpected Cn" if $gc eq 'Cn';
102 die "unexpected Co" if $gc eq 'Cn';
103 die "unexpected Cs" if $gc eq 'Cs';
104 my $ccc = $f[3]; # Canonical_Combining_Class
105 my $dm = $f[5]; # Decomposition_Type + Decomposition_Mapping
106 my $sum = hex($f[12]) || $c; # Simple_Uppercase_Mapping
107 my $slm = hex($f[13]) || $c; # Simple_Lowercase_Mapping
108 # recalculate the upper/lower case mappings as offsets
111 # update bounds on various values
112 $maxccc = $ccc if $ccc > $maxccc; # assumed never to be -ve
113 $minud = $ud if $ud < $minud;
114 $maxud = $ud if $ud > $maxud;
115 $minld = $ld if $ld < $minld;
116 $maxld = $ld if $ld > $maxld;
125 # This is a canonical decomposition
126 $data{$c}->{canon} = $dm;
127 $data{$c}->{compat} = $dm;
129 # This is only a compatibility decomposition
131 $data{$c}->{compat} = $dm;
135 $max = $c if $c > $max;
138 sub read_prop_with_ranges {
146 my ($range, $propval) = split(/\s*;\s*/, $_);
147 if($range =~ /(.*)\.\.(.*)/) {
148 for my $c (hex($1) .. hex($2)) {
149 if(exists $data{$c}) {
150 $data{$c}->{$propkey} = $propval;
155 if(exists $data{$c}) {
156 $data{$c}->{$propkey} = $propval;
163 # NB we do this BEFORE filling in blanks so that the Hangul characters
164 # don't get filled in; we can compute their properties mechanically.
165 read_prop_with_ranges("auxiliary/GraphemeBreakProperty.txt", "gbreak");
166 read_prop_with_ranges("auxiliary/WordBreakProperty.txt", "wbreak");
167 read_prop_with_ranges("auxiliary/SentenceBreakProperty.txt", "sbreak");
169 # Compute the full list and fill in the Extend category properly
173 for my $c (keys %data) {
174 if(!exists $data{$c}->{gbreak}) {
175 $data{$c}->{gbreak} = 'Other';
177 $gbreak{$data{$c}->{gbreak}} = 1;
179 if(!exists $data{$c}->{wbreak}) {
180 if($data{$c}->{gbreak} eq 'Extend') {
181 $data{$c}->{wbreak} = 'Extend';
183 $data{$c}->{wbreak} = 'Other';
186 $wbreak{$data{$c}->{wbreak}} = 1;
188 if(!exists $data{$c}->{sbreak}) {
189 if($data{$c}->{gbreak} eq 'Extend') {
190 $data{$c}->{sbreak} = 'Extend';
192 $data{$c}->{sbreak} = 'Other';
195 $sbreak{$data{$c}->{sbreak}} = 1;
198 # Round up the maximum value to a whole number of subtables
199 $max += ($modulus - 1) - ($max % $modulus);
203 "gc" => "Cs", # UTF-16 surrogate
208 for(my $c = 0xD800; $c <= 0xDFFF; ++$c) {
212 # Private use characters
213 # We only fill in values below $max, utf32__unidata()
220 for(my $c = 0xE000; $c <= 0xF8FF && $c <= $max; ++$c) {
223 for(my $c = 0xF0000; $c <= 0xFFFFD && $c <= $max; ++$c) {
226 for(my $c = 0x100000; $c <= 0x10FFFD && $c <= $max; ++$c) {
230 # Anything left is not assigned
232 "gc" => "Cn", # not assigned
237 for(my $c = 0; $c <= $max; ++$c) {
238 if(!exists $data{$c}) {
241 if(!exists $data{$c}->{wbreak}) {
242 $data{$c}->{wbreak} = 'Other';
244 if(!exists $data{$c}->{gbreak}) {
245 $data{$c}->{gbreak} = 'Other';
247 if(!exists $data{$c}->{sbreak}) {
248 $data{$c}->{sbreak} = 'Other';
253 # Read the casefolding data too
254 input("CaseFolding.txt");
257 next if /^\#/ or $_ eq '';
258 my @f = split(/\s*;\s*/, $_);
259 # Full case folding means use status C and F.
260 # We discard status T, Turkish users may wish to change this.
261 if($f[1] eq 'C' or $f[1] eq 'F') {
263 $data{$c}->{casefold} = $f[2];
264 # We are particularly interest in combining characters that
265 # case-fold to non-combining characters, or characters that
266 # case-fold to sequences with combining characters in non-initial
267 # positions, as these required decomposiiton before case-folding
268 my @d = map(hex($_), split(/\s+/, $data{$c}->{casefold}));
269 if($data{$c}->{ccc} != 0) {
270 # This is a combining character
271 if($data{$d[0]}->{ccc} == 0) {
272 # The first character of its case-folded form is NOT
273 # a combining character. The field name is the example
274 # explicitly mentioned in the spec.
275 $data{$c}->{ypogegrammeni} = 1;
278 # This is a non-combining character; inspect the non-initial
279 # code points of the case-folded sequence
281 if(grep($data{$_}->{ccc} != 0, @d)) {
282 # Some non-initial code point in the case-folded for is NOT a
283 # a combining character.
284 $data{$c}->{ypogegrammeni} = 1;
290 # Generate the header file
291 print STDERR "Generating unidata.h...\n";
292 open(STDOUT, ">unidata.h") or die "unidata.h: $!\n";
294 out("/* Automatically generated file, see scripts/make-unidata */\n",
295 "#ifndef UNIDATA_H\n",
296 "#define UNIDATA_H\n");
298 # TODO choose stable values for General_Category
299 out("enum unicode_General_Category {\n",
301 map(" unicode_General_Category_$_", sort keys %cats)), "\n};\n");
303 out("enum unicode_Grapheme_Break {\n",
305 map(" unicode_Grapheme_Break_$_", sort keys %gbreak)),
307 out("extern const char *const unicode_Grapheme_Break_names[];\n");
309 out("enum unicode_Word_Break {\n",
311 map(" unicode_Word_Break_$_", sort keys %wbreak)),
313 out("extern const char *const unicode_Word_Break_names[];\n");
315 out("enum unicode_Sentence_Break {\n",
317 map(" unicode_Sentence_Break_$_", sort keys %sbreak)),
319 out("extern const char *const unicode_Sentence_Break_names[];\n");
321 out("enum unicode_flags {\n",
322 " unicode_normalize_before_casefold = 1\n",
326 # Choose the narrowest type that will fit the required values
328 my ($min, $max) = @_;
330 return "char" if $max <= 127;
331 return "unsigned char" if $max <= 255;
332 return "int16_t" if $max < 32767;
333 return "uint16_t" if $max < 65535;
336 return "char" if $min >= -127 && $max <= 127;
337 return "int16_t" if $min >= -32767 && $max <= 32767;
342 out("struct unidata {\n",
343 " const uint32_t *compat;\n",
344 " const uint32_t *canon;\n",
345 " const uint32_t *casefold;\n",
346 " ".choosetype($minud, $maxud)." upper_offset;\n",
347 " ".choosetype($minld, $maxld)." lower_offset;\n",
348 " ".choosetype(0, $maxccc)." ccc;\n",
349 " char general_category;\n",
351 " char grapheme_break;\n",
352 " char word_break;\n",
353 " char sentence_break;\n",
355 # compat, canon and casefold do have have non-BMP characters, so we
356 # can't use a simple 16-bit table. We could use UTF-8 or UTF-16
357 # though, saving a bit of space (probably not that much...) at the
358 # cost of marginally reduced performance and additional complexity
360 out("extern const struct unidata *const unidata[];\n");
362 out("#define UNICODE_NCHARS ", ($max + 1), "\n");
363 out("#define UNICODE_MODULUS $modulus\n");
367 close STDOUT or die "unidata.h: $!\n";
369 print STDERR "Generating unidata.c...\n";
370 open(STDOUT, ">unidata.c") or die "unidata.c: $!\n";
372 out("/* Automatically generated file, see scripts/make-unidata */\n",
373 "#include <config.h>\n",
374 "#include \"types.h\"\n",
375 "#include \"unidata.h\"\n");
377 # Short aliases to keep .c file small
379 out(map(sprintf("#define %s unicode_General_Category_%s\n", $_, $_),
381 out(map(sprintf("#define GB%s unicode_Grapheme_Break_%s\n", $_, $_),
383 out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_),
385 out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_),
388 # Names for *_Break properties
389 out("const char *const unicode_Grapheme_Break_names[] = {\n",
391 map(" \"$_\"", sort keys %gbreak)),
393 out("const char *const unicode_Word_Break_names[] = {\n",
395 map(" \"$_\"", sort keys %wbreak)),
397 out("const char *const unicode_Sentence_Break_names[] = {\n",
399 map(" \"$_\"", sort keys %sbreak)),
402 # Generate the decomposition mapping tables. We look out for duplicates
403 # in order to save space and report this as decompsaved at the end. In
404 # Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes.
408 out("static const uint32_t ");
409 for(my $c = 0; $c <= $max; ++$c) {
410 # If canon is set then compat will be too and will be identical.
411 # If compat is set the canon might be clear. So we use the
412 # compat version and fix up the symbols after.
413 if(exists $data{$c} && exists $data{$c}->{compat}) {
415 (map(hex($_), split(/\s+/, $data{$c}->{compat})), 0));
416 if(!exists $decompnums{$s}) {
417 out(",\n") if $decompnum != 0;
418 out("cd$decompnum\[]={$s}");
419 $decompnums{$s} = $decompnum++;
423 $data{$c}->{compatsym} = "cd$decompnums{$s}";
424 if(exists $data{$c}->{canon}) {
425 $data{$c}->{canonsym} = "cd$decompnums{$s}";
431 # ...and the case folding table. Again we compress equal entries to save
432 # space. In Unicode 5.0.0 this saves 51 entries or at least 408 bytes.
433 # This doesns't seem as worthwhile as the decomposition mapping saving above.
437 out("static const uint32_t ");
438 for(my $c = 0; $c <= $max; ++$c) {
439 if(exists $data{$c} && exists $data{$c}->{casefold}) {
441 (map(hex($_), split(/\s+/, $data{$c}->{casefold})), 0));
442 if(!exists $cfnums{$s}) {
443 out(",\n") if $cfnum != 0;
444 out("cf$cfnum\[]={$s}");
445 $cfnums{$s} = $cfnum++;
449 $data{$c}->{cfsym} = "cf$cfnums{$s}";
454 # Visit all the $modulus-character blocks in turn and generate the
455 # required subtables. As above we spot duplicates to save space. In
456 # Unicode 5.0.0 with $modulus=128 and current table data this saves
457 # 1372 subtables or at least three and a half megabytes on 32-bit
460 my %subtable = (); # base->subtable number
461 my %subtableno = (); # subtable number -> content
462 my $subtablecounter = 0; # counter for subtable numbers
463 my $subtablessaved = 0; # number of tables saved
464 for(my $base = 0; $base <= $max; $base += $modulus) {
466 for(my $c = $base; $c < $base + $modulus; ++$c) {
468 my $canonsym = ($data{$c}->{canonsym} or "0");
469 my $compatsym = ($data{$c}->{compatsym} or "0");
470 my $cfsym = ($data{$c}->{cfsym} or "0");
472 if($data{$c}->{ypogegrammeni}) {
473 push(@flags, "unicode_normalize_before_casefold");
475 my $flags = @flags ? join("|", @flags) : 0;
491 my $t = join(",\n", @t);
492 if(!exists $subtable{$t}) {
493 out("static const struct unidata st$subtablecounter\[] = {\n",
496 $subtable{$t} = $subtablecounter++;
500 $subtableno{$base} = $subtable{$t};
503 out("const struct unidata *const unidata[]={\n");
504 for(my $base = 0; $base <= $max; $base += $modulus) {
505 #out("st$subtableno{$base} /* ".sprintf("%04x", $base)." */,\n");
506 out("st$subtableno{$base},\n");
510 close STDOUT or die "unidata.c: $!\n";
512 printf STDERR "max=%04X\n", $max;
513 print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n";
514 print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n";