chiark / gitweb /
start on ut32__unidata() which will provide a uniform interface
[disorder] / scripts / make-unidata
CommitLineData
61507e3c
RK
1#! /usr/bin/perl -w
2#
e5a5a138
RK
3# This file is part of DisOrder.
4# Copyright (C) 2007 Richard Kettlewell
5#
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
10#
11# This program is distributed in the hope that it will be useful, but
12# WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14# General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
19# USA
20#
21#
22# Generate Unicode support tables
23#
24# This script will download data from unicode.org if the required files
25# aren't in the current directory.
26#
27# After modifying this script you should run:
28# make -C lib rebuild-unicode check
29#
30# Things not supported yet:
31# - SpecialCasing.txt data for case mapping
32# - Title case offsets
33# - Some kind of hinting for composition
34# - Word boundary support
35# - ...
36#
37# NB the generated files DO NOT offer a stable ABI and so are not immediately
38# suitable for use in a general-purpose library. Things that would need to
39# be done:
40# - Hide unidata.h from applications; it will never be ABI- or even API-stable.
41# - Stablized General_Category values
42# - Extend the unicode.h API to general utility rather than just what
43# DisOrder needs.
44# - ...
45#
61507e3c 46use strict;
35b651f0 47use File::Basename;
61507e3c
RK
48
49sub out {
50 print @_ or die "$!\n";
51}
52
53sub key {
54 my $d = shift;
55 local $_;
56
57 return join("-", map($d->{$_}, sort keys %$d));
58}
59
e5a5a138
RK
60# Size of a subtable
61#
62# This can be varied to trade off the number of subtables against their size.
63our $modulus = 128;
64
61507e3c
RK
65my %cats = (); # known general categories
66my %data = (); # mapping of codepoints to information
61507e3c 67my $max = 0; # maximum codepoint
e5a5a138
RK
68my $maxccc = 0; # maximum combining class
69my $maxud = 0;
70my $minud = 0; # max/min upper case offset
71my $maxld = 0;
72my $minld = 0; # max/min lower case offset
73
74# Make sure we have our desired input files. We explicitly specify a
75# Unicode standard version to make sure that a given version of DisOrder
76# supports a given version of Unicode.
0b7052da 77sub input {
35b651f0
RK
78 my $path = shift;
79 my $lpath = basename($path);
80 if(!-e $lpath) {
81 system("wget http://www.unicode.org/Public/5.0.0/ucd/$path");
82 chmod(0444, $lpath) or die "$lpath: $!\n";
e5a5a138 83 }
0b7052da 84 open(STDIN, "<$lpath") or die "$lpath: $!\n";
bcf9ed7f 85 print STDERR "Reading $lpath...\n";
e5a5a138 86}
61507e3c 87
e5a5a138
RK
88
89# Read the main data file
0b7052da 90input("UnicodeData.txt");
61507e3c
RK
91while(<>) {
92 my @f = split(/;/, $_);
93 my $c = hex($f[0]); # codepoint
94 next if $c >= 0xE0000; # ignore various high-numbered stuff
e5a5a138 95 # TODO justify this exclusion!
61507e3c 96 my $name = $f[1];
e5a5a138 97 my $gc = $f[2]; # General_Category
bcf9ed7f 98 # Variuos GCs we don't expect to see in UnicodeData.txt
0a4af692
RK
99 $cats{$gc} = 1; # always record all GCs
100 next if $name =~ /(first|last)>/i; # ignore placeholders
bcf9ed7f
RK
101 die "unexpected Cn" if $gc eq 'Cn';
102 die "unexpected Co" if $gc eq 'Cn';
103 die "unexpected Cs" if $gc eq 'Cs';
e5a5a138
RK
104 my $ccc = $f[3]; # Canonical_Combining_Class
105 my $dm = $f[5]; # Decomposition_Type + Decomposition_Mapping
106 my $sum = hex($f[12]) || $c; # Simple_Uppercase_Mapping
107 my $slm = hex($f[13]) || $c; # Simple_Lowercase_Mapping
61507e3c
RK
108 # recalculate the upper/lower case mappings as offsets
109 my $ud = $sum - $c;
110 my $ld = $slm - $c;
e5a5a138
RK
111 # update bounds on various values
112 $maxccc = $ccc if $ccc > $maxccc; # assumed never to be -ve
113 $minud = $ud if $ud < $minud;
114 $maxud = $ud if $ud > $maxud;
115 $minld = $ld if $ld < $minld;
116 $maxld = $ld if $ld > $maxld;
61507e3c
RK
117 $data{$c} = {
118 "gc" => $gc,
119 "ccc" => $ccc,
120 "ud" => $ud,
e5a5a138 121 "ld" => $ld,
61507e3c 122 };
e5a5a138
RK
123 if($dm ne '') {
124 if($dm !~ /</) {
125 # This is a canonical decomposition
126 $data{$c}->{canon} = $dm;
127 $data{$c}->{compat} = $dm;
128 } else {
129 # This is only a compatibility decomposition
130 $dm =~ s/^<.*>\s*//;
131 $data{$c}->{compat} = $dm;
132 }
133 }
61507e3c
RK
134 $cats{$gc} = 1;
135 $max = $c if $c > $max;
136}
137
0b7052da
RK
138sub read_prop_with_ranges {
139 my $path = shift;
140 my $propkey = shift;
141 input($path);
142 while(<>) {
143 chomp;
144 s/\s*\#.*//;
145 next if $_ eq '';
146 my ($range, $propval) = split(/\s*;\s*/, $_);
147 if($range =~ /(.*)\.\.(.*)/) {
148 for my $c (hex($1) .. hex($2)) {
149 if(exists $data{$c}) {
150 $data{$c}->{$propkey} = $propval;
151 }
152 }
153 } else {
154 my $c = hex($range);
35b651f0 155 if(exists $data{$c}) {
0b7052da 156 $data{$c}->{$propkey} = $propval;
35b651f0
RK
157 }
158 }
0b7052da
RK
159 }
160}
161
349b7b74 162# Grapheme_Break etc
0b7052da
RK
163# NB we do this BEFORE filling in blanks so that the Hangul characters
164# don't get filled in; we can compute their properties mechanically.
165read_prop_with_ranges("auxiliary/GraphemeBreakProperty.txt", "gbreak");
0b7052da 166read_prop_with_ranges("auxiliary/WordBreakProperty.txt", "wbreak");
349b7b74 167read_prop_with_ranges("auxiliary/SentenceBreakProperty.txt", "sbreak");
0b7052da 168
349b7b74
RK
169# Compute the full list and fill in the Extend category properly
170my %gbreak = ();
171my %wbreak = ();
172my %sbreak = ();
0b7052da 173for my $c (keys %data) {
349b7b74
RK
174 if(!exists $data{$c}->{gbreak}) {
175 $data{$c}->{gbreak} = 'Other';
176 }
177 $gbreak{$data{$c}->{gbreak}} = 1;
178
0b7052da 179 if(!exists $data{$c}->{wbreak}) {
349b7b74 180 if($data{$c}->{gbreak} eq 'Extend') {
0b7052da
RK
181 $data{$c}->{wbreak} = 'Extend';
182 } else {
183 $data{$c}->{wbreak} = 'Other';
35b651f0
RK
184 }
185 }
349b7b74
RK
186 $wbreak{$data{$c}->{wbreak}} = 1;
187
188 if(!exists $data{$c}->{sbreak}) {
189 if($data{$c}->{gbreak} eq 'Extend') {
190 $data{$c}->{sbreak} = 'Extend';
191 } else {
192 $data{$c}->{sbreak} = 'Other';
193 }
194 }
195 $sbreak{$data{$c}->{sbreak}} = 1;
35b651f0
RK
196}
197
e5a5a138
RK
198# Round up the maximum value to a whole number of subtables
199$max += ($modulus - 1) - ($max % $modulus);
61507e3c 200
bcf9ed7f
RK
201# Surrogates
202my $Cs = {
203 "gc" => "Cs", # UTF-16 surrogate
204 "ccc" => 0,
205 "ud" => 0,
206 "ld" => 0
207};
208for(my $c = 0xD800; $c <= 0xDFFF; ++$c) {
209 $data{$c} = $Cs;
210}
211
212# Private use characters
213# We only fill in values below $max, utf32__unidata()
214my $Co = {
215 "gc" => "Co",
216 "ccc" => 0,
217 "ud" => 0,
218 "ld" => 0
219};
220for(my $c = 0xE000; $c <= 0xF8FF && $c <= $max; ++$c) {
221 $data{$c} = $Co;
222}
223for(my $c = 0xF0000; $c <= 0xFFFFD && $c <= $max; ++$c) {
224 $data{$c} = $Co;
225}
226for(my $c = 0x100000; $c <= 0x10FFFD && $c <= $max; ++$c) {
227 $data{$c} = $Co;
228}
229
230# Anything left is not assigned
231my $Cn = {
232 "gc" => "Cn", # not assigned
233 "ccc" => 0,
234 "ud" => 0,
235 "ld" => 0
236};
61507e3c
RK
237for(my $c = 0; $c <= $max; ++$c) {
238 if(!exists $data{$c}) {
bcf9ed7f
RK
239 $data{$c} = $Cn;
240 }
241 if(!exists $data{$c}->{wbreak}) {
242 $data{$c}->{wbreak} = 'Other';
243 }
244 if(!exists $data{$c}->{gbreak}) {
245 $data{$c}->{gbreak} = 'Other';
246 }
247 if(!exists $data{$c}->{sbreak}) {
248 $data{$c}->{sbreak} = 'Other';
61507e3c
RK
249 }
250}
251$cats{'Cn'} = 1;
252
e5a5a138 253# Read the casefolding data too
0b7052da 254input("CaseFolding.txt");
e5a5a138
RK
255while(<>) {
256 chomp;
257 next if /^\#/ or $_ eq '';
258 my @f = split(/\s*;\s*/, $_);
259 # Full case folding means use status C and F.
260 # We discard status T, Turkish users may wish to change this.
261 if($f[1] eq 'C' or $f[1] eq 'F') {
262 my $c = hex($f[0]);
263 $data{$c}->{casefold} = $f[2];
264 # We are particularly interest in combining characters that
265 # case-fold to non-combining characters, or characters that
266 # case-fold to sequences with combining characters in non-initial
267 # positions, as these required decomposiiton before case-folding
268 my @d = map(hex($_), split(/\s+/, $data{$c}->{casefold}));
269 if($data{$c}->{ccc} != 0) {
270 # This is a combining character
271 if($data{$d[0]}->{ccc} == 0) {
272 # The first character of its case-folded form is NOT
273 # a combining character. The field name is the example
274 # explicitly mentioned in the spec.
275 $data{$c}->{ypogegrammeni} = 1;
276 }
277 } else {
278 # This is a non-combining character; inspect the non-initial
279 # code points of the case-folded sequence
280 shift(@d);
281 if(grep($data{$_}->{ccc} != 0, @d)) {
282 # Some non-initial code point in the case-folded for is NOT a
283 # a combining character.
284 $data{$c}->{ypogegrammeni} = 1;
285 }
286 }
287 }
288}
289
290# Generate the header file
bcf9ed7f 291print STDERR "Generating unidata.h...\n";
61507e3c
RK
292open(STDOUT, ">unidata.h") or die "unidata.h: $!\n";
293
e5a5a138
RK
294out("/* Automatically generated file, see scripts/make-unidata */\n",
295 "#ifndef UNIDATA_H\n",
61507e3c
RK
296 "#define UNIDATA_H\n");
297
e5a5a138 298# TODO choose stable values for General_Category
14523635 299out("enum unicode_General_Category {\n",
61507e3c 300 join(",\n",
14523635 301 map(" unicode_General_Category_$_", sort keys %cats)), "\n};\n");
e5a5a138 302
349b7b74
RK
303out("enum unicode_Grapheme_Break {\n",
304 join(",\n",
305 map(" unicode_Grapheme_Break_$_", sort keys %gbreak)),
306 "\n};\n");
307out("extern const char *const unicode_Grapheme_Break_names[];\n");
308
0b7052da
RK
309out("enum unicode_Word_Break {\n",
310 join(",\n",
349b7b74 311 map(" unicode_Word_Break_$_", sort keys %wbreak)),
0b7052da 312 "\n};\n");
bb48024f 313out("extern const char *const unicode_Word_Break_names[];\n");
0b7052da 314
349b7b74
RK
315out("enum unicode_Sentence_Break {\n",
316 join(",\n",
317 map(" unicode_Sentence_Break_$_", sort keys %sbreak)),
318 "\n};\n");
319out("extern const char *const unicode_Sentence_Break_names[];\n");
320
e5a5a138 321out("enum unicode_flags {\n",
0b7052da 322 " unicode_normalize_before_casefold = 1\n",
e5a5a138
RK
323 "};\n",
324 "\n");
325
326# Choose the narrowest type that will fit the required values
327sub choosetype {
328 my ($min, $max) = @_;
329 if($min >= 0) {
330 return "char" if $max <= 127;
331 return "unsigned char" if $max <= 255;
332 return "int16_t" if $max < 32767;
333 return "uint16_t" if $max < 65535;
334 return "int32_t";
335 } else {
336 return "char" if $min >= -127 && $max <= 127;
337 return "int16_t" if $min >= -32767 && $max <= 32767;
338 return "int32_t";
339 }
340}
341
61507e3c 342out("struct unidata {\n",
e5a5a138
RK
343 " const uint32_t *compat;\n",
344 " const uint32_t *canon;\n",
345 " const uint32_t *casefold;\n",
346 " ".choosetype($minud, $maxud)." upper_offset;\n",
347 " ".choosetype($minld, $maxld)." lower_offset;\n",
348 " ".choosetype(0, $maxccc)." ccc;\n",
14523635 349 " char general_category;\n",
e5a5a138 350 " uint8_t flags;\n",
349b7b74 351 " char grapheme_break;\n",
0b7052da 352 " char word_break;\n",
349b7b74 353 " char sentence_break;\n",
61507e3c 354 "};\n");
e5a5a138
RK
355# compat, canon and casefold do have have non-BMP characters, so we
356# can't use a simple 16-bit table. We could use UTF-8 or UTF-16
357# though, saving a bit of space (probably not that much...) at the
358# cost of marginally reduced performance and additional complexity
61507e3c
RK
359
360out("extern const struct unidata *const unidata[];\n");
361
362out("#define UNICODE_NCHARS ", ($max + 1), "\n");
e5a5a138 363out("#define UNICODE_MODULUS $modulus\n");
61507e3c
RK
364
365out("#endif\n");
366
367close STDOUT or die "unidata.h: $!\n";
368
bcf9ed7f 369print STDERR "Generating unidata.c...\n";
61507e3c
RK
370open(STDOUT, ">unidata.c") or die "unidata.c: $!\n";
371
e5a5a138
RK
372out("/* Automatically generated file, see scripts/make-unidata */\n",
373 "#include <config.h>\n",
374 "#include \"types.h\"\n",
375 "#include \"unidata.h\"\n");
376
349b7b74 377# Short aliases to keep .c file small
e5a5a138 378
14523635
RK
379out(map(sprintf("#define %s unicode_General_Category_%s\n", $_, $_),
380 sort keys %cats));
381out(map(sprintf("#define GB%s unicode_Grapheme_Break_%s\n", $_, $_),
382 sort keys %gbreak));
383out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_),
384 sort keys %wbreak));
385out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_),
386 sort keys %sbreak));
e5a5a138 387
349b7b74
RK
388# Names for *_Break properties
389out("const char *const unicode_Grapheme_Break_names[] = {\n",
390 join(",\n",
391 map(" \"$_\"", sort keys %gbreak)),
392 "\n};\n");
bb48024f
RK
393out("const char *const unicode_Word_Break_names[] = {\n",
394 join(",\n",
349b7b74
RK
395 map(" \"$_\"", sort keys %wbreak)),
396 "\n};\n");
397out("const char *const unicode_Sentence_Break_names[] = {\n",
398 join(",\n",
399 map(" \"$_\"", sort keys %sbreak)),
bb48024f
RK
400 "\n};\n");
401
e5a5a138
RK
402# Generate the decomposition mapping tables. We look out for duplicates
403# in order to save space and report this as decompsaved at the end. In
404# Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes.
405my $decompnum = 0;
406my %decompnums = ();
407my $decompsaved = 0;
408out("static const uint32_t ");
409for(my $c = 0; $c <= $max; ++$c) {
410 # If canon is set then compat will be too and will be identical.
411 # If compat is set the canon might be clear. So we use the
412 # compat version and fix up the symbols after.
bcf9ed7f 413 if(exists $data{$c} && exists $data{$c}->{compat}) {
e5a5a138
RK
414 my $s = join(",",
415 (map(hex($_), split(/\s+/, $data{$c}->{compat})), 0));
416 if(!exists $decompnums{$s}) {
417 out(",\n") if $decompnum != 0;
418 out("cd$decompnum\[]={$s}");
419 $decompnums{$s} = $decompnum++;
420 } else {
421 ++$decompsaved;
422 }
423 $data{$c}->{compatsym} = "cd$decompnums{$s}";
424 if(exists $data{$c}->{canon}) {
425 $data{$c}->{canonsym} = "cd$decompnums{$s}";
426 }
427 }
428}
429out(";\n");
430
431# ...and the case folding table. Again we compress equal entries to save
432# space. In Unicode 5.0.0 this saves 51 entries or at least 408 bytes.
433# This doesns't seem as worthwhile as the decomposition mapping saving above.
434my $cfnum = 0;
435my %cfnums = ();
436my $cfsaved = 0;
437out("static const uint32_t ");
438for(my $c = 0; $c <= $max; ++$c) {
bcf9ed7f 439 if(exists $data{$c} && exists $data{$c}->{casefold}) {
e5a5a138
RK
440 my $s = join(",",
441 (map(hex($_), split(/\s+/, $data{$c}->{casefold})), 0));
442 if(!exists $cfnums{$s}) {
443 out(",\n") if $cfnum != 0;
444 out("cf$cfnum\[]={$s}");
445 $cfnums{$s} = $cfnum++;
446 } else {
447 ++$cfsaved;
448 }
449 $data{$c}->{cfsym} = "cf$cfnums{$s}";
450 }
451}
452out(";\n");
453
454# Visit all the $modulus-character blocks in turn and generate the
455# required subtables. As above we spot duplicates to save space. In
456# Unicode 5.0.0 with $modulus=128 and current table data this saves
457# 1372 subtables or at least three and a half megabytes on 32-bit
458# platforms.
61507e3c 459
61507e3c
RK
460my %subtable = (); # base->subtable number
461my %subtableno = (); # subtable number -> content
462my $subtablecounter = 0; # counter for subtable numbers
e5a5a138
RK
463my $subtablessaved = 0; # number of tables saved
464for(my $base = 0; $base <= $max; $base += $modulus) {
61507e3c 465 my @t;
e5a5a138 466 for(my $c = $base; $c < $base + $modulus; ++$c) {
61507e3c 467 my $d = $data{$c};
e5a5a138
RK
468 my $canonsym = ($data{$c}->{canonsym} or "0");
469 my $compatsym = ($data{$c}->{compatsym} or "0");
470 my $cfsym = ($data{$c}->{cfsym} or "0");
35b651f0
RK
471 my @flags = ();
472 if($data{$c}->{ypogegrammeni}) {
473 push(@flags, "unicode_normalize_before_casefold");
474 }
35b651f0 475 my $flags = @flags ? join("|", @flags) : 0;
e5a5a138
RK
476 push(@t, "{".
477 join(",",
478 $compatsym,
479 $canonsym,
480 $cfsym,
0b7052da
RK
481 $d->{ud},
482 $d->{ld},
483 $d->{ccc},
484 $d->{gc},
e5a5a138 485 $flags,
349b7b74
RK
486 "GB$d->{gbreak}",
487 "WB$d->{wbreak}",
488 "SB$d->{sbreak}",
e5a5a138 489 )."}");
61507e3c
RK
490 }
491 my $t = join(",\n", @t);
492 if(!exists $subtable{$t}) {
e5a5a138 493 out("static const struct unidata st$subtablecounter\[] = {\n",
61507e3c
RK
494 "$t\n",
495 "};\n");
496 $subtable{$t} = $subtablecounter++;
e5a5a138
RK
497 } else {
498 ++$subtablessaved;
61507e3c
RK
499 }
500 $subtableno{$base} = $subtable{$t};
501}
502
bcf9ed7f 503out("const struct unidata *const unidata[]={\n");
e5a5a138 504for(my $base = 0; $base <= $max; $base += $modulus) {
0a4af692 505 #out("st$subtableno{$base} /* ".sprintf("%04x", $base)." */,\n");
e5a5a138 506 out("st$subtableno{$base},\n");
61507e3c
RK
507}
508out("};\n");
509
510close STDOUT or die "unidata.c: $!\n";
511
bcf9ed7f
RK
512printf STDERR "max=%04X\n", $max;
513print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n";
e5a5a138 514print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n";