X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/c2e01e0ac91c61de69158113d2ab59f6831b2941..7489d3af05a4394e9621e14a196a9f6c4f788a26:/scripts/make-unidata diff --git a/scripts/make-unidata b/scripts/make-unidata index 9fa0dec..28532a9 100755 --- a/scripts/make-unidata +++ b/scripts/make-unidata @@ -3,20 +3,18 @@ # This file is part of DisOrder. # Copyright (C) 2007 Richard Kettlewell # -# This program is free software; you can redistribute it and/or modify +# This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 -# USA +# along with this program. If not, see . # # # Generate Unicode support tables @@ -94,7 +92,7 @@ sub input { my $path = shift; my $lpath = basename($path); if(!-e $lpath) { - system("wget http://www.unicode.org/Public/5.0.0/ucd/$path"); + system("wget http://www.unicode.org/Public/6.0.0/ucd/$path"); chmod(0444, $lpath) or die "$lpath: $!\n"; } open(STDIN, "<$lpath") or die "$lpath: $!\n"; @@ -105,6 +103,10 @@ sub input { # Read the main data file input("UnicodeData.txt"); my ($start, $end); +my $maxcompat = 0; +my $maxcanon = 0; +my $hangul_syllable_decomps = 0; +my $hangul_choseong_decomps = 0; while(<>) { my @f = split(/;/, $_); my $c = hex($f[0]); # codepoint @@ -147,14 +149,25 @@ while(<>) { "ld" => $ld, }; if($dm ne '') { - if($dm !~ /{canon} = $dm; - $d->{compat} = $dm; - } else { - # This is only a compatibility decomposition + my $maxref; + if($dm =~ /\s*//; - $d->{compat} = $dm; + $d->{compat} = 1; + $maxref = \$maxcompat; + } else { + $maxref = \$maxcanon; + } + $d->{decomp} = [map(hex($_), split(/\s+/, $dm))]; + my $len = scalar @{$d->{decomp}}; + $$maxref = $len if $len > $$maxref; + if(!$d->{compat}) { + if(${$d->{decomp}}[0] >= 0xAC00 && ${$d->{decomp}}[0] <= 0xD7A3) { + ++$hangul_syllable_decomps; + } + if(${$d->{decomp}}[0] >= 0x1100 && ${$d->{decomp}}[0] <= 0x115F) { + ++$hangul_choseong_decomps; + } } } $data{$c} = $d; @@ -174,7 +187,6 @@ sub read_prop_with_ranges { my ($range, $propval) = split(/\s*;\s*/, $_); if($range =~ /(.*)\.\.(.*)/) { for my $c (hex($1) .. hex($2)) { - die "($range)\n" if($c == 0xAC00 and $propkey eq 'gbreak'); $data{$c}->{$propkey} = $propval; } } else { @@ -218,6 +230,27 @@ for my $c (keys %data) { $sbreak{$data{$c}->{sbreak}} = 1; } +# Various derived properties +input("DerivedNormalizationProps.txt"); +while(<>) { + chomp; + s/\s*\#.*//; + next if $_ eq ''; + my @f = split(/\s*;\s*/, $_); + if(@f == 2) { + push(@f, 1); + } + my ($range, $propkey, $propval) = @f; + if($range =~ /(.*)\.\.(.*)/) { + for my $c (hex($1) .. hex($2)) { + $data{$c}->{$propkey} = $propval + } + } else { + my $c = hex($range); + $data{$c}->{$propkey} = $propval + } +} + # Round up the maximum value to a whole number of subtables $max += ($modulus - 1) - ($max % $modulus); @@ -303,7 +336,13 @@ while(<>) { print STDERR "Generating unidata.h...\n"; open(STDOUT, ">unidata.h") or die "unidata.h: $!\n"; -out("/* Automatically generated file, see scripts/make-unidata */\n", +out("/** \@file lib/unidata.h\n", + " * \@brief Unicode tables\n", + " *\n", + " * Automatically generated file, see scripts/make-unidata\n", + " *\n", + " * DO NOT EDIT.\n", + " */\n", "#ifndef UNIDATA_H\n", "#define UNIDATA_H\n"); @@ -331,7 +370,8 @@ out("enum unicode_Sentence_Break {\n", out("extern const char *const unicode_Sentence_Break_names[];\n"); out("enum unicode_flags {\n", - " unicode_normalize_before_casefold = 1\n", + " unicode_normalize_before_casefold = 1,\n", + " unicode_compatibility_decomposition = 2\n", "};\n", "\n"); @@ -352,25 +392,43 @@ sub choosetype { } out("struct unidata {\n", - " const uint32_t *compat;\n", - " const uint32_t *canon;\n", + # decomposition (canonical or compatibility; + # unicode_compatibility_decomposition distinguishes) or NULL + " const uint32_t *decomp;\n", + + # case-folded string or NULL " const uint32_t *casefold;\n", + + # composed characters that start with this code point. This only + # includes primary composites, i.e. the decomposition mapping is + # canonical and this code point is not in the exclusion table. + " const uint32_t *composed;\n", + # " ".choosetype($minud, $maxud)." upper_offset;\n", # " ".choosetype($minld, $maxld)." lower_offset;\n", + + # canonical combining class " ".choosetype(0, $maxccc)." ccc;\n", " char general_category;\n", + + # see unicode_flags enum " uint8_t flags;\n", " char grapheme_break;\n", " char word_break;\n", " char sentence_break;\n", "};\n"); -# compat, canon and casefold do have have non-BMP characters, so we +# decomp and casefold do have have non-BMP characters, so we # can't use a simple 16-bit table. We could use UTF-8 or UTF-16 # though, saving a bit of space (probably not that much...) at the # cost of marginally reduced performance and additional complexity out("extern const struct unidata *const unidata[];\n"); +out("extern const struct unicode_utf8_row {\n", + " uint8_t count;\n", + " uint8_t min2, max2;\n", + "} unicode_utf8_valid[];\n"); + out("#define UNICODE_NCHARS ", ($max + 1), "\n"); out("#define UNICODE_MODULUS $modulus\n"); out("#define UNICODE_BREAK_START $break_start\n"); @@ -384,9 +442,14 @@ close STDOUT or die "unidata.h: $!\n"; print STDERR "Generating unidata.c...\n"; open(STDOUT, ">unidata.c") or die "unidata.c: $!\n"; -out("/* Automatically generated file, see scripts/make-unidata */\n", - "#include \n", - "#include \"types.h\"\n", +out("/** \@file lib/unidata.c\n", + " * \@brief Unicode tables\n", + " *\n", + " * Automatically generated file, see scripts/make-unidata\n", + " *\n", + " * DO NOT EDIT.\n", + " */\n", + "#include \"common.h\"\n", "#include \"unidata.h\"\n"); # Short aliases to keep .c file small @@ -399,6 +462,8 @@ out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_), sort keys %wbreak)); out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_), sort keys %sbreak)); +out("#define NBC unicode_normalize_before_casefold\n"); +out("#define CD unicode_compatibility_decomposition\n"); # Names for *_Break properties out("const char *const unicode_Grapheme_Break_names[] = {\n", @@ -414,56 +479,71 @@ out("const char *const unicode_Sentence_Break_names[] = {\n", map(" \"$_\"", sort keys %sbreak)), "\n};\n"); -# Generate the decomposition mapping tables. We look out for duplicates -# in order to save space and report this as decompsaved at the end. In -# Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes. -my $decompnum = 0; -my %decompnums = (); -my $decompsaved = 0; +our $ddnum = 0; +our $ddsaved = 0; +our %ddnums = (); +my $ddfirst = 1; out("static const uint32_t "); -for(my $c = 0; $c <= $max; ++$c) { - # If canon is set then compat will be too and will be identical. - # If compat is set the canon might be clear. So we use the - # compat version and fix up the symbols after. - if(exists $data{$c} && exists $data{$c}->{compat}) { - my $s = join(",", - (map(hex($_), split(/\s+/, $data{$c}->{compat})), 0)); - if(!exists $decompnums{$s}) { - out(",\n") if $decompnum != 0; - out("cd$decompnum\[]={$s}"); - $decompnums{$s} = $decompnum++; +sub dedupe { + my $s = join(",", @_); + if(!exists $ddnums{$s}) { + if($ddfirst) { + $ddfirst = 0; } else { - ++$decompsaved; - } - $data{$c}->{compatsym} = "cd$decompnums{$s}"; - if(exists $data{$c}->{canon}) { - $data{$c}->{canonsym} = "cd$decompnums{$s}"; + out(",\n"); } + out("dd$ddnum\[]={$s}"); + $ddnums{$s} = $ddnum++; + } else { + ++$ddsaved; } + return "dd$ddnums{$s}"; } -out(";\n"); -# ...and the case folding table. Again we compress equal entries to save -# space. In Unicode 5.0.0 this saves 51 entries or at least 408 bytes. -# This doesns't seem as worthwhile as the decomposition mapping saving above. -my $cfnum = 0; -my %cfnums = (); -my $cfsaved = 0; -out("static const uint32_t "); +# Generate the decomposition mapping tables. +print STDERR "> decomposition mappings\n"; for(my $c = 0; $c <= $max; ++$c) { - if(exists $data{$c} && exists $data{$c}->{casefold}) { - my $s = join(",", - (map(hex($_), split(/\s+/, $data{$c}->{casefold})), 0)); - if(!exists $cfnums{$s}) { - out(",\n") if $cfnum != 0; - out("cf$cfnum\[]={$s}"); - $cfnums{$s} = $cfnum++; + if(exists $data{$c} && exists $data{$c}->{decomp}) { + $data{$c}->{decompsym} = dedupe(@{$data{$c}->{decomp}}, 0); + } +} + +print STDERR "> composition mappings\n"; +# First we must generate the mapping of each code point to possible +# compositions. +for(my $c = 0; $c <= $max; ++$c) { + if(exists $data{$c} + && exists $data{$c}->{decomp} + && !exists $data{$c}->{compat} + && !$data{$c}->{Full_Composition_Exclusion}) { + # $c has a non-excluded canonical decomposition, i.e. it is + # a primary composite. Find the first code point of the decomposition + my $first = ${$data{$c}->{decomp}}[0]; + if(!exists $data{$first}->{compose}) { + $data{$first}->{compose} = [$c]; } else { - ++$cfsaved; + push(@{$data{$first}->{compose}}, $c); } - $data{$c}->{cfsym} = "cf$cfnums{$s}"; } } +# Then we can generate the tables. +for(my $c = 0; $c <= $max; ++$c) { + if(exists $data{$c} && exists $data{$c}->{compose}) { + $data{$c}->{compsym} = dedupe(@{$data{$c}->{compose}}, 0); + } +} + +# The case folding table. +print STDERR "> case-fold mappings\n"; +for(my $c = 0; $c <= $max; ++$c) { + if(exists $data{$c} && exists $data{$c}->{casefold}) { + $data{$c}->{cfsym} = dedupe(map(hex($_), split(/\s+/, + $data{$c}->{casefold})), + 0); + } +} + +# End of de-dupable arrays out(";\n"); # Visit all the $modulus-character blocks in turn and generate the @@ -471,7 +551,7 @@ out(";\n"); # Unicode 5.0.0 with $modulus=128 and current table data this saves # 1372 subtables or at least three and a half megabytes on 32-bit # platforms. - +print STDERR "> subtables\n"; my %subtable = (); # base->subtable number my %subtableno = (); # subtable number -> content my $subtablecounter = 0; # counter for subtable numbers @@ -482,23 +562,28 @@ for(my $base = 0; $base <= $max; $base += $modulus) { my @t; for(my $c = $base; $c < $base + $modulus; ++$c) { my $d = $data{$c}; - my $canonsym = ($data{$c}->{canonsym} or "0"); - my $compatsym = ($data{$c}->{compatsym} or "0"); - my $cfsym = ($data{$c}->{cfsym} or "0"); + my $decompsym = ($d->{decompsym} or "0"); + my $cfsym = ($d->{cfsym} or "0"); + my $compsym = ($d->{compsym} or "0"); + my $ccc = ($d->{ccc} or "0"); + my $gc = ($d->{gc} or "Cn"); my @flags = (); if($data{$c}->{ypogegrammeni}) { - push(@flags, "unicode_normalize_before_casefold"); + push(@flags, "NBC"); + } + if($data{$c}->{compat}) { + push(@flags, "CD"); } my $flags = @flags ? join("|", @flags) : 0; - push(@t, "{". + push(@t, "{". join(",", - $compatsym, - $canonsym, + $decompsym, $cfsym, + $compsym, # $d->{ud}, # $d->{ld}, - $d->{ccc}, - $d->{gc}, + $ccc, + $gc, $flags, "GB$d->{gbreak}", "WB$d->{wbreak}", @@ -518,6 +603,7 @@ for(my $base = 0; $base <= $max; $base += $modulus) { $subtableno{$base} = $subtable{$t}; } +print STDERR "> main table\n"; out("const struct unidata *const unidata[]={\n"); for(my $base = 0; $base <= $max; $base += $modulus) { next if $base >= $break_start && $base < $break_end; @@ -527,9 +613,56 @@ for(my $base = 0; $base <= $max; $base += $modulus) { } out("};\n"); +print STDERR "> UTF-8 table\n"; +out("const struct unicode_utf8_row unicode_utf8_valid[] = {\n"); +for(my $c = 0; $c <= 0x7F; ++$c) { + out(" { 1, 0, 0 }, /* $c */\n"); +} +for(my $c = 0x80; $c < 0xC2; ++$c) { + out(" { 0, 0, 0 }, /* $c */\n"); +} +for(my $c = 0xC2; $c <= 0xDF; ++$c) { + out(" { 2, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xE0; $c <= 0xE0; ++$c) { + out(" { 3, 0xA0, 0xBF }, /* $c */\n"); +} +for(my $c = 0xE1; $c <= 0xEC; ++$c) { + out(" { 3, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xED; $c <= 0xED; ++$c) { + out(" { 3, 0x80, 0x9F }, /* $c */\n"); +} +for(my $c = 0xEE; $c <= 0xEF; ++$c) { + out(" { 3, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xF0; $c <= 0xF0; ++$c) { + out(" { 4, 0x90, 0xBF }, /* $c */\n"); +} +for(my $c = 0xF1; $c <= 0xF3; ++$c) { + out(" { 4, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xF4; $c <= 0xF4; ++$c) { + out(" { 4, 0x80, 0x8F }, /* $c */\n"); +} +for(my $c = 0xF5; $c <= 0xFF; ++$c) { + out(" { 0, 0, 0 }, /* $c */\n"); +} +out("};\n"); + close STDOUT or die "unidata.c: $!\n"; +print STDERR "Done.\n\n"; printf STDERR "modulus=%d\n", $modulus; printf STDERR "max=%04X\n", $max; print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n"; -print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n"; +print STDERR "ddsaved=$ddsaved\n"; +print STDERR "maxcompat=$maxcompat maxcanon=$maxcanon\n"; +print STDERR "$hangul_syllable_decomps canonical decompositions to Hangul syllables\n"; +print STDERR "$hangul_choseong_decomps canonical decompositions to Hangul Choseong\n"; + +die "We assumed that canonical decompositions were never more than 2 long!\n" + if $maxcanon > 2; + +die "We assumed no canonical decompositions to Hangul syllables/Choseong!\n" + if $hangul_syllable_decomps || $hangul_choseong_decomps;