X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/f98fcddbd5762e3f3a24ad71dca37f7a05fbe3d9..7489d3af05a4394e9621e14a196a9f6c4f788a26:/scripts/make-unidata diff --git a/scripts/make-unidata b/scripts/make-unidata index ce641ed..28532a9 100755 --- a/scripts/make-unidata +++ b/scripts/make-unidata @@ -3,20 +3,18 @@ # This file is part of DisOrder. # Copyright (C) 2007 Richard Kettlewell # -# This program is free software; you can redistribute it and/or modify +# This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 -# USA +# along with this program. If not, see . # # # Generate Unicode support tables @@ -94,7 +92,7 @@ sub input { my $path = shift; my $lpath = basename($path); if(!-e $lpath) { - system("wget http://www.unicode.org/Public/5.0.0/ucd/$path"); + system("wget http://www.unicode.org/Public/6.0.0/ucd/$path"); chmod(0444, $lpath) or die "$lpath: $!\n"; } open(STDIN, "<$lpath") or die "$lpath: $!\n"; @@ -105,6 +103,10 @@ sub input { # Read the main data file input("UnicodeData.txt"); my ($start, $end); +my $maxcompat = 0; +my $maxcanon = 0; +my $hangul_syllable_decomps = 0; +my $hangul_choseong_decomps = 0; while(<>) { my @f = split(/;/, $_); my $c = hex($f[0]); # codepoint @@ -147,12 +149,26 @@ while(<>) { "ld" => $ld, }; if($dm ne '') { + my $maxref; if($dm =~ /\s*//; $d->{compat} = 1; + $maxref = \$maxcompat; + } else { + $maxref = \$maxcanon; + } + $d->{decomp} = [map(hex($_), split(/\s+/, $dm))]; + my $len = scalar @{$d->{decomp}}; + $$maxref = $len if $len > $$maxref; + if(!$d->{compat}) { + if(${$d->{decomp}}[0] >= 0xAC00 && ${$d->{decomp}}[0] <= 0xD7A3) { + ++$hangul_syllable_decomps; + } + if(${$d->{decomp}}[0] >= 0x1100 && ${$d->{decomp}}[0] <= 0x115F) { + ++$hangul_choseong_decomps; + } } - $d->{decomp} = $dm; } $data{$c} = $d; } @@ -171,7 +187,6 @@ sub read_prop_with_ranges { my ($range, $propval) = split(/\s*;\s*/, $_); if($range =~ /(.*)\.\.(.*)/) { for my $c (hex($1) .. hex($2)) { - die "($range)\n" if($c == 0xAC00 and $propkey eq 'gbreak'); $data{$c}->{$propkey} = $propval; } } else { @@ -215,6 +230,27 @@ for my $c (keys %data) { $sbreak{$data{$c}->{sbreak}} = 1; } +# Various derived properties +input("DerivedNormalizationProps.txt"); +while(<>) { + chomp; + s/\s*\#.*//; + next if $_ eq ''; + my @f = split(/\s*;\s*/, $_); + if(@f == 2) { + push(@f, 1); + } + my ($range, $propkey, $propval) = @f; + if($range =~ /(.*)\.\.(.*)/) { + for my $c (hex($1) .. hex($2)) { + $data{$c}->{$propkey} = $propval + } + } else { + my $c = hex($range); + $data{$c}->{$propkey} = $propval + } +} + # Round up the maximum value to a whole number of subtables $max += ($modulus - 1) - ($max % $modulus); @@ -300,7 +336,13 @@ while(<>) { print STDERR "Generating unidata.h...\n"; open(STDOUT, ">unidata.h") or die "unidata.h: $!\n"; -out("/* Automatically generated file, see scripts/make-unidata */\n", +out("/** \@file lib/unidata.h\n", + " * \@brief Unicode tables\n", + " *\n", + " * Automatically generated file, see scripts/make-unidata\n", + " *\n", + " * DO NOT EDIT.\n", + " */\n", "#ifndef UNIDATA_H\n", "#define UNIDATA_H\n"); @@ -350,12 +392,26 @@ sub choosetype { } out("struct unidata {\n", + # decomposition (canonical or compatibility; + # unicode_compatibility_decomposition distinguishes) or NULL " const uint32_t *decomp;\n", + + # case-folded string or NULL " const uint32_t *casefold;\n", + + # composed characters that start with this code point. This only + # includes primary composites, i.e. the decomposition mapping is + # canonical and this code point is not in the exclusion table. + " const uint32_t *composed;\n", + # " ".choosetype($minud, $maxud)." upper_offset;\n", # " ".choosetype($minld, $maxld)." lower_offset;\n", + + # canonical combining class " ".choosetype(0, $maxccc)." ccc;\n", " char general_category;\n", + + # see unicode_flags enum " uint8_t flags;\n", " char grapheme_break;\n", " char word_break;\n", @@ -386,9 +442,14 @@ close STDOUT or die "unidata.h: $!\n"; print STDERR "Generating unidata.c...\n"; open(STDOUT, ">unidata.c") or die "unidata.c: $!\n"; -out("/* Automatically generated file, see scripts/make-unidata */\n", - "#include \n", - "#include \"types.h\"\n", +out("/** \@file lib/unidata.c\n", + " * \@brief Unicode tables\n", + " *\n", + " * Automatically generated file, see scripts/make-unidata\n", + " *\n", + " * DO NOT EDIT.\n", + " */\n", + "#include \"common.h\"\n", "#include \"unidata.h\"\n"); # Short aliases to keep .c file small @@ -401,6 +462,8 @@ out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_), sort keys %wbreak)); out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_), sort keys %sbreak)); +out("#define NBC unicode_normalize_before_casefold\n"); +out("#define CD unicode_compatibility_decomposition\n"); # Names for *_Break properties out("const char *const unicode_Grapheme_Break_names[] = {\n", @@ -416,50 +479,71 @@ out("const char *const unicode_Sentence_Break_names[] = {\n", map(" \"$_\"", sort keys %sbreak)), "\n};\n"); -# Generate the decomposition mapping tables. We look out for duplicates -# in order to save space and report this as decompsaved at the end. In -# Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes. -my $decompnum = 0; -my %decompnums = (); -my $decompsaved = 0; +our $ddnum = 0; +our $ddsaved = 0; +our %ddnums = (); +my $ddfirst = 1; out("static const uint32_t "); +sub dedupe { + my $s = join(",", @_); + if(!exists $ddnums{$s}) { + if($ddfirst) { + $ddfirst = 0; + } else { + out(",\n"); + } + out("dd$ddnum\[]={$s}"); + $ddnums{$s} = $ddnum++; + } else { + ++$ddsaved; + } + return "dd$ddnums{$s}"; +} + +# Generate the decomposition mapping tables. +print STDERR "> decomposition mappings\n"; for(my $c = 0; $c <= $max; ++$c) { if(exists $data{$c} && exists $data{$c}->{decomp}) { - my $s = join(",", - (map(hex($_), split(/\s+/, $data{$c}->{decomp})), 0)); - if(!exists $decompnums{$s}) { - out(",\n") if $decompnum != 0; - out("cd$decompnum\[]={$s}"); - $decompnums{$s} = $decompnum++; + $data{$c}->{decompsym} = dedupe(@{$data{$c}->{decomp}}, 0); + } +} + +print STDERR "> composition mappings\n"; +# First we must generate the mapping of each code point to possible +# compositions. +for(my $c = 0; $c <= $max; ++$c) { + if(exists $data{$c} + && exists $data{$c}->{decomp} + && !exists $data{$c}->{compat} + && !$data{$c}->{Full_Composition_Exclusion}) { + # $c has a non-excluded canonical decomposition, i.e. it is + # a primary composite. Find the first code point of the decomposition + my $first = ${$data{$c}->{decomp}}[0]; + if(!exists $data{$first}->{compose}) { + $data{$first}->{compose} = [$c]; } else { - ++$decompsaved; + push(@{$data{$first}->{compose}}, $c); } - $data{$c}->{decompsym} = "cd$decompnums{$s}"; } } -out(";\n"); +# Then we can generate the tables. +for(my $c = 0; $c <= $max; ++$c) { + if(exists $data{$c} && exists $data{$c}->{compose}) { + $data{$c}->{compsym} = dedupe(@{$data{$c}->{compose}}, 0); + } +} -# ...and the case folding table. Again we compress equal entries to save -# space. In Unicode 5.0.0 this saves 51 entries or at least 408 bytes. -# This doesns't seem as worthwhile as the decomposition mapping saving above. -my $cfnum = 0; -my %cfnums = (); -my $cfsaved = 0; -out("static const uint32_t "); +# The case folding table. +print STDERR "> case-fold mappings\n"; for(my $c = 0; $c <= $max; ++$c) { if(exists $data{$c} && exists $data{$c}->{casefold}) { - my $s = join(",", - (map(hex($_), split(/\s+/, $data{$c}->{casefold})), 0)); - if(!exists $cfnums{$s}) { - out(",\n") if $cfnum != 0; - out("cf$cfnum\[]={$s}"); - $cfnums{$s} = $cfnum++; - } else { - ++$cfsaved; - } - $data{$c}->{cfsym} = "cf$cfnums{$s}"; + $data{$c}->{cfsym} = dedupe(map(hex($_), split(/\s+/, + $data{$c}->{casefold})), + 0); } } + +# End of de-dupable arrays out(";\n"); # Visit all the $modulus-character blocks in turn and generate the @@ -467,7 +551,7 @@ out(";\n"); # Unicode 5.0.0 with $modulus=128 and current table data this saves # 1372 subtables or at least three and a half megabytes on 32-bit # platforms. - +print STDERR "> subtables\n"; my %subtable = (); # base->subtable number my %subtableno = (); # subtable number -> content my $subtablecounter = 0; # counter for subtable numbers @@ -478,24 +562,28 @@ for(my $base = 0; $base <= $max; $base += $modulus) { my @t; for(my $c = $base; $c < $base + $modulus; ++$c) { my $d = $data{$c}; - my $decompsym = ($data{$c}->{decompsym} or "0"); - my $cfsym = ($data{$c}->{cfsym} or "0"); + my $decompsym = ($d->{decompsym} or "0"); + my $cfsym = ($d->{cfsym} or "0"); + my $compsym = ($d->{compsym} or "0"); + my $ccc = ($d->{ccc} or "0"); + my $gc = ($d->{gc} or "Cn"); my @flags = (); if($data{$c}->{ypogegrammeni}) { - push(@flags, "unicode_normalize_before_casefold"); + push(@flags, "NBC"); } if($data{$c}->{compat}) { - push(@flags, "unicode_compatibility_decomposition"); + push(@flags, "CD"); } my $flags = @flags ? join("|", @flags) : 0; - push(@t, "{". + push(@t, "{". join(",", $decompsym, $cfsym, + $compsym, # $d->{ud}, # $d->{ld}, - $d->{ccc}, - $d->{gc}, + $ccc, + $gc, $flags, "GB$d->{gbreak}", "WB$d->{wbreak}", @@ -515,6 +603,7 @@ for(my $base = 0; $base <= $max; $base += $modulus) { $subtableno{$base} = $subtable{$t}; } +print STDERR "> main table\n"; out("const struct unidata *const unidata[]={\n"); for(my $base = 0; $base <= $max; $base += $modulus) { next if $base >= $break_start && $base < $break_end; @@ -524,6 +613,7 @@ for(my $base = 0; $base <= $max; $base += $modulus) { } out("};\n"); +print STDERR "> UTF-8 table\n"; out("const struct unicode_utf8_row unicode_utf8_valid[] = {\n"); for(my $c = 0; $c <= 0x7F; ++$c) { out(" { 1, 0, 0 }, /* $c */\n"); @@ -562,7 +652,17 @@ out("};\n"); close STDOUT or die "unidata.c: $!\n"; +print STDERR "Done.\n\n"; printf STDERR "modulus=%d\n", $modulus; printf STDERR "max=%04X\n", $max; print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n"; -print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n"; +print STDERR "ddsaved=$ddsaved\n"; +print STDERR "maxcompat=$maxcompat maxcanon=$maxcanon\n"; +print STDERR "$hangul_syllable_decomps canonical decompositions to Hangul syllables\n"; +print STDERR "$hangul_choseong_decomps canonical decompositions to Hangul Choseong\n"; + +die "We assumed that canonical decompositions were never more than 2 long!\n" + if $maxcanon > 2; + +die "We assumed no canonical decompositions to Hangul syllables/Choseong!\n" + if $hangul_syllable_decomps || $hangul_choseong_decomps;