X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/bcf9ed7f5b44c177d927d147f87c5c08e377adfa..f98fcddbd5762e3f3a24ad71dca37f7a05fbe3d9:/scripts/make-unidata diff --git a/scripts/make-unidata b/scripts/make-unidata index b78d289..ce641ed 100755 --- a/scripts/make-unidata +++ b/scripts/make-unidata @@ -31,7 +31,6 @@ # - SpecialCasing.txt data for case mapping # - Title case offsets # - Some kind of hinting for composition -# - Word boundary support # - ... # # NB the generated files DO NOT offer a stable ABI and so are not immediately @@ -60,7 +59,24 @@ sub key { # Size of a subtable # # This can be varied to trade off the number of subtables against their size. -our $modulus = 128; +# 16 gave the smallest results last time I checked (on a Mac with a 32-bit +# build). +our $modulus = 16; + +if(@ARGV) { + $modulus = shift; +} + +# Where to break the table. There is a huge empty section of the Unicode +# code space and we deal with this by simply leaving it out of the table. +# This complicates the lookup function a little but should not affect +# performance in the cases we care about. +our $break_start = 0x30000; +our $break_end = 0xE0000; + +# Similarly we simply omit the very top of the table and sort it out in the +# lookup function. +our $break_top = 0xE0200; my %cats = (); # known general categories my %data = (); # mapping of codepoints to information @@ -88,19 +104,25 @@ sub input { # Read the main data file input("UnicodeData.txt"); +my ($start, $end); while(<>) { my @f = split(/;/, $_); my $c = hex($f[0]); # codepoint - next if $c >= 0xE0000; # ignore various high-numbered stuff - # TODO justify this exclusion! my $name = $f[1]; + die "$f[0] $name is in the break\n" + if $c >= $break_start && $c < $break_end; my $gc = $f[2]; # General_Category # Variuos GCs we don't expect to see in UnicodeData.txt $cats{$gc} = 1; # always record all GCs - next if $name =~ /(first|last)>/i; # ignore placeholders + if($name =~ /first>/i) { + $start = $c; + next; + } elsif($name =~ /last>/i) { + $end = $c; + } else { + $start = $end = $c; + } die "unexpected Cn" if $gc eq 'Cn'; - die "unexpected Co" if $gc eq 'Cn'; - die "unexpected Cs" if $gc eq 'Cs'; my $ccc = $f[3]; # Canonical_Combining_Class my $dm = $f[5]; # Decomposition_Type + Decomposition_Mapping my $sum = hex($f[12]) || $c; # Simple_Uppercase_Mapping @@ -114,25 +136,28 @@ while(<>) { $maxud = $ud if $ud > $maxud; $minld = $ld if $ld < $minld; $maxld = $ld if $ld > $maxld; - $data{$c} = { - "gc" => $gc, - "ccc" => $ccc, - "ud" => $ud, - "ld" => $ld, + if($start != $end) { + printf STDERR "> range %04X-%04X is %s\n", $start, $end, $gc; + } + for($c = $start; $c <= $end; ++$c) { + my $d = { + "gc" => $gc, + "ccc" => $ccc, + "ud" => $ud, + "ld" => $ld, }; - if($dm ne '') { - if($dm !~ /{canon} = $dm; - $data{$c}->{compat} = $dm; - } else { - # This is only a compatibility decomposition - $dm =~ s/^<.*>\s*//; - $data{$c}->{compat} = $dm; + if($dm ne '') { + if($dm =~ /\s*//; + $d->{compat} = 1; + } + $d->{decomp} = $dm; } + $data{$c} = $d; } $cats{$gc} = 1; - $max = $c if $c > $max; + $max = $end if $end > $max; } sub read_prop_with_ranges { @@ -146,22 +171,17 @@ sub read_prop_with_ranges { my ($range, $propval) = split(/\s*;\s*/, $_); if($range =~ /(.*)\.\.(.*)/) { for my $c (hex($1) .. hex($2)) { - if(exists $data{$c}) { - $data{$c}->{$propkey} = $propval; - } + die "($range)\n" if($c == 0xAC00 and $propkey eq 'gbreak'); + $data{$c}->{$propkey} = $propval; } } else { my $c = hex($range); - if(exists $data{$c}) { - $data{$c}->{$propkey} = $propval; - } + $data{$c}->{$propkey} = $propval; } } } # Grapheme_Break etc -# NB we do this BEFORE filling in blanks so that the Hangul characters -# don't get filled in; we can compute their properties mechanically. read_prop_with_ranges("auxiliary/GraphemeBreakProperty.txt", "gbreak"); read_prop_with_ranges("auxiliary/WordBreakProperty.txt", "wbreak"); read_prop_with_ranges("auxiliary/SentenceBreakProperty.txt", "sbreak"); @@ -198,17 +218,6 @@ for my $c (keys %data) { # Round up the maximum value to a whole number of subtables $max += ($modulus - 1) - ($max % $modulus); -# Surrogates -my $Cs = { - "gc" => "Cs", # UTF-16 surrogate - "ccc" => 0, - "ud" => 0, - "ld" => 0 -}; -for(my $c = 0xD800; $c <= 0xDFFF; ++$c) { - $data{$c} = $Cs; -} - # Private use characters # We only fill in values below $max, utf32__unidata() my $Co = { @@ -319,7 +328,8 @@ out("enum unicode_Sentence_Break {\n", out("extern const char *const unicode_Sentence_Break_names[];\n"); out("enum unicode_flags {\n", - " unicode_normalize_before_casefold = 1\n", + " unicode_normalize_before_casefold = 1,\n", + " unicode_compatibility_decomposition = 2\n", "};\n", "\n"); @@ -340,11 +350,10 @@ sub choosetype { } out("struct unidata {\n", - " const uint32_t *compat;\n", - " const uint32_t *canon;\n", + " const uint32_t *decomp;\n", " const uint32_t *casefold;\n", - " ".choosetype($minud, $maxud)." upper_offset;\n", - " ".choosetype($minld, $maxld)." lower_offset;\n", +# " ".choosetype($minud, $maxud)." upper_offset;\n", +# " ".choosetype($minld, $maxld)." lower_offset;\n", " ".choosetype(0, $maxccc)." ccc;\n", " char general_category;\n", " uint8_t flags;\n", @@ -352,15 +361,23 @@ out("struct unidata {\n", " char word_break;\n", " char sentence_break;\n", "};\n"); -# compat, canon and casefold do have have non-BMP characters, so we +# decomp and casefold do have have non-BMP characters, so we # can't use a simple 16-bit table. We could use UTF-8 or UTF-16 # though, saving a bit of space (probably not that much...) at the # cost of marginally reduced performance and additional complexity out("extern const struct unidata *const unidata[];\n"); +out("extern const struct unicode_utf8_row {\n", + " uint8_t count;\n", + " uint8_t min2, max2;\n", + "} unicode_utf8_valid[];\n"); + out("#define UNICODE_NCHARS ", ($max + 1), "\n"); out("#define UNICODE_MODULUS $modulus\n"); +out("#define UNICODE_BREAK_START $break_start\n"); +out("#define UNICODE_BREAK_END $break_end\n"); +out("#define UNICODE_BREAK_TOP $break_top\n"); out("#endif\n"); @@ -407,12 +424,9 @@ my %decompnums = (); my $decompsaved = 0; out("static const uint32_t "); for(my $c = 0; $c <= $max; ++$c) { - # If canon is set then compat will be too and will be identical. - # If compat is set the canon might be clear. So we use the - # compat version and fix up the symbols after. - if(exists $data{$c} && exists $data{$c}->{compat}) { + if(exists $data{$c} && exists $data{$c}->{decomp}) { my $s = join(",", - (map(hex($_), split(/\s+/, $data{$c}->{compat})), 0)); + (map(hex($_), split(/\s+/, $data{$c}->{decomp})), 0)); if(!exists $decompnums{$s}) { out(",\n") if $decompnum != 0; out("cd$decompnum\[]={$s}"); @@ -420,10 +434,7 @@ for(my $c = 0; $c <= $max; ++$c) { } else { ++$decompsaved; } - $data{$c}->{compatsym} = "cd$decompnums{$s}"; - if(exists $data{$c}->{canon}) { - $data{$c}->{canonsym} = "cd$decompnums{$s}"; - } + $data{$c}->{decompsym} = "cd$decompnums{$s}"; } } out(";\n"); @@ -462,24 +473,27 @@ my %subtableno = (); # subtable number -> content my $subtablecounter = 0; # counter for subtable numbers my $subtablessaved = 0; # number of tables saved for(my $base = 0; $base <= $max; $base += $modulus) { + next if $base >= $break_start && $base < $break_end; + next if $base >= $break_top; my @t; for(my $c = $base; $c < $base + $modulus; ++$c) { my $d = $data{$c}; - my $canonsym = ($data{$c}->{canonsym} or "0"); - my $compatsym = ($data{$c}->{compatsym} or "0"); + my $decompsym = ($data{$c}->{decompsym} or "0"); my $cfsym = ($data{$c}->{cfsym} or "0"); my @flags = (); if($data{$c}->{ypogegrammeni}) { push(@flags, "unicode_normalize_before_casefold"); } + if($data{$c}->{compat}) { + push(@flags, "unicode_compatibility_decomposition"); + } my $flags = @flags ? join("|", @flags) : 0; push(@t, "{". join(",", - $compatsym, - $canonsym, + $decompsym, $cfsym, - $d->{ud}, - $d->{ld}, +# $d->{ud}, +# $d->{ld}, $d->{ccc}, $d->{gc}, $flags, @@ -490,6 +504,7 @@ for(my $base = 0; $base <= $max; $base += $modulus) { } my $t = join(",\n", @t); if(!exists $subtable{$t}) { + out(sprintf("/* %04X-%04X */\n", $base, $base + $modulus - 1)); out("static const struct unidata st$subtablecounter\[] = {\n", "$t\n", "};\n"); @@ -502,13 +517,52 @@ for(my $base = 0; $base <= $max; $base += $modulus) { out("const struct unidata *const unidata[]={\n"); for(my $base = 0; $base <= $max; $base += $modulus) { + next if $base >= $break_start && $base < $break_end; + next if $base >= $break_top; #out("st$subtableno{$base} /* ".sprintf("%04x", $base)." */,\n"); out("st$subtableno{$base},\n"); } out("};\n"); +out("const struct unicode_utf8_row unicode_utf8_valid[] = {\n"); +for(my $c = 0; $c <= 0x7F; ++$c) { + out(" { 1, 0, 0 }, /* $c */\n"); +} +for(my $c = 0x80; $c < 0xC2; ++$c) { + out(" { 0, 0, 0 }, /* $c */\n"); +} +for(my $c = 0xC2; $c <= 0xDF; ++$c) { + out(" { 2, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xE0; $c <= 0xE0; ++$c) { + out(" { 3, 0xA0, 0xBF }, /* $c */\n"); +} +for(my $c = 0xE1; $c <= 0xEC; ++$c) { + out(" { 3, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xED; $c <= 0xED; ++$c) { + out(" { 3, 0x80, 0x9F }, /* $c */\n"); +} +for(my $c = 0xEE; $c <= 0xEF; ++$c) { + out(" { 3, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xF0; $c <= 0xF0; ++$c) { + out(" { 4, 0x90, 0xBF }, /* $c */\n"); +} +for(my $c = 0xF1; $c <= 0xF3; ++$c) { + out(" { 4, 0x80, 0xBF }, /* $c */\n"); +} +for(my $c = 0xF4; $c <= 0xF4; ++$c) { + out(" { 4, 0x80, 0x8F }, /* $c */\n"); +} +for(my $c = 0xF5; $c <= 0xFF; ++$c) { + out(" { 0, 0, 0 }, /* $c */\n"); +} +out("};\n"); + close STDOUT or die "unidata.c: $!\n"; +printf STDERR "modulus=%d\n", $modulus; printf STDERR "max=%04X\n", $max; print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n"; print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n";