start on ut32__unidata() which will provide a uniform interface

[disorder] / scripts / make-unidata

diff --git a/scripts/make-unidata b/scripts/make-unidata
index bbb4aff995ac0148f1f25224efe5019581d36c92..b78d289fb593f0143a9e7550aa61c19e3ee964fc 100755 (executable)
--- a/scripts/make-unidata
+++ b/scripts/make-unidata
@@ -82,6 +82,7 @@ sub input {
        chmod(0444, $lpath) or die "$lpath: $!\n";
     }
     open(STDIN, "<$lpath") or die "$lpath: $!\n";
+    print STDERR "Reading $lpath...\n";
 }
 
 
@@ -94,6 +95,12 @@ while(<>) {
     # TODO justify this exclusion!
     my $name = $f[1];
     my $gc = $f[2];            # General_Category
+    # Variuos GCs we don't expect to see in UnicodeData.txt
+    $cats{$gc} = 1;            # always record all GCs
+    next if $name =~ /(first|last)>/i; # ignore placeholders
+    die "unexpected Cn" if $gc eq 'Cn';
+    die "unexpected Co" if $gc eq 'Cn';
+    die "unexpected Cs" if $gc eq 'Cs';
     my $ccc = $f[3];           # Canonical_Combining_Class
     my $dm = $f[5];            # Decomposition_Type + Decomposition_Mapping
     my $sum = hex($f[12]) || $c; # Simple_Uppercase_Mapping
@@ -191,18 +198,54 @@ for my $c (keys %data) {
 # Round up the maximum value to a whole number of subtables
 $max += ($modulus - 1) - ($max % $modulus);
 
-# Make sure there are no gaps
+# Surrogates
+my $Cs = {
+    "gc" => "Cs",              # UTF-16 surrogate
+    "ccc" => 0,
+    "ud" => 0,
+    "ld" => 0
+};
+for(my $c = 0xD800; $c <= 0xDFFF; ++$c) {
+    $data{$c} = $Cs;
+}
+
+# Private use characters
+# We only fill in values below $max, utf32__unidata() 
+my $Co = {
+    "gc" => "Co",
+    "ccc" => 0,
+    "ud" => 0,
+    "ld" => 0
+};
+for(my $c = 0xE000; $c <= 0xF8FF && $c <= $max; ++$c) {
+    $data{$c} = $Co;
+}
+for(my $c = 0xF0000; $c <= 0xFFFFD && $c <= $max; ++$c) {
+    $data{$c} = $Co;
+}
+for(my $c = 0x100000; $c <= 0x10FFFD && $c <= $max; ++$c) {
+    $data{$c} = $Co;
+}
+
+# Anything left is not assigned
+my $Cn = {
+    "gc" => "Cn",              # not assigned
+    "ccc" => 0,
+    "ud" => 0,
+    "ld" => 0
+};
 for(my $c = 0; $c <= $max; ++$c) {
     if(!exists $data{$c}) {
-       $data{$c} = {
-           "gc" => "Cn",       # not assigned
-           "ccc" => 0,
-           "ud" => 0,
-           "ld" => 0,
-           "wbreak" => 'Other',
-           "gbreak" => 'Other',
-           "sbreak" => 'Other',
-           };
+       $data{$c} = $Cn;
+    }
+    if(!exists $data{$c}->{wbreak}) {
+       $data{$c}->{wbreak} = 'Other';
+    }
+    if(!exists $data{$c}->{gbreak}) {
+       $data{$c}->{gbreak} = 'Other';
+    }
+    if(!exists $data{$c}->{sbreak}) {
+       $data{$c}->{sbreak} = 'Other';
     }
 }
 $cats{'Cn'} = 1;
@@ -245,6 +288,7 @@ while(<>) {
 }
 
 # Generate the header file
+print STDERR "Generating unidata.h...\n";
 open(STDOUT, ">unidata.h") or die "unidata.h: $!\n";
 
 out("/* Automatically generated file, see scripts/make-unidata */\n",
@@ -252,9 +296,9 @@ out("/* Automatically generated file, see scripts/make-unidata */\n",
     "#define UNIDATA_H\n");
 
 # TODO choose stable values for General_Category
-out("enum unicode_gc_cat {\n",
+out("enum unicode_General_Category {\n",
     join(",\n",
-        map("  unicode_gc_$_", sort keys %cats)), "\n};\n");
+        map("  unicode_General_Category_$_", sort keys %cats)), "\n};\n");
 
 out("enum unicode_Grapheme_Break {\n",
     join(",\n",
@@ -302,7 +346,7 @@ out("struct unidata {\n",
     "  ".choosetype($minud, $maxud)." upper_offset;\n",
     "  ".choosetype($minld, $maxld)." lower_offset;\n",
     "  ".choosetype(0, $maxccc)." ccc;\n",
-    "  char gc;\n",
+    "  char general_category;\n",
     "  uint8_t flags;\n",
     "  char grapheme_break;\n",
     "  char word_break;\n",
@@ -322,6 +366,7 @@ out("#endif\n");
 
 close STDOUT or die "unidata.h: $!\n";
 
+print STDERR "Generating unidata.c...\n";
 open(STDOUT, ">unidata.c") or die "unidata.c: $!\n";
 
 out("/* Automatically generated file, see scripts/make-unidata */\n",
@@ -331,10 +376,14 @@ out("/* Automatically generated file, see scripts/make-unidata */\n",
 
 # Short aliases to keep .c file small
 
-out(map(sprintf("#define %s unicode_gc_%s\n", $_, $_), sort keys %cats));
-out(map(sprintf("#define GB%s unicode_Grapheme_Break_%s\n", $_, $_), sort keys %gbreak));
-out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_), sort keys %wbreak));
-out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_), sort keys %sbreak));
+out(map(sprintf("#define %s unicode_General_Category_%s\n", $_, $_),
+       sort keys %cats));
+out(map(sprintf("#define GB%s unicode_Grapheme_Break_%s\n", $_, $_),
+       sort keys %gbreak));
+out(map(sprintf("#define WB%s unicode_Word_Break_%s\n", $_, $_),
+       sort keys %wbreak));
+out(map(sprintf("#define SB%s unicode_Sentence_Break_%s\n", $_, $_),
+       sort keys %sbreak));
 
 # Names for *_Break properties
 out("const char *const unicode_Grapheme_Break_names[] = {\n",
@@ -361,7 +410,7 @@ for(my $c = 0; $c <= $max; ++$c) {
     # If canon is set then compat will be too and will be identical.
     # If compat is set the canon might be clear.  So we use the
     # compat version and fix up the symbols after.
-    if(exists $data{$c}->{compat}) {
+    if(exists $data{$c} && exists $data{$c}->{compat}) {
        my $s = join(",",
                     (map(hex($_), split(/\s+/, $data{$c}->{compat})), 0));
        if(!exists $decompnums{$s}) {
@@ -387,7 +436,7 @@ my %cfnums = ();
 my $cfsaved = 0;
 out("static const uint32_t ");
 for(my $c = 0; $c <= $max; ++$c) {
-    if(exists $data{$c}->{casefold}) {
+    if(exists $data{$c} && exists $data{$c}->{casefold}) {
        my $s = join(",",
                     (map(hex($_), split(/\s+/, $data{$c}->{casefold})), 0));
        if(!exists $cfnums{$s}) {
@@ -451,13 +500,15 @@ for(my $base = 0; $base <= $max; $base += $modulus) {
     $subtableno{$base} = $subtable{$t};
 }
 
-out("const struct unidata*const unidata[]={\n");
+out("const struct unidata *const unidata[]={\n");
 for(my $base = 0; $base <= $max; $base += $modulus) {
+    #out("st$subtableno{$base} /* ".sprintf("%04x", $base)." */,\n");
     out("st$subtableno{$base},\n");
 }
 out("};\n");
 
 close STDOUT or die "unidata.c: $!\n";
 
-print STDERR "max=$max, subtables=$subtablecounter, subtablessaved=$subtablessaved\n";
+printf STDERR "max=%04X\n", $max;
+print STDERR "subtables=$subtablecounter, subtablessaved=$subtablessaved\n";
 print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n";