untested utf32_is_word_boundary() and associated table changes

[disorder] / lib / test.c

diff --git a/lib/test.c b/lib/test.c
index f5d94fdb3c14a2ef6e57041a74fb84d0bccd00f8..ddd6664ad8bc061b26f03172b586e5ad42ece091 100644 (file)
--- a/lib/test.c
+++ b/lib/test.c
@@ -42,6 +42,7 @@
 #include "heap.h"
 #include "unicode.h"
 #include "inputline.h"
+#include "wstat.h"
 
 static int tests, errors;
 
@@ -355,13 +356,17 @@ static void test_hex(void) {
 
 static void test_casefold(void) {
   uint32_t c, l;
-  const char *input, *folded, *expected;
+  const char *input, *canon_folded, *compat_folded, *canon_expected, *compat_expected;
 
   fprintf(stderr, "test_casefold\n");
+
+  /* This isn't a very exhaustive test.  Unlike for normalization, there don't
+   * seem to be any public test vectors for these algorithms. */
   
   for(c = 1; c < 256; ++c) {
     input = utf32_to_utf8(&c, 1, 0);
-    folded = utf8_casefold_canon(input, strlen(input), 0);
+    canon_folded = utf8_casefold_canon(input, strlen(input), 0);
+    compat_folded = utf8_casefold_compat(input, strlen(input), 0);
     switch(c) {
     default:
       if((c >= 'A' && c <= 'Z')
@@ -374,17 +379,26 @@ static void test_casefold(void) {
       l = 0x3BC;                       /* GREEK SMALL LETTER MU */
       break;
     case 0xDF:                         /* LATIN SMALL LETTER SHARP S */
-      insist(!strcmp(folded, "ss"));
+      insist(!strcmp(canon_folded, "ss"));
+      insist(!strcmp(compat_folded, "ss"));
       l = 0;
       break;
     }
     if(l) {
       /* Case-folded data is now normalized */
-      expected = ucs42utf8(utf32_decompose_canon(&l, 1, 0));
-      if(strcmp(folded, expected)) {
-       fprintf(stderr, "%s:%d: casefolding %#lx got '%s', expected '%s'\n",
+      canon_expected = ucs42utf8(utf32_decompose_canon(&l, 1, 0));
+      if(strcmp(canon_folded, canon_expected)) {
+       fprintf(stderr, "%s:%d: canon-casefolding %#lx got '%s', expected '%s'\n",
                __FILE__, __LINE__, (unsigned long)c,
-               format(folded), format(expected));
+               format(canon_folded), format(canon_expected));
+       ++errors;
+      }
+      ++tests;
+      compat_expected = ucs42utf8(utf32_decompose_compat(&l, 1, 0));
+      if(strcmp(compat_folded, compat_expected)) {
+       fprintf(stderr, "%s:%d: compat-casefolding %#lx got '%s', expected '%s'\n",
+               __FILE__, __LINE__, (unsigned long)c,
+               format(compat_folded), format(compat_expected));
        ++errors;
       }
       ++tests;
@@ -422,6 +436,30 @@ static void test_heap(void) {
   putchar('\n');
 }
 
+/** @brief Open a Unicode test file */
+static FILE *open_unicode_test(const char *path) {
+  const char *base;
+  FILE *fp;
+  char buffer[1024];
+  int w;
+
+  if((base = strrchr(path, '/')))
+    ++base;
+  else
+    base = path;
+  if(!(fp = fopen(base, "r"))) {
+    snprintf(buffer, sizeof buffer,
+             "wget http://www.unicode.org/Public/5.0.0/ucd/%s", path);
+    if((w = system(buffer)))
+      fatal(0, "%s: %s", buffer, wstat(w));
+    if(chmod(base, 0444) < 0)
+      fatal(errno, "chmod %s", base);
+    if(!(fp = fopen(base, "r")))
+      fatal(errno, "%s", base);
+  }
+  return fp;
+}
+
 /** @brief Tests for @ref lib/unicode.h */
 static void test_unicode(void) {
   FILE *fp;
@@ -430,18 +468,10 @@ static void test_unicode(void) {
   uint32_t buffer[1024];
   uint32_t *c[6], *NFD_c[6],  *NFKD_c[6]; /* 1-indexed */
   int cn, bn;
+  char break_allowed[1024];
 
   fprintf(stderr, "test_unicode\n");
-  if(!(fp = fopen("NormalizationTest.txt", "r"))) {
-    system("wget http://www.unicode.org/Public/5.0.0/ucd/NormalizationTest.txt");
-    chmod("NormalizationTest.txt", 0444);
-    if(!(fp = fopen("NormalizationTest.txt", "r"))) {
-      perror("NormalizationTest.txt");
-      ++tests;                         /* don't know how many... */
-      ++errors;
-      return;
-    }
-  }
+  fp = open_unicode_test("NormalizationTest.txt");
   while(!inputline("NormalizationTest.txt", fp, &l, '\n')) {
     ++lineno;
     if(*l == '#' || *l == '@')
@@ -474,7 +504,9 @@ static void test_unicode(void) {
 #define unt_check(T, A, B) do {                                        \
     ++tests;                                                   \
     if(utf32_cmp(c[A], T##_c[B])) {                            \
-      fprintf(stderr, "L%d: c%d != "#T"(c%d)\n", lineno, A, B);        \
+      fprintf(stderr,                                           \
+              "NormalizationTest.txt:%d: c%d != "#T"(c%d)\n",   \
+              lineno, A, B);                                    \
       fprintf(stderr, "    c%d:      %s\n",                    \
               A, format_utf32(c[A]));                          \
       fprintf(stderr, "%4s(c%d): %s\n",                                \
@@ -498,6 +530,50 @@ static void test_unicode(void) {
     }
     xfree(l);
   }
+  fclose(fp);
+  fp = open_unicode_test("auxiliary/GraphemeBreakTest.txt");
+  lineno = 0;
+  while(!inputline("GraphemeBreakTest.txt.txt", fp, &l, '\n')) {
+    ++lineno;
+    if(l[0] == '#') continue;
+    bn = 0;
+    lp = l;
+    while(*lp) {
+      if(*lp == ' ' || *lp == '\t') {
+        ++lp;
+        continue;
+      }
+      if(*lp == '#')
+        break;
+      if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0xB7) {
+        /* 00F7 DIVISION SIGN */
+        break_allowed[bn] = 1;
+        lp += 2;
+        continue;
+      }
+      if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0x97) {
+        /* 00D7 MULTIPLICATION SIGN */
+        break_allowed[bn] = 0;
+        lp += 2;
+        continue;
+      }
+      if(isxdigit((unsigned char)*lp)) {
+        buffer[bn++] = strtoul(lp, &lp, 16);
+        continue;
+      }
+      fatal(0, "GraphemeBreakTest.txt:%d: evil line: %s", lineno, l);
+    }
+    for(cn = 0; cn <= bn; ++cn) {
+      if(utf32_is_gcb(buffer, bn, cn) != break_allowed[cn]) {
+        fprintf(stderr,
+                "GraphemeBreakTest.txt:%d: offset %d: utf32_is_gcb wrong\n",
+                lineno, cn);
+        ++errors;
+      }
+      ++tests;
+    }
+    xfree(l);
+  }
 }
 
 int main(void) {
@@ -555,6 +631,7 @@ int main(void) {
 Local Variables:
 c-basic-offset:2
 comment-column:40
+fill-column:79
+indent-tabs-mode:nil
 End:
 */
-