tests and a correction for grapheme cluster boundary detection

diff --git a/.bzrignore b/.bzrignore
index 8bd29c106e2826cdeff44dd2093035a2db251695..0a019adc92e5ef736b5714f6df5d54dc79ab4b9f 100644 (file)
--- a/.bzrignore
+++ b/.bzrignore
@@ -121,3 +121,4 @@ lib/NormalizationTest.txt
 lib/CaseFolding.txt
 lib/UnicodeData.txt
 lib/GraphemeBreakProperty.txt
+lib/GraphemeBreakTest.txt

diff --git a/lib/test.c b/lib/test.c
index 50300adb67cd9eb0207931127d6a777595d08ca8..ddd6664ad8bc061b26f03172b586e5ad42ece091 100644 (file)
--- a/lib/test.c
+++ b/lib/test.c
@@ -42,6 +42,7 @@
 #include "heap.h"
 #include "unicode.h"
 #include "inputline.h"
+#include "wstat.h"
 
 static int tests, errors;
 
@@ -435,6 +436,30 @@ static void test_heap(void) {
   putchar('\n');
 }
 
+/** @brief Open a Unicode test file */
+static FILE *open_unicode_test(const char *path) {
+  const char *base;
+  FILE *fp;
+  char buffer[1024];
+  int w;
+
+  if((base = strrchr(path, '/')))
+    ++base;
+  else
+    base = path;
+  if(!(fp = fopen(base, "r"))) {
+    snprintf(buffer, sizeof buffer,
+             "wget http://www.unicode.org/Public/5.0.0/ucd/%s", path);
+    if((w = system(buffer)))
+      fatal(0, "%s: %s", buffer, wstat(w));
+    if(chmod(base, 0444) < 0)
+      fatal(errno, "chmod %s", base);
+    if(!(fp = fopen(base, "r")))
+      fatal(errno, "%s", base);
+  }
+  return fp;
+}
+
 /** @brief Tests for @ref lib/unicode.h */
 static void test_unicode(void) {
   FILE *fp;
@@ -443,18 +468,10 @@ static void test_unicode(void) {
   uint32_t buffer[1024];
   uint32_t *c[6], *NFD_c[6],  *NFKD_c[6]; /* 1-indexed */
   int cn, bn;
+  char break_allowed[1024];
 
   fprintf(stderr, "test_unicode\n");
-  if(!(fp = fopen("NormalizationTest.txt", "r"))) {
-    system("wget http://www.unicode.org/Public/5.0.0/ucd/NormalizationTest.txt");
-    chmod("NormalizationTest.txt", 0444);
-    if(!(fp = fopen("NormalizationTest.txt", "r"))) {
-      perror("NormalizationTest.txt");
-      ++tests;                         /* don't know how many... */
-      ++errors;
-      return;
-    }
-  }
+  fp = open_unicode_test("NormalizationTest.txt");
   while(!inputline("NormalizationTest.txt", fp, &l, '\n')) {
     ++lineno;
     if(*l == '#' || *l == '@')
@@ -487,7 +504,9 @@ static void test_unicode(void) {
 #define unt_check(T, A, B) do {                                        \
     ++tests;                                                   \
     if(utf32_cmp(c[A], T##_c[B])) {                            \
-      fprintf(stderr, "L%d: c%d != "#T"(c%d)\n", lineno, A, B);        \
+      fprintf(stderr,                                           \
+              "NormalizationTest.txt:%d: c%d != "#T"(c%d)\n",   \
+              lineno, A, B);                                    \
       fprintf(stderr, "    c%d:      %s\n",                    \
               A, format_utf32(c[A]));                          \
       fprintf(stderr, "%4s(c%d): %s\n",                                \
@@ -511,6 +530,50 @@ static void test_unicode(void) {
     }
     xfree(l);
   }
+  fclose(fp);
+  fp = open_unicode_test("auxiliary/GraphemeBreakTest.txt");
+  lineno = 0;
+  while(!inputline("GraphemeBreakTest.txt.txt", fp, &l, '\n')) {
+    ++lineno;
+    if(l[0] == '#') continue;
+    bn = 0;
+    lp = l;
+    while(*lp) {
+      if(*lp == ' ' || *lp == '\t') {
+        ++lp;
+        continue;
+      }
+      if(*lp == '#')
+        break;
+      if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0xB7) {
+        /* 00F7 DIVISION SIGN */
+        break_allowed[bn] = 1;
+        lp += 2;
+        continue;
+      }
+      if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0x97) {
+        /* 00D7 MULTIPLICATION SIGN */
+        break_allowed[bn] = 0;
+        lp += 2;
+        continue;
+      }
+      if(isxdigit((unsigned char)*lp)) {
+        buffer[bn++] = strtoul(lp, &lp, 16);
+        continue;
+      }
+      fatal(0, "GraphemeBreakTest.txt:%d: evil line: %s", lineno, l);
+    }
+    for(cn = 0; cn <= bn; ++cn) {
+      if(utf32_is_gcb(buffer, bn, cn) != break_allowed[cn]) {
+        fprintf(stderr,
+                "GraphemeBreakTest.txt:%d: offset %d: utf32_is_gcb wrong\n",
+                lineno, cn);
+        ++errors;
+      }
+      ++tests;
+    }
+    xfree(l);
+  }
 }
 
 int main(void) {

diff --git a/lib/unicode.c b/lib/unicode.c
index bac9e83c2e3bd9af467b494b6d6a2f2f0408111d..0a17b37a58167df0e9b9a9274460437970f3bfc5 100644 (file)
--- a/lib/unicode.c
+++ b/lib/unicode.c
@@ -670,7 +670,10 @@ int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) {
   hafter = utf32__hangul_syllable_type(after);
   /* GB6 */
   if(hbefore == Hangul_Syllable_Type_L
-     && hafter != Hangul_Syllable_Type_NA)
+     && (hafter == Hangul_Syllable_Type_L
+         || hafter == Hangul_Syllable_Type_V
+         || hafter == Hangul_Syllable_Type_LV
+         || hafter == Hangul_Syllable_Type_LVT))
     return 0;
   /* GB7 */
   if((hbefore == Hangul_Syllable_Type_LV