X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/35b651f0cc0999deae42c92f2cbca3ecf88fe157..0a4af69255e6fee5274e645e96eee16b6c67a70c:/lib/unicode.c

diff --git a/lib/unicode.c b/lib/unicode.c
index bac9e83..618ff06 100644
--- a/lib/unicode.c
+++ b/lib/unicode.c
@@ -577,12 +577,12 @@ int utf32_cmp(const uint32_t *a, const uint32_t *b) {
  * @param Code point
  * @return General_Category property value
  */
-static inline enum unicode_gc_cat utf32__general_category(uint32_t c) {
+static inline enum unicode_General_Category utf32__general_category(uint32_t c) {
   if(c < UNICODE_NCHARS) {
     const struct unidata *const ud = &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
-    return ud->gc;
+    return ud->general_category;
   } else
-    return unicode_gc_Cn;
+    return unicode_General_Category_Cn;
 }
 
 /** @brief Check Grapheme_Cluster_Break property
@@ -593,11 +593,11 @@ static int utf32__is_control_or_cr_or_lf(uint32_t c) {
   switch(utf32__general_category(c)) {
   default:
     return 0;
-  case unicode_gc_Zl:
-  case unicode_gc_Zp:
-  case unicode_gc_Cc:
+  case unicode_General_Category_Zl:
+  case unicode_General_Category_Zp:
+  case unicode_General_Category_Cc:
     return 1;
-  case unicode_gc_Cf:
+  case unicode_General_Category_Cf:
     if(c == 0x200C || c == 0x200D)
       return 0;
     return 1;
@@ -637,6 +637,20 @@ static uint32_t utf32__hangul_syllable_type(uint32_t c) {
   return Hangul_Syllable_Type_NA;
 }
 
+/** @brief Determine Word_Break property
+ * @param c Code point
+ * @return Word_Break property value of @p c
+ */
+static enum unicode_Word_Break utf32__word_break(uint32_t c) {
+  if(c < 0xAC00 || c > 0xD7A3) {
+    if(c < UNICODE_NCHARS)
+      return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].word_break;
+    else
+      return unicode_Word_Break_Other;
+  } else
+    return unicode_Word_Break_ALetter;
+}
+
 /** @brief Identify a grapheme cluster boundary
  * @param s Start of string (must be NFD)
  * @param ns Length of string
@@ -647,8 +661,6 @@ static uint32_t utf32__hangul_syllable_type(uint32_t c) {
  * UAX #29 s3.  It returns 1 if @p n points at the code point just after a
  * grapheme cluster boundary (including the hypothetical code point just after
  * the end of the string).
- *
- * The string must be in NFD (or NFKD) for this function to work (currently).
  */
 int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) {
   uint32_t before, after;
@@ -670,7 +682,10 @@ int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) {
   hafter = utf32__hangul_syllable_type(after);
   /* GB6 */
   if(hbefore == Hangul_Syllable_Type_L
-     && hafter != Hangul_Syllable_Type_NA)
+     && (hafter == Hangul_Syllable_Type_L
+         || hafter == Hangul_Syllable_Type_V
+         || hafter == Hangul_Syllable_Type_LV
+         || hafter == Hangul_Syllable_Type_LVT))
     return 0;
   /* GB7 */
   if((hbefore == Hangul_Syllable_Type_LV
@@ -684,16 +699,150 @@ int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) {
      && hafter == Hangul_Syllable_Type_T)
     return 0;
   /* GB9 */
-  if(after < UNICODE_NCHARS
-     && (unidata[after / UNICODE_MODULUS][after % UNICODE_MODULUS].flags
-         & unicode_grapheme_break_extend))
+  if(utf32__word_break(after) == unicode_Word_Break_Extend)
     return 0;
   /* GB10 */
   return 1;
 }
 
+/** @brief Return true if @p c is ignorable for boundary specifications */
+static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
+  return (wb == unicode_Word_Break_Extend
+          || wb == unicode_Word_Break_Format);
+}
+
+/** @brief Identify a word boundary
+ * @param s Start of string (must be NFD)
+ * @param ns Length of string
+ * @param n Index within string (in [0,ns].)
+ * @return 1 at a word boundary, 0 otherwise
+ *
+ * This function identifies default word boundaries as described in UAX #29 s4.
+ * It returns 1 if @p n points at the code point just after a word boundary
+ * (including the hypothetical code point just after the end of the string).
+ */
+int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) {
+  enum unicode_Word_Break twobefore, before, after, twoafter;
+  size_t nn;
+
+  /* WB1 and WB2 */
+  if(n == 0 || n == ns)
+    return 1;
+  /* WB3 */
+  if(s[n-1] == 0x000D && s[n] == 0x000A)
+    return 0;
+  /* WB4 */
+  /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
+  switch(s[n-1]) {                      /* bit of a bodge */
+  case 0x000A:
+  case 0x000D:
+  case 0x0085:
+  case 0x2028:
+  case 0x2029:
+    break;
+  default:
+    if(utf32__boundary_ignorable(utf32__word_break(s[n])))
+      return 0;
+    break;
+  }
+  /* Gather the property values we'll need for the rest of the test taking the
+   * s6.2 changes into account */
+  /* First we look at the code points after the proposed boundary */
+  nn = n;                               /* <ns */
+  after = utf32__word_break(s[nn++]);
+  if(!utf32__boundary_ignorable(after)) {
+    /* X (Extend|Format)* -> X */
+    while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn])))
+      ++nn;
+  }
+  /* It's possible now that nn=ns */
+  if(nn < ns)
+    twoafter = utf32__word_break(s[nn]);
+  else
+    twoafter = unicode_Word_Break_Other;
+
+  /* Next we look at the code points before the proposed boundary.  This is a
+   * bit fiddlier. */
+  nn = n;
+  while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+    --nn;
+  if(nn == 0) {
+    /* s[nn] must be ignorable */
+    before = utf32__word_break(s[nn]);
+    twobefore = unicode_Word_Break_Other;
+  } else {
+    /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not
+     * ignorable. */
+    before = utf32__word_break(s[nn - 1]);
+    --nn;
+    /* Repeat the exercise */
+    while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+      --nn;
+    if(nn == 0)
+      twobefore = utf32__word_break(s[nn]);
+    else
+      twobefore = utf32__word_break(s[nn - 1]);
+  }
+
+  /* WB5 */
+  if(before == unicode_Word_Break_ALetter
+     && after == unicode_Word_Break_ALetter)
+    return 0;
+  /* WB6 */
+  if(before == unicode_Word_Break_ALetter
+     && after == unicode_Word_Break_MidLetter
+     && twoafter == unicode_Word_Break_ALetter)
+    return 0;
+  /* WB7 */
+  if(twobefore == unicode_Word_Break_ALetter
+     && before == unicode_Word_Break_MidLetter
+     && after == unicode_Word_Break_ALetter)
+    return 0;
+  /* WB8 */  
+  if(before == unicode_Word_Break_Numeric
+     && after == unicode_Word_Break_Numeric)
+    return 0;
+  /* WB9 */
+  if(before == unicode_Word_Break_ALetter
+     && after == unicode_Word_Break_Numeric)
+    return 0;
+  /* WB10 */
+  if(before == unicode_Word_Break_Numeric
+     && after == unicode_Word_Break_ALetter)
+    return 0;
+   /* WB11 */
+  if(twobefore == unicode_Word_Break_Numeric
+     && before == unicode_Word_Break_MidNum
+     && after == unicode_Word_Break_Numeric)
+    return 0;
+  /* WB12 */
+  if(before == unicode_Word_Break_Numeric
+     && after == unicode_Word_Break_MidNum
+     && twoafter == unicode_Word_Break_Numeric)
+    return 0;
+  /* WB13 */
+  if(before == unicode_Word_Break_Katakana
+     && after == unicode_Word_Break_Katakana)
+    return 0;
+  /* WB13a */
+  if((before == unicode_Word_Break_ALetter
+      || before == unicode_Word_Break_Numeric
+      || before == unicode_Word_Break_Katakana
+      || before == unicode_Word_Break_ExtendNumLet)
+     && after == unicode_Word_Break_ExtendNumLet)
+    return 0;
+  /* WB13b */
+  if(before == unicode_Word_Break_ExtendNumLet
+     && (after == unicode_Word_Break_ALetter
+         || after == unicode_Word_Break_Numeric
+         || after == unicode_Word_Break_Katakana))
+    return 0;
+  /* WB14 */
+  return 1;
+}
+
 /*@}*/
-/** @defgroup Functions that operate on UTF-8 strings */
+/** @defgroup utf8 Functions that operate on UTF-8 strings */
 /*@{*/
 
 /** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */