Update word break algorithm for Unicode 5.1.0 (based on UAX #29).

We look up the word break property more than once, which is ugly, so
some further work wouldn't be out of place.

diff --git a/lib/unicode.c b/lib/unicode.c
index 16c593282cb8f13232ada5066e80026d3fe1b485..6eadb6c1fc41cca292a4fef9703008ba5a96fc3d 100644 (file)
--- a/lib/unicode.c
+++ b/lib/unicode.c
@@ -633,6 +633,16 @@ int utf32_iterator_word_boundary(utf32_iterator it) {
   /* WB3 */
   if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
     return 0;
   /* WB3 */
   if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
     return 0;

+  /* WB3a */
+  if(utf32__iterator_word_break(it, it->s[it->n-1]) == unicode_Word_Break_Newline
+     || it->s[it->n-1] == 0x000D
+     || it->s[it->n-1] == 0x000A)
+    return 1;
+  /* WB3b */
+  if(utf32__iterator_word_break(it, it->s[it->n]) == unicode_Word_Break_Newline
+     || it->s[it->n] == 0x000D
+     || it->s[it->n] == 0x000A)
+    return 1;

   /* WB4 */
   /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
   if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep
   /* WB4 */
   /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
   if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep


@@ -667,12 +677,14 @@ int utf32_iterator_word_boundary(utf32_iterator it) {
     return 0;
   /* WB6 */
   if(before == unicode_Word_Break_ALetter
     return 0;
   /* WB6 */
   if(before == unicode_Word_Break_ALetter

-     && after == unicode_Word_Break_MidLetter
+     && (after == unicode_Word_Break_MidLetter
+         || after == unicode_Word_Break_MidNumLet)

      && twoafter == unicode_Word_Break_ALetter)
     return 0;
   /* WB7 */
   if(twobefore == unicode_Word_Break_ALetter
      && twoafter == unicode_Word_Break_ALetter)
     return 0;
   /* WB7 */
   if(twobefore == unicode_Word_Break_ALetter

-     && before == unicode_Word_Break_MidLetter
+     && (before == unicode_Word_Break_MidLetter
+         || before == unicode_Word_Break_MidNumLet)

      && after == unicode_Word_Break_ALetter)
     return 0;
   /* WB8 */
      && after == unicode_Word_Break_ALetter)
     return 0;
   /* WB8 */


@@ -689,12 +701,14 @@ int utf32_iterator_word_boundary(utf32_iterator it) {
     return 0;
    /* WB11 */
   if(twobefore == unicode_Word_Break_Numeric
     return 0;
    /* WB11 */
   if(twobefore == unicode_Word_Break_Numeric

-     && before == unicode_Word_Break_MidNum
+     && (before == unicode_Word_Break_MidNum
+         || before == unicode_Word_Break_MidNumLet)

      && after == unicode_Word_Break_Numeric)
     return 0;
   /* WB12 */
   if(before == unicode_Word_Break_Numeric
      && after == unicode_Word_Break_Numeric)
     return 0;
   /* WB12 */
   if(before == unicode_Word_Break_Numeric

-     && after == unicode_Word_Break_MidNum
+     && (after == unicode_Word_Break_MidNum
+         || after == unicode_Word_Break_MidNumLet)

      && twoafter == unicode_Word_Break_Numeric)
     return 0;
   /* WB13 */
      && twoafter == unicode_Word_Break_Numeric)
     return 0;
   /* WB13 */