From fb4c61dadb5bc3f21ff6f9488eaf7019b28a3f8a Mon Sep 17 00:00:00 2001 Message-Id: From: Mark Wooding Date: Sun, 5 Apr 2009 10:56:55 +0100 Subject: [PATCH] Update word break algorithm for Unicode 5.1.0 (based on UAX #29). We look up the word break property more than once, which is ugly, so some further work wouldn't be out of place. Organization: Straylight/Edgeware From: Richard Kettlewell --- lib/unicode.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/unicode.c b/lib/unicode.c index 16c5932..6eadb6c 100644 --- a/lib/unicode.c +++ b/lib/unicode.c @@ -633,6 +633,16 @@ int utf32_iterator_word_boundary(utf32_iterator it) { /* WB3 */ if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A) return 0; + /* WB3a */ + if(utf32__iterator_word_break(it, it->s[it->n-1]) == unicode_Word_Break_Newline + || it->s[it->n-1] == 0x000D + || it->s[it->n-1] == 0x000A) + return 1; + /* WB3b */ + if(utf32__iterator_word_break(it, it->s[it->n]) == unicode_Word_Break_Newline + || it->s[it->n] == 0x000D + || it->s[it->n] == 0x000A) + return 1; /* WB4 */ /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */ if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep @@ -667,12 +677,14 @@ int utf32_iterator_word_boundary(utf32_iterator it) { return 0; /* WB6 */ if(before == unicode_Word_Break_ALetter - && after == unicode_Word_Break_MidLetter + && (after == unicode_Word_Break_MidLetter + || after == unicode_Word_Break_MidNumLet) && twoafter == unicode_Word_Break_ALetter) return 0; /* WB7 */ if(twobefore == unicode_Word_Break_ALetter - && before == unicode_Word_Break_MidLetter + && (before == unicode_Word_Break_MidLetter + || before == unicode_Word_Break_MidNumLet) && after == unicode_Word_Break_ALetter) return 0; /* WB8 */ @@ -689,12 +701,14 @@ int utf32_iterator_word_boundary(utf32_iterator it) { return 0; /* WB11 */ if(twobefore == unicode_Word_Break_Numeric - && before == unicode_Word_Break_MidNum + && (before == unicode_Word_Break_MidNum + || before == unicode_Word_Break_MidNumLet) && after == unicode_Word_Break_Numeric) return 0; /* WB12 */ if(before == unicode_Word_Break_Numeric - && after == unicode_Word_Break_MidNum + && (after == unicode_Word_Break_MidNum + || after == unicode_Word_Break_MidNumLet) && twoafter == unicode_Word_Break_Numeric) return 0; /* WB13 */ -- [mdw]