From bb48024fd1afb83557a91604a27eb398bf5e2139 Mon Sep 17 00:00:00 2001 Message-Id: From: Mark Wooding Date: Sun, 18 Nov 2007 11:31:31 +0000 Subject: [PATCH] test and corrections for utf32_is_word_boundary() Organization: Straylight/Edgeware From: Richard Kettlewell --- lib/test.c | 120 ++++++++++++++++++++++---------------- lib/unicode.c | 133 ++++++++++++++++++------------------------- lib/unidata.c | 11 ++++ lib/unidata.h | 1 + scripts/make-unidata | 8 +++ 5 files changed, 144 insertions(+), 129 deletions(-) diff --git a/lib/test.c b/lib/test.c index ddd6664..69ca609 100644 --- a/lib/test.c +++ b/lib/test.c @@ -45,11 +45,18 @@ #include "wstat.h" static int tests, errors; +static int fail_first; + +static void count_error() { + ++errors; + if(fail_first) + abort(); +} /** @brief Checks that @p expr is nonzero */ #define insist(expr) do { \ if(!(expr)) { \ - ++errors; \ + count_error(); \ fprintf(stderr, "%s:%d: error checking %s\n", \ __FILE__, __LINE__, #expr); \ } \ @@ -99,11 +106,11 @@ static const char *format_utf32(const uint32_t *s) { if(w == 0) { \ fprintf(stderr, "%s:%d: %s returned 0\n", \ __FILE__, __LINE__, #GOT); \ - ++errors; \ + count_error(); \ } else if(strcmp(w, g)) { \ fprintf(stderr, "%s:%d: %s returned:\n%s\nexpected:\n%s\n", \ __FILE__, __LINE__, #GOT, format(g), format(w)); \ - ++errors; \ + count_error(); \ } \ ++tests; \ } while(0) @@ -391,7 +398,7 @@ static void test_casefold(void) { fprintf(stderr, "%s:%d: canon-casefolding %#lx got '%s', expected '%s'\n", __FILE__, __LINE__, (unsigned long)c, format(canon_folded), format(canon_expected)); - ++errors; + count_error(); } ++tests; compat_expected = ucs42utf8(utf32_decompose_compat(&l, 1, 0)); @@ -399,7 +406,7 @@ static void test_casefold(void) { fprintf(stderr, "%s:%d: compat-casefolding %#lx got '%s', expected '%s'\n", __FILE__, __LINE__, (unsigned long)c, format(compat_folded), format(compat_expected)); - ++errors; + count_error(); } ++tests; } @@ -460,6 +467,60 @@ static FILE *open_unicode_test(const char *path) { return fp; } +/** @brief Run breaking tests for utf32_is_gcb() etc */ +static void breaktest(const char *path, + int (*breakfn)(const uint32_t *, size_t, size_t)) { + FILE *fp = open_unicode_test(path); + int lineno = 0; + char *l, *lp; + size_t bn, n; + char break_allowed[1024]; + uint32_t buffer[1024]; + + while(!inputline(path, fp, &l, '\n')) { + ++lineno; + if(l[0] == '#') continue; + bn = 0; + lp = l; + while(*lp) { + if(*lp == ' ' || *lp == '\t') { + ++lp; + continue; + } + if(*lp == '#') + break; + if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0xB7) { + /* 00F7 DIVISION SIGN */ + break_allowed[bn] = 1; + lp += 2; + continue; + } + if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0x97) { + /* 00D7 MULTIPLICATION SIGN */ + break_allowed[bn] = 0; + lp += 2; + continue; + } + if(isxdigit((unsigned char)*lp)) { + buffer[bn++] = strtoul(lp, &lp, 16); + continue; + } + fatal(0, "%s:%d: evil line: %s", path, lineno, l); + } + for(n = 0; n <= bn; ++n) { + if(breakfn(buffer, bn, n) != break_allowed[n]) { + fprintf(stderr, + "%s:%d: offset %zu: mismatch\n", + path, lineno, n); + count_error(); + } + ++tests; + } + xfree(l); + } + fclose(fp); +} + /** @brief Tests for @ref lib/unicode.h */ static void test_unicode(void) { FILE *fp; @@ -468,7 +529,6 @@ static void test_unicode(void) { uint32_t buffer[1024]; uint32_t *c[6], *NFD_c[6], *NFKD_c[6]; /* 1-indexed */ int cn, bn; - char break_allowed[1024]; fprintf(stderr, "test_unicode\n"); fp = open_unicode_test("NormalizationTest.txt"); @@ -511,7 +571,7 @@ static void test_unicode(void) { A, format_utf32(c[A])); \ fprintf(stderr, "%4s(c%d): %s\n", \ #T, B, format_utf32(T##_c[B])); \ - ++errors; \ + count_error(); \ } \ } while(0) unt_check(NFD, 3, 1); @@ -531,52 +591,12 @@ static void test_unicode(void) { xfree(l); } fclose(fp); - fp = open_unicode_test("auxiliary/GraphemeBreakTest.txt"); - lineno = 0; - while(!inputline("GraphemeBreakTest.txt.txt", fp, &l, '\n')) { - ++lineno; - if(l[0] == '#') continue; - bn = 0; - lp = l; - while(*lp) { - if(*lp == ' ' || *lp == '\t') { - ++lp; - continue; - } - if(*lp == '#') - break; - if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0xB7) { - /* 00F7 DIVISION SIGN */ - break_allowed[bn] = 1; - lp += 2; - continue; - } - if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0x97) { - /* 00D7 MULTIPLICATION SIGN */ - break_allowed[bn] = 0; - lp += 2; - continue; - } - if(isxdigit((unsigned char)*lp)) { - buffer[bn++] = strtoul(lp, &lp, 16); - continue; - } - fatal(0, "GraphemeBreakTest.txt:%d: evil line: %s", lineno, l); - } - for(cn = 0; cn <= bn; ++cn) { - if(utf32_is_gcb(buffer, bn, cn) != break_allowed[cn]) { - fprintf(stderr, - "GraphemeBreakTest.txt:%d: offset %d: utf32_is_gcb wrong\n", - lineno, cn); - ++errors; - } - ++tests; - } - xfree(l); - } + breaktest("auxiliary/GraphemeBreakTest.txt", utf32_is_gcb); + breaktest("auxiliary/WordBreakTest.txt", utf32_is_word_boundary); } int main(void) { + fail_first = !!getenv("FAIL_FIRST"); insist('\n' == 0x0A); insist('\r' == 0x0D); insist(' ' == 0x20); diff --git a/lib/unicode.c b/lib/unicode.c index ab2bc2f..48b7b2c 100644 --- a/lib/unicode.c +++ b/lib/unicode.c @@ -705,65 +705,10 @@ int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) { return 1; } -/** @brief Return true if code point @p n is part of an initial sequence of Format/Extend - * @param s Start of string - * @param ns Length of string - * @param n Start position - * @return True if it is, false otherwise - * - * This whole stack is not very efficient; we assume we don't see many of the - * problematic characters. - */ -static int utf32__is_initial_sequence(const uint32_t *s, - size_t attribute((unused)) ns, - size_t n) { - enum unicode_Word_Break wb; - - while(n > 0) { - --n; - wb = utf32__word_break(s[n]); - if(wb != unicode_Word_Break_Extend - && wb != unicode_Word_Break_Format) - return 0; - } - return 1; -} - -/** @brief Return the index of the first non-Extend/Format character from n - * @param s Start of string - * @param ns Length of string - * @param n Start position - * @return Index of first suitable character or @p ns - */ -static size_t utf32__first_not_ignorable(const uint32_t *s, size_t ns, - size_t n) { - while(n < ns) { - const enum unicode_Word_Break wb = utf32__word_break(s[n]); - if((wb != unicode_Word_Break_Extend - && wb != unicode_Word_Break_Format) - || utf32__is_initial_sequence(s, ns, n)) - return n; - ++n; - } - return ns; -} - -/** @brief Return the index of the last non-Extend/Format character from n - * @param s Start of string - * @param ns Length of string - * @param n Start position - * @return Index of first suitable character or (size_t)-1 - */ -static size_t utf32__last_not_ignorable(const uint32_t *s, size_t ns, - size_t n) { - do { - const enum unicode_Word_Break wb = utf32__word_break(s[n]); - if((wb != unicode_Word_Break_Extend - && wb != unicode_Word_Break_Format) - || utf32__is_initial_sequence(s, ns, n)) - return n; - } while(n--); - return n; /* will be (size_t)-1 */ +/** @brief Return true if @p c is ignorable for boundary specifications */ +static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) { + return (wb == unicode_Word_Break_Extend + || wb == unicode_Word_Break_Format); } /** @brief Identify a word boundary @@ -787,28 +732,58 @@ int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) { if(s[n-1] == 0x000D && s[n] == 0x000A) return 0; /* WB4 */ - /* The stated requirement here is to ignore code points with a Word_Break - * value of _Extend or _Format wherever they may appear unless they are part - * of an initial sequence of such characters. */ - twobefore = before = after = twoafter = unicode_Word_Break_Other; - nn = utf32__last_not_ignorable(s, ns, n - 1/* > 0 */); - if(nn != (size_t)-1) { - before = utf32__word_break(s[nn]); - if(nn != 0) { - nn = utf32__last_not_ignorable(s, ns, nn - 1); - if(nn != (size_t)-1) - twobefore = utf32__word_break(s[nn]); - } + /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */ + switch(s[n-1]) { /* bit of a bodge */ + case 0x000A: + case 0x000D: + case 0x0085: + case 0x2028: + case 0x2029: + break; + default: + if(utf32__boundary_ignorable(utf32__word_break(s[n]))) + return 0; + break; } - nn = utf32__first_not_ignorable(s, ns, n); - if(nn < ns) { - after = utf32__word_break(s[nn]); - if(nn < ns) { - nn = utf32__first_not_ignorable(s, ns, nn + 1); - if(nn < ns) - twoafter = utf32__word_break(s[nn]); - } + /* Gather the property values we'll need for the rest of the test taking the + * s6.2 changes into account */ + /* First we look at the code points after the proposed boundary */ + nn = n; /* X */ + while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn]))) + ++nn; + } + /* It's possible now that nn=ns */ + if(nn < ns) + twoafter = utf32__word_break(s[nn]); + else + twoafter = unicode_Word_Break_Other; + + /* Next we look at the code points before the proposed boundary. This is a + * bit fiddlier. */ + nn = n; + while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1]))) + --nn; + if(nn == 0) { + /* s[nn] must be ignorable */ + before = utf32__word_break(s[nn]); + twobefore = unicode_Word_Break_Other; + } else { + /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not + * ignorable. */ + before = utf32__word_break(s[nn - 1]); + --nn; + /* Repeat the exercise */ + while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1]))) + --nn; + if(nn == 0) + twobefore = utf32__word_break(s[nn]); + else + twobefore = utf32__word_break(s[nn - 1]); } + /* WB5 */ if(before == unicode_Word_Break_ALetter && after == unicode_Word_Break_ALetter) diff --git a/lib/unidata.c b/lib/unidata.c index c520e5c..9e2afb2 100644 --- a/lib/unidata.c +++ b/lib/unidata.c @@ -32,6 +32,17 @@ #define Zl unicode_gc_Zl #define Zp unicode_gc_Zp #define Zs unicode_gc_Zs +const char *const unicode_Word_Break_names[] = { + "ALetter", + "Extend", + "ExtendNumLet", + "Format", + "Katakana", + "MidLetter", + "MidNum", + "Numeric", + "Other" +}; static const uint32_t cd0[]={32,0}, cd1[]={32,776,0}, cd2[]={97,0}, diff --git a/lib/unidata.h b/lib/unidata.h index 5e6df0d..04ba220 100644 --- a/lib/unidata.h +++ b/lib/unidata.h @@ -44,6 +44,7 @@ enum unicode_Word_Break { unicode_Word_Break_Numeric, unicode_Word_Break_Other }; +extern const char *const unicode_Word_Break_names[]; enum unicode_flags { unicode_normalize_before_casefold = 1 }; diff --git a/scripts/make-unidata b/scripts/make-unidata index b00eb0a..58de22f 100755 --- a/scripts/make-unidata +++ b/scripts/make-unidata @@ -245,6 +245,7 @@ out("enum unicode_Word_Break {\n", join(",\n", map(" unicode_Word_Break_$_", sort keys %wbpropvals)), "\n};\n"); +out("extern const char *const unicode_Word_Break_names[];\n"); out("enum unicode_flags {\n", " unicode_normalize_before_casefold = 1\n", @@ -303,6 +304,13 @@ out("/* Automatically generated file, see scripts/make-unidata */\n", out(map(sprintf("#define %s unicode_gc_%s\n", $_, $_), sort keys %cats)); +# Names for Word_Break property + +out("const char *const unicode_Word_Break_names[] = {\n", + join(",\n", + map(" \"$_\"", sort keys %wbpropvals)), + "\n};\n"); + # Generate the decomposition mapping tables. We look out for duplicates # in order to save space and report this as decompsaved at the end. In # Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes. -- [mdw]