#include "wstat.h"
static int tests, errors;
+static int fail_first;
+
+static void count_error() {
+ ++errors;
+ if(fail_first)
+ abort();
+}
/** @brief Checks that @p expr is nonzero */
#define insist(expr) do { \
if(!(expr)) { \
- ++errors; \
+ count_error(); \
fprintf(stderr, "%s:%d: error checking %s\n", \
__FILE__, __LINE__, #expr); \
} \
if(w == 0) { \
fprintf(stderr, "%s:%d: %s returned 0\n", \
__FILE__, __LINE__, #GOT); \
- ++errors; \
+ count_error(); \
} else if(strcmp(w, g)) { \
fprintf(stderr, "%s:%d: %s returned:\n%s\nexpected:\n%s\n", \
__FILE__, __LINE__, #GOT, format(g), format(w)); \
- ++errors; \
+ count_error(); \
} \
++tests; \
} while(0)
fprintf(stderr, "%s:%d: canon-casefolding %#lx got '%s', expected '%s'\n",
__FILE__, __LINE__, (unsigned long)c,
format(canon_folded), format(canon_expected));
- ++errors;
+ count_error();
}
++tests;
compat_expected = ucs42utf8(utf32_decompose_compat(&l, 1, 0));
fprintf(stderr, "%s:%d: compat-casefolding %#lx got '%s', expected '%s'\n",
__FILE__, __LINE__, (unsigned long)c,
format(compat_folded), format(compat_expected));
- ++errors;
+ count_error();
}
++tests;
}
return fp;
}
+/** @brief Run breaking tests for utf32_is_gcb() etc */
+static void breaktest(const char *path,
+ int (*breakfn)(const uint32_t *, size_t, size_t)) {
+ FILE *fp = open_unicode_test(path);
+ int lineno = 0;
+ char *l, *lp;
+ size_t bn, n;
+ char break_allowed[1024];
+ uint32_t buffer[1024];
+
+ while(!inputline(path, fp, &l, '\n')) {
+ ++lineno;
+ if(l[0] == '#') continue;
+ bn = 0;
+ lp = l;
+ while(*lp) {
+ if(*lp == ' ' || *lp == '\t') {
+ ++lp;
+ continue;
+ }
+ if(*lp == '#')
+ break;
+ if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0xB7) {
+ /* 00F7 DIVISION SIGN */
+ break_allowed[bn] = 1;
+ lp += 2;
+ continue;
+ }
+ if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0x97) {
+ /* 00D7 MULTIPLICATION SIGN */
+ break_allowed[bn] = 0;
+ lp += 2;
+ continue;
+ }
+ if(isxdigit((unsigned char)*lp)) {
+ buffer[bn++] = strtoul(lp, &lp, 16);
+ continue;
+ }
+ fatal(0, "%s:%d: evil line: %s", path, lineno, l);
+ }
+ for(n = 0; n <= bn; ++n) {
+ if(breakfn(buffer, bn, n) != break_allowed[n]) {
+ fprintf(stderr,
+ "%s:%d: offset %zu: mismatch\n",
+ path, lineno, n);
+ count_error();
+ }
+ ++tests;
+ }
+ xfree(l);
+ }
+ fclose(fp);
+}
+
/** @brief Tests for @ref lib/unicode.h */
static void test_unicode(void) {
FILE *fp;
uint32_t buffer[1024];
uint32_t *c[6], *NFD_c[6], *NFKD_c[6]; /* 1-indexed */
int cn, bn;
- char break_allowed[1024];
fprintf(stderr, "test_unicode\n");
fp = open_unicode_test("NormalizationTest.txt");
A, format_utf32(c[A])); \
fprintf(stderr, "%4s(c%d): %s\n", \
#T, B, format_utf32(T##_c[B])); \
- ++errors; \
+ count_error(); \
} \
} while(0)
unt_check(NFD, 3, 1);
xfree(l);
}
fclose(fp);
- fp = open_unicode_test("auxiliary/GraphemeBreakTest.txt");
- lineno = 0;
- while(!inputline("GraphemeBreakTest.txt.txt", fp, &l, '\n')) {
- ++lineno;
- if(l[0] == '#') continue;
- bn = 0;
- lp = l;
- while(*lp) {
- if(*lp == ' ' || *lp == '\t') {
- ++lp;
- continue;
- }
- if(*lp == '#')
- break;
- if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0xB7) {
- /* 00F7 DIVISION SIGN */
- break_allowed[bn] = 1;
- lp += 2;
- continue;
- }
- if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0x97) {
- /* 00D7 MULTIPLICATION SIGN */
- break_allowed[bn] = 0;
- lp += 2;
- continue;
- }
- if(isxdigit((unsigned char)*lp)) {
- buffer[bn++] = strtoul(lp, &lp, 16);
- continue;
- }
- fatal(0, "GraphemeBreakTest.txt:%d: evil line: %s", lineno, l);
- }
- for(cn = 0; cn <= bn; ++cn) {
- if(utf32_is_gcb(buffer, bn, cn) != break_allowed[cn]) {
- fprintf(stderr,
- "GraphemeBreakTest.txt:%d: offset %d: utf32_is_gcb wrong\n",
- lineno, cn);
- ++errors;
- }
- ++tests;
- }
- xfree(l);
- }
+ breaktest("auxiliary/GraphemeBreakTest.txt", utf32_is_gcb);
+ breaktest("auxiliary/WordBreakTest.txt", utf32_is_word_boundary);
}
int main(void) {
+ fail_first = !!getenv("FAIL_FIRST");
insist('\n' == 0x0A);
insist('\r' == 0x0D);
insist(' ' == 0x20);
return 1;
}
-/** @brief Return true if code point @p n is part of an initial sequence of Format/Extend
- * @param s Start of string
- * @param ns Length of string
- * @param n Start position
- * @return True if it is, false otherwise
- *
- * This whole stack is not very efficient; we assume we don't see many of the
- * problematic characters.
- */
-static int utf32__is_initial_sequence(const uint32_t *s,
- size_t attribute((unused)) ns,
- size_t n) {
- enum unicode_Word_Break wb;
-
- while(n > 0) {
- --n;
- wb = utf32__word_break(s[n]);
- if(wb != unicode_Word_Break_Extend
- && wb != unicode_Word_Break_Format)
- return 0;
- }
- return 1;
-}
-
-/** @brief Return the index of the first non-Extend/Format character from n
- * @param s Start of string
- * @param ns Length of string
- * @param n Start position
- * @return Index of first suitable character or @p ns
- */
-static size_t utf32__first_not_ignorable(const uint32_t *s, size_t ns,
- size_t n) {
- while(n < ns) {
- const enum unicode_Word_Break wb = utf32__word_break(s[n]);
- if((wb != unicode_Word_Break_Extend
- && wb != unicode_Word_Break_Format)
- || utf32__is_initial_sequence(s, ns, n))
- return n;
- ++n;
- }
- return ns;
-}
-
-/** @brief Return the index of the last non-Extend/Format character from n
- * @param s Start of string
- * @param ns Length of string
- * @param n Start position
- * @return Index of first suitable character or (size_t)-1
- */
-static size_t utf32__last_not_ignorable(const uint32_t *s, size_t ns,
- size_t n) {
- do {
- const enum unicode_Word_Break wb = utf32__word_break(s[n]);
- if((wb != unicode_Word_Break_Extend
- && wb != unicode_Word_Break_Format)
- || utf32__is_initial_sequence(s, ns, n))
- return n;
- } while(n--);
- return n; /* will be (size_t)-1 */
+/** @brief Return true if @p c is ignorable for boundary specifications */
+static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
+ return (wb == unicode_Word_Break_Extend
+ || wb == unicode_Word_Break_Format);
}
/** @brief Identify a word boundary
if(s[n-1] == 0x000D && s[n] == 0x000A)
return 0;
/* WB4 */
- /* The stated requirement here is to ignore code points with a Word_Break
- * value of _Extend or _Format wherever they may appear unless they are part
- * of an initial sequence of such characters. */
- twobefore = before = after = twoafter = unicode_Word_Break_Other;
- nn = utf32__last_not_ignorable(s, ns, n - 1/* > 0 */);
- if(nn != (size_t)-1) {
- before = utf32__word_break(s[nn]);
- if(nn != 0) {
- nn = utf32__last_not_ignorable(s, ns, nn - 1);
- if(nn != (size_t)-1)
- twobefore = utf32__word_break(s[nn]);
- }
+ /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
+ switch(s[n-1]) { /* bit of a bodge */
+ case 0x000A:
+ case 0x000D:
+ case 0x0085:
+ case 0x2028:
+ case 0x2029:
+ break;
+ default:
+ if(utf32__boundary_ignorable(utf32__word_break(s[n])))
+ return 0;
+ break;
}
- nn = utf32__first_not_ignorable(s, ns, n);
- if(nn < ns) {
- after = utf32__word_break(s[nn]);
- if(nn < ns) {
- nn = utf32__first_not_ignorable(s, ns, nn + 1);
- if(nn < ns)
- twoafter = utf32__word_break(s[nn]);
- }
+ /* Gather the property values we'll need for the rest of the test taking the
+ * s6.2 changes into account */
+ /* First we look at the code points after the proposed boundary */
+ nn = n; /* <ns */
+ after = utf32__word_break(s[nn++]);
+ if(!utf32__boundary_ignorable(after)) {
+ /* X (Extend|Format)* -> X */
+ while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn])))
+ ++nn;
+ }
+ /* It's possible now that nn=ns */
+ if(nn < ns)
+ twoafter = utf32__word_break(s[nn]);
+ else
+ twoafter = unicode_Word_Break_Other;
+
+ /* Next we look at the code points before the proposed boundary. This is a
+ * bit fiddlier. */
+ nn = n;
+ while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+ --nn;
+ if(nn == 0) {
+ /* s[nn] must be ignorable */
+ before = utf32__word_break(s[nn]);
+ twobefore = unicode_Word_Break_Other;
+ } else {
+ /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not
+ * ignorable. */
+ before = utf32__word_break(s[nn - 1]);
+ --nn;
+ /* Repeat the exercise */
+ while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+ --nn;
+ if(nn == 0)
+ twobefore = utf32__word_break(s[nn]);
+ else
+ twobefore = utf32__word_break(s[nn - 1]);
}
+
/* WB5 */
if(before == unicode_Word_Break_ALetter
&& after == unicode_Word_Break_ALetter)
#define Zl unicode_gc_Zl
#define Zp unicode_gc_Zp
#define Zs unicode_gc_Zs
+const char *const unicode_Word_Break_names[] = {
+ "ALetter",
+ "Extend",
+ "ExtendNumLet",
+ "Format",
+ "Katakana",
+ "MidLetter",
+ "MidNum",
+ "Numeric",
+ "Other"
+};
static const uint32_t cd0[]={32,0},
cd1[]={32,776,0},
cd2[]={97,0},
unicode_Word_Break_Numeric,
unicode_Word_Break_Other
};
+extern const char *const unicode_Word_Break_names[];
enum unicode_flags {
unicode_normalize_before_casefold = 1
};
join(",\n",
map(" unicode_Word_Break_$_", sort keys %wbpropvals)),
"\n};\n");
+out("extern const char *const unicode_Word_Break_names[];\n");
out("enum unicode_flags {\n",
" unicode_normalize_before_casefold = 1\n",
out(map(sprintf("#define %s unicode_gc_%s\n", $_, $_), sort keys %cats));
+# Names for Word_Break property
+
+out("const char *const unicode_Word_Break_names[] = {\n",
+ join(",\n",
+ map(" \"$_\"", sort keys %wbpropvals)),
+ "\n};\n");
+
# Generate the decomposition mapping tables. We look out for duplicates
# in order to save space and report this as decompsaved at the end. In
# Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes.
~mdw / disorder / commitdiff