X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/22b9fa74de8e80471a5033ea067d3b360930b91d..0b7052dac24e57edb180f07c0bd3479d397ebb74:/lib/test.c diff --git a/lib/test.c b/lib/test.c index ed90183..ddd6664 100644 --- a/lib/test.c +++ b/lib/test.c @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include "utf8.h" #include "mem.h" @@ -38,6 +40,9 @@ #include "hex.h" #include "words.h" #include "heap.h" +#include "unicode.h" +#include "inputline.h" +#include "wstat.h" static int tests, errors; @@ -69,6 +74,24 @@ static const char *format(const char *s) { return d.vec; } +static const char *format_utf32(const uint32_t *s) { + struct dynstr d; + uint32_t c; + char buf[64]; + + dynstr_init(&d); + while((c = *s++)) { + if(c >= 32 && c <= 127) + dynstr_append(&d, c); + else { + sprintf(buf, "\\x%04lX", (unsigned long)c); + dynstr_append_string(&d, buf); + } + } + dynstr_terminate(&d); + return d.vec; +} + #define check_string(GOT, WANT) do { \ const char *g = GOT; \ const char *w = WANT; \ @@ -332,16 +355,18 @@ static void test_hex(void) { } static void test_casefold(void) { - uint32_t c, l, u[2]; - const char *s, *ls; + uint32_t c, l; + const char *input, *canon_folded, *compat_folded, *canon_expected, *compat_expected; fprintf(stderr, "test_casefold\n"); + /* This isn't a very exhaustive test. Unlike for normalization, there don't + * seem to be any public test vectors for these algorithms. */ + for(c = 1; c < 256; ++c) { - u[0] = c; - u[1] = 0; - s = ucs42utf8(u); - ls = casefold(s); + input = utf32_to_utf8(&c, 1, 0); + canon_folded = utf8_casefold_canon(input, strlen(input), 0); + compat_folded = utf8_casefold_compat(input, strlen(input), 0); switch(c) { default: if((c >= 'A' && c <= 'Z') @@ -354,18 +379,26 @@ static void test_casefold(void) { l = 0x3BC; /* GREEK SMALL LETTER MU */ break; case 0xDF: /* LATIN SMALL LETTER SHARP S */ - insist(!strcmp(ls, "ss")); + insist(!strcmp(canon_folded, "ss")); + insist(!strcmp(compat_folded, "ss")); l = 0; break; } if(l) { - u[0] = l; - u[1] = 0; - s = ucs42utf8(u); - if(strcmp(s, ls)) { - fprintf(stderr, "%s:%d: casefolding %#lx got '%s', expected '%s'\n", + /* Case-folded data is now normalized */ + canon_expected = ucs42utf8(utf32_decompose_canon(&l, 1, 0)); + if(strcmp(canon_folded, canon_expected)) { + fprintf(stderr, "%s:%d: canon-casefolding %#lx got '%s', expected '%s'\n", __FILE__, __LINE__, (unsigned long)c, - format(ls), format(s)); + format(canon_folded), format(canon_expected)); + ++errors; + } + ++tests; + compat_expected = ucs42utf8(utf32_decompose_compat(&l, 1, 0)); + if(strcmp(compat_folded, compat_expected)) { + fprintf(stderr, "%s:%d: compat-casefolding %#lx got '%s', expected '%s'\n", + __FILE__, __LINE__, (unsigned long)c, + format(compat_folded), format(compat_expected)); ++errors; } ++tests; @@ -380,6 +413,7 @@ static inline int int_lt(int a, int b) { return a < b; } /** @struct iheap * @brief A heap with @c int elements */ HEAP_TYPE(iheap, int, int_lt); +HEAP_DEFINE(iheap, int, int_lt); /** @brief Tests for @ref heap.h */ static void test_heap(void) { @@ -402,6 +436,146 @@ static void test_heap(void) { putchar('\n'); } +/** @brief Open a Unicode test file */ +static FILE *open_unicode_test(const char *path) { + const char *base; + FILE *fp; + char buffer[1024]; + int w; + + if((base = strrchr(path, '/'))) + ++base; + else + base = path; + if(!(fp = fopen(base, "r"))) { + snprintf(buffer, sizeof buffer, + "wget http://www.unicode.org/Public/5.0.0/ucd/%s", path); + if((w = system(buffer))) + fatal(0, "%s: %s", buffer, wstat(w)); + if(chmod(base, 0444) < 0) + fatal(errno, "chmod %s", base); + if(!(fp = fopen(base, "r"))) + fatal(errno, "%s", base); + } + return fp; +} + +/** @brief Tests for @ref lib/unicode.h */ +static void test_unicode(void) { + FILE *fp; + int lineno = 0; + char *l, *lp; + uint32_t buffer[1024]; + uint32_t *c[6], *NFD_c[6], *NFKD_c[6]; /* 1-indexed */ + int cn, bn; + char break_allowed[1024]; + + fprintf(stderr, "test_unicode\n"); + fp = open_unicode_test("NormalizationTest.txt"); + while(!inputline("NormalizationTest.txt", fp, &l, '\n')) { + ++lineno; + if(*l == '#' || *l == '@') + continue; + bn = 0; + cn = 1; + lp = l; + c[cn++] = &buffer[bn]; + while(*lp && *lp != '#') { + if(*lp == ' ') { + ++lp; + continue; + } + if(*lp == ';') { + buffer[bn++] = 0; + if(cn == 6) + break; + c[cn++] = &buffer[bn]; + ++lp; + continue; + } + buffer[bn++] = strtoul(lp, &lp, 16); + } + buffer[bn] = 0; + assert(cn == 6); + for(cn = 1; cn <= 5; ++cn) { + NFD_c[cn] = utf32_decompose_canon(c[cn], utf32_len(c[cn]), 0); + NFKD_c[cn] = utf32_decompose_compat(c[cn], utf32_len(c[cn]), 0); + } +#define unt_check(T, A, B) do { \ + ++tests; \ + if(utf32_cmp(c[A], T##_c[B])) { \ + fprintf(stderr, \ + "NormalizationTest.txt:%d: c%d != "#T"(c%d)\n", \ + lineno, A, B); \ + fprintf(stderr, " c%d: %s\n", \ + A, format_utf32(c[A])); \ + fprintf(stderr, "%4s(c%d): %s\n", \ + #T, B, format_utf32(T##_c[B])); \ + ++errors; \ + } \ + } while(0) + unt_check(NFD, 3, 1); + unt_check(NFD, 3, 2); + unt_check(NFD, 3, 3); + unt_check(NFD, 5, 4); + unt_check(NFD, 5, 5); + unt_check(NFKD, 5, 1); + unt_check(NFKD, 5, 2); + unt_check(NFKD, 5, 3); + unt_check(NFKD, 5, 4); + unt_check(NFKD, 5, 5); + for(cn = 1; cn <= 5; ++cn) { + xfree(NFD_c[cn]); + xfree(NFKD_c[cn]); + } + xfree(l); + } + fclose(fp); + fp = open_unicode_test("auxiliary/GraphemeBreakTest.txt"); + lineno = 0; + while(!inputline("GraphemeBreakTest.txt.txt", fp, &l, '\n')) { + ++lineno; + if(l[0] == '#') continue; + bn = 0; + lp = l; + while(*lp) { + if(*lp == ' ' || *lp == '\t') { + ++lp; + continue; + } + if(*lp == '#') + break; + if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0xB7) { + /* 00F7 DIVISION SIGN */ + break_allowed[bn] = 1; + lp += 2; + continue; + } + if((unsigned char)*lp == 0xC3 && (unsigned char)lp[1] == 0x97) { + /* 00D7 MULTIPLICATION SIGN */ + break_allowed[bn] = 0; + lp += 2; + continue; + } + if(isxdigit((unsigned char)*lp)) { + buffer[bn++] = strtoul(lp, &lp, 16); + continue; + } + fatal(0, "GraphemeBreakTest.txt:%d: evil line: %s", lineno, l); + } + for(cn = 0; cn <= bn; ++cn) { + if(utf32_is_gcb(buffer, bn, cn) != break_allowed[cn]) { + fprintf(stderr, + "GraphemeBreakTest.txt:%d: offset %d: utf32_is_gcb wrong\n", + lineno, cn); + ++errors; + } + ++tests; + } + xfree(l); + } +} + int main(void) { insist('\n' == 0x0A); insist('\r' == 0x0D); @@ -440,6 +614,8 @@ int main(void) { /* split.c */ /* syscalls.c */ /* table.c */ + /* unicode.c */ + test_unicode(); /* utf8.c */ test_utf8(); /* vector.c */ @@ -455,6 +631,7 @@ int main(void) { Local Variables: c-basic-offset:2 comment-column:40 +fill-column:79 +indent-tabs-mode:nil End: */ -