X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~mdw/git/mLib/blobdiff_plain/c5e0e40378b7e209521d2e9a52f055575a948313..c81c35dfd10050ffef85d57dc2ad73f52f38a3f2:/test/tvec-core.c diff --git a/test/tvec-core.c b/test/tvec-core.c index 6aa49fa..65c74cf 100644 --- a/test/tvec-core.c +++ b/test/tvec-core.c @@ -37,72 +37,144 @@ /*----- Output ------------------------------------------------------------*/ -int tvec_error(struct tvec_state *tv, const char *msg, ...) +/* --- @tvec_strlevel@ --- * + * + * Arguments: @unsigned level@ = level code + * + * Returns: A human-readable description. + * + * Use: Converts a level code into something that you can print in a + * message. + */ + +const char *tvec_strlevel(unsigned level) +{ + switch (level) { +#define CASE(tag, name, val) \ + case TVLEV_##tag: return (name); + TVEC_LEVELS(CASE) +#undef CASE + default: return ("??"); + } +} + +/* --- @tvec_report@, @tvec_report_v@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @unsigned level@ = severity level (@TVlEV_...@) + * @const char *msg@, @va_list ap@ = error message + * + * Returns: --- + * + * Use: Report an message with a given severity. Messages with level + * @TVLEV_ERR@ or higher force a nonzero exit code. + */ + +void tvec_report(struct tvec_state *tv, unsigned level, const char *msg, ...) { va_list ap; - va_start(ap, msg); tvec_error_v(tv, msg, &ap); va_end(ap); - tv->f |= TVSF_ERROR; return (-1); + va_start(ap, msg); tvec_report_v(tv, level, msg, &ap); va_end(ap); } -int tvec_error_v(struct tvec_state *tv, const char *msg, va_list *ap) - { tv->output->ops->error(tv->output, msg, ap); return (-1); } -void tvec_notice(struct tvec_state *tv, const char *msg, ...) +void tvec_report_v(struct tvec_state *tv, unsigned level, + const char *msg, va_list *ap) { - va_list ap; - va_start(ap, msg); tvec_notice_v(tv, msg, &ap); va_end(ap); + tv->output->ops->report(tv->output, level, msg, ap); + if (level >= TVLEV_ERR) tv->f |= TVSF_ERROR; } -void tvec_notice_v(struct tvec_state *tv, const char *msg, va_list *ap) - { tv->output->ops->notice(tv->output, msg, ap); } -int tvec_syntax(struct tvec_state *tv, int ch, const char *expect, ...) +/* --- @tvec_error@, @tvec_notice@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *msg@, @va_list ap@ = error message + * + * Returns: The @tvec_error@ function returns @-1@ as a trivial + * convenience; @tvec_notice@ does not return a value. + * + * Use: Report an error or a notice. Errors are distinct from test + * failures, and indicate that a problem was encountered which + * compromised the activity of testing. Notices are important + * information which doesn't fit into any other obvious + * category. + */ + +int tvec_error(struct tvec_state *tv, const char *msg, ...) { va_list ap; - va_start(ap, expect); tvec_syntax_v(tv, ch, expect, &ap); va_end(ap); + va_start(ap, msg); tvec_report_v(tv, TVLEV_ERR, msg, &ap); va_end(ap); return (-1); } -int tvec_syntax_v(struct tvec_state *tv, int ch, - const char *expect, va_list *ap) + +void tvec_notice(struct tvec_state *tv, const char *msg, ...) { - dstr d = DSTR_INIT; - char found[8]; + va_list ap; - switch (ch) { - case EOF: strcpy(found, "#"); break; - case '\n': strcpy(found, "#"); ungetc(ch, tv->fp); break; - default: - if (isprint(ch)) sprintf(found, "`%c'", ch); - else sprintf(found, "#<\\x%02x>", ch); - break; - } - dstr_vputf(&d, expect, ap); - tvec_error(tv, "syntax error: expected %s but found %s", expect, found); - return (-1); + va_start(ap, msg); tvec_report_v(tv, TVLEV_NOTE, msg, &ap); va_end(ap); } +/*----- Test processing ---------------------------------------------------*/ + +/* --- @tvec_skipgroup@, @tvec_skipgroup_v@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *excuse@, @va_list *ap@ = reason why skipped + * + * Returns: --- + * + * Use: Skip the current group. This should only be called from a + * test environment @setup@ function; a similar effect occurs if + * the @setup@ function fails. + */ + void tvec_skipgroup(struct tvec_state *tv, const char *excuse, ...) { va_list ap; + va_start(ap, excuse); tvec_skipgroup_v(tv, excuse, &ap); va_end(ap); } + void tvec_skipgroup_v(struct tvec_state *tv, const char *excuse, va_list *ap) { - tv->f |= TVSF_SKIP; tv->grps[TVOUT_SKIP]++; - tv->output->ops->skipgroup(tv->output, excuse, ap); + if (!(tv->f&TVSF_SKIP)) { + tv->f |= TVSF_SKIP; tv->grps[TVOUT_SKIP]++; + tv->output->ops->skipgroup(tv->output, excuse, ap); + } } +/* --- @set_outcome@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @unsigned out@ = the new outcome + * + * Returns: --- + * + * Use: Sets the outcome bits in the test state flags, and clears + * @TVSF_ACTIVE@. + */ + static void set_outcome(struct tvec_state *tv, unsigned out) -{ - tv->f &= ~(TVSF_ACTIVE | TVSF_OUTMASK); - tv->f |= out << TVSF_OUTSHIFT; -} + { tv->f = (tv->f&~(TVSF_ACTIVE | TVSF_OUTMASK)) | (out << TVSF_OUTSHIFT); } + +/* --- @tvec_skip@, @tvec_skip_v@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *excuse@, @va_list *ap@ = reason why test skipped + * + * Returns: --- + * + * Use: Skip the current test. This should only be called from a + * test environment @run@ function; a similar effect occurs if + * the @before@ function fails. + */ void tvec_skip(struct tvec_state *tv, const char *excuse, ...) { va_list ap; va_start(ap, excuse); tvec_skip_v(tv, excuse, &ap); va_end(ap); } + void tvec_skip_v(struct tvec_state *tv, const char *excuse, va_list *ap) { assert(tv->f&TVSF_ACTIVE); @@ -110,11 +182,29 @@ void tvec_skip_v(struct tvec_state *tv, const char *excuse, va_list *ap) tv->output->ops->skip(tv->output, excuse, ap); } +/* --- @tvec_fail@, @tvec_fail_v@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *detail@, @va_list *ap@ = description of test + * + * Returns: --- + * + * Use: Report the current test as a failure. This function can be + * called multiple times for a single test, e.g., if the test + * environment's @run@ function invokes the test function + * repeatedly; but a single test that fails repeatedly still + * only counts as a single failure in the statistics. The + * @detail@ string and its format parameters can be used to + * distinguish which of several invocations failed; it can + * safely be left null if the test function is run only once. + */ + void tvec_fail(struct tvec_state *tv, const char *detail, ...) { va_list ap; va_start(ap, detail); tvec_fail_v(tv, detail, &ap); va_end(ap); } + void tvec_fail_v(struct tvec_state *tv, const char *detail, va_list *ap) { assert((tv->f&TVSF_ACTIVE) || @@ -122,11 +212,43 @@ void tvec_fail_v(struct tvec_state *tv, const char *detail, va_list *ap) set_outcome(tv, TVOUT_LOSE); tv->output->ops->fail(tv->output, detail, ap); } +/* --- @tvec_dumpreg@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @unsigned disp@ = the register disposition (@TVRD_...@) + * @const union tvec_regval *tv@ = register value, or null + * @const struct tvec_regdef *rd@ = register definition + * + * Returns: --- + * + * Use: Dump a register value to the output. This is the lowest- + * level function for dumping registers, and calls the output + * formatter directly. + * + * Usually @tvec_mismatch@ is much more convenient. Low-level + * access is required for reporting `virtual' registers + * corresponding to test environment settings. + */ + void tvec_dumpreg(struct tvec_state *tv, unsigned disp, const union tvec_regval *r, const struct tvec_regdef *rd) { tv->output->ops->dumpreg(tv->output, disp, r, rd); } +/* --- @tvec_mismatch@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @unsigned f@ = flags (@TVMF_...@) + * + * Returns: --- + * + * Use: Dumps registers suitably to report a mismatch. The flag bits + * @TVMF_IN@ and @TVF_OUT@ select input-only and output + * registers. If both are reset then nothing happens. + * Suppressing the output registers may be useful, e.g., if the + * test function crashed rather than returning outputs. + */ + void tvec_mismatch(struct tvec_state *tv, unsigned f) { const struct tvec_regdef *rd; @@ -152,12 +274,73 @@ void tvec_mismatch(struct tvec_state *tv, unsigned f) } } -/*----- Main machinery ----------------------------------------------------*/ +/*----- Parsing -----------------------------------------------------------*/ -struct groupstate { - void *ctx; -}; -#define GROUPSTATE_INIT { 0 } +/* --- @tvec_syntax@, @tvec_syntax_v@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @int ch@ = the character found (in @fgetc@ format) + * @const char *expect@, @va_list ap@ = what was expected + * + * Returns: %$-1$%. + * + * Use: Report a syntax error quoting @ch@ and @expect@. If @ch@ is + * a newline, then back up so that it can be read again (e.g., + * by @tvec_flushtoeol@ or @tvec_nexttoken@, which will also + * advance the line number). + */ + +int tvec_syntax(struct tvec_state *tv, int ch, const char *expect, ...) +{ + va_list ap; + + va_start(ap, expect); tvec_syntax_v(tv, ch, expect, &ap); va_end(ap); + return (-1); +} + +int tvec_syntax_v(struct tvec_state *tv, int ch, + const char *expect, va_list *ap) +{ + dstr d = DSTR_INIT; + char found[8]; + + switch (ch) { + case EOF: strcpy(found, "#eof"); break; + case '\n': strcpy(found, "#eol"); ungetc(ch, tv->fp); break; + default: + if (isprint(ch)) sprintf(found, "`%c'", ch); + else sprintf(found, "#\\x%02x", ch); + break; + } + dstr_vputf(&d, expect, ap); + tvec_error(tv, "syntax error: expected %s but found %s", d.buf, found); + dstr_destroy(&d); return (-1); +} + +/* --- @tvec_dupreg@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *name@ = register or pseudoregister name + * + * Returns: %$-1$%. + * + * Use: Reports an error that the register or pseudoregister has been + * assigned already in the current test. + */ + +int tvec_dupreg(struct tvec_state *tv, const char *name) + { return (tvec_error(tv, "register `%s' is already set", name)); } + +/* --- @tvec_skipspc@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: Advance over any whitespace characters other than newlines. + * This will stop at `;', end-of-file, or any other kind of + * non-whitespace; and it won't consume a newline. + */ void tvec_skipspc(struct tvec_state *tv) { @@ -170,6 +353,24 @@ void tvec_skipspc(struct tvec_state *tv) } } +/* --- @tvec_flushtoeol@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @unsigned f@ = flags (@TVFF_...@) + * + * Returns: Zero on success, @-1@ on error. + * + * Use: Advance to the start of the next line, consuming the + * preceding newline. + * + * A syntax error is reported if no newline character is found, + * i.e., the file ends in mid-line. A syntax error is also + * reported if material other than whitespace or a comment is + * found before the end of the line end, and @TVFF_ALLOWANY@ is + * not set in @f@. The line number count is updated + * appropriately. + */ + int tvec_flushtoeol(struct tvec_state *tv, unsigned f) { int ch, rc = 0; @@ -190,6 +391,29 @@ int tvec_flushtoeol(struct tvec_state *tv, unsigned f) } } +/* --- @tvec_nexttoken@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: Zero if there is a next token which can be read; @-1@ if no + * token is available. + * + * Use: Advance to the next whitespace-separated token, which may be + * on the next line. + * + * Tokens are separated by non-newline whitespace, comments, and + * newlines followed by whitespace; a newline /not/ followed by + * whitespace instead begins the next assignment, and two + * newlines separated only by whitespace terminate the data for + * a test. + * + * If this function returns zero, then the next character in the + * file begins a suitable token which can be read and + * processed. If it returns @-1@ then there is no such token, + * and the file position is left correctly. The line number + * count is updated appropriately. + */ + int tvec_nexttoken(struct tvec_state *tv) { enum { TAIL, NEWLINE, INDENT, COMMENT }; @@ -224,6 +448,34 @@ int tvec_nexttoken(struct tvec_state *tv) } } +/* --- @tvec_readword@, @tvec_readword_v@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @dstr *d@ = string to append the word to + * @const char *delims@ = additional delimiters to stop at + * @const char *expect@, @va_list ap@ = what was expected + * + * Returns: Zero on success, @-1@ on failure. + * + * Use: A `word' consists of characters other than whitespace, null + * characters, and other than those listed in @delims@; + * furthermore, a word does not begin with a `;'. (If you want + * reading to stop at comments not preceded by whitespace, then + * include `;' in @delims@. This is a common behaviour.) + * + * If there is no word beginning at the current file position, + * then return @-1@; furthermore, if @expect@ is not null, then + * report an appropriate error via @tvec_syntax@. + * + * Otherwise, the word is accumulated in @d@ and zero is + * returned; if @d@ was not empty at the start of the call, the + * newly read word is separated from the existing material by a + * single space character. Since null bytes are never valid + * word constituents, a null terminator is written to @d@, and + * it is safe to treat the string in @d@ as being null- + * terminated. + */ + int tvec_readword(struct tvec_state *tv, dstr *d, const char *delims, const char *expect, ...) { @@ -235,6 +487,7 @@ int tvec_readword(struct tvec_state *tv, dstr *d, const char *delims, va_end(ap); return (rc); } + int tvec_readword_v(struct tvec_state *tv, dstr *d, const char *delims, const char *expect, va_list *ap) { @@ -256,22 +509,26 @@ int tvec_readword_v(struct tvec_state *tv, dstr *d, const char *delims, return (0); } -void tvec_resetoutputs(struct tvec_state *tv) -{ - const struct tvec_regdef *rd; - struct tvec_reg *r; +/*----- Main machinery ----------------------------------------------------*/ - for (rd = tv->test->regs; rd->name; rd++) { - assert(rd->i < tv->nreg); - if (rd->i >= tv->nrout) continue; - r = TVEC_REG(tv, out, rd->i); - rd->ty->release(&r->v, rd); - rd->ty->init(&r->v, rd); - r->f = TVEC_REG(tv, in, rd->i)->f&TVRF_LIVE; - } -} +struct groupstate { + void *ctx; /* test environment context */ +}; +#define GROUPSTATE_INIT { 0 } + +/* --- @tvec_initregs@, @tvec_releaseregs@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: Initialize or release, respectively, the registers required + * by the current test. All of the registers, both input and + * output, are effected. Initialized registers are not marked + * live. + */ -static void init_registers(struct tvec_state *tv) +void tvec_initregs(struct tvec_state *tv) { const struct tvec_regdef *rd; struct tvec_reg *r; @@ -284,7 +541,7 @@ static void init_registers(struct tvec_state *tv) } } -static void release_registers(struct tvec_state *tv) +void tvec_releaseregs(struct tvec_state *tv) { const struct tvec_regdef *rd; struct tvec_reg *r; @@ -297,6 +554,51 @@ static void release_registers(struct tvec_state *tv) } } +/* --- @tvec_resetoutputs@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: Reset (releases and reinitializes) the output registers in + * the test state. This is mostly of use to test environment + * @run@ functions, between invocations of the test function. + * Output registers are marked live if and only if the + * corresponding input register is live. + */ + +void tvec_resetoutputs(struct tvec_state *tv) +{ + const struct tvec_regdef *rd; + struct tvec_reg *r; + + for (rd = tv->test->regs; rd->name; rd++) { + assert(rd->i < tv->nreg); + if (rd->i >= tv->nrout) continue; + r = TVEC_REG(tv, out, rd->i); + rd->ty->release(&r->v, rd); + rd->ty->init(&r->v, rd); + r->f = TVEC_REG(tv, in, rd->i)->f&TVRF_LIVE; + } +} + +/* --- @tvec_checkregs@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: Zero on success, @-1@ on mismatch. + * + * Use: Compare the active output registers (according to the current + * test group definition) with the corresponding input register + * values. A mismatch occurs if the two values differ + * (according to the register type's @eq@ method), or if the + * input is live but the output is dead. + * + * This function only checks for a mismatch and returns the + * result; it takes no other action. In particular, it doesn't + * report a failure, or dump register values. + */ + int tvec_checkregs(struct tvec_state *tv) { const struct tvec_regdef *rd; @@ -312,38 +614,137 @@ int tvec_checkregs(struct tvec_state *tv) return (0); } +/* --- @tvec_check@, @tvec_check_v@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *detail@, @va_list *ap@ = description of test + * + * Returns: --- + * + * Use: Check the register values, reporting a failure and dumping + * the registers in the event of a mismatch. This just wraps up + * @tvec_checkregs@, @tvec_fail@ and @tvec_mismatch@ in the + * obvious way. + */ + void tvec_check(struct tvec_state *tv, const char *detail, ...) { va_list ap; va_start(ap, detail); tvec_check_v(tv, detail, &ap); va_end(ap); } + void tvec_check_v(struct tvec_state *tv, const char *detail, va_list *ap) { if (tvec_checkregs(tv)) { tvec_fail_v(tv, detail, ap); tvec_mismatch(tv, TVMF_IN | TVMF_OUT); } } +/* --- @open_test@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: Note that we are now collecting data for a new test. The + * current line number is recorded in @test_lno@. The + * @TVSF_OPEN@ flag is set, and @TVSF_XFAIL@ is reset. + * + * If a test is already open, then do nothing. + */ + +static void open_test(struct tvec_state *tv) +{ + if (!(tv->f&TVSF_OPEN)) { + tv->test_lno = tv->lno; + tv->f |= TVSF_OPEN; tv->f &= ~TVSF_XFAIL; + } +} + +/* --- @begin_test@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: Note that we're about to try running a state. This is called + * before the test environment's @before@ function. Mark the + * test as active, clear the outcome, and inform the output + * driver. + */ + static void begin_test(struct tvec_state *tv) { tv->f |= TVSF_ACTIVE; tv->f &= ~TVSF_OUTMASK; tv->output->ops->btest(tv->output); } +/* --- @tvec_endtest@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: End an ad-hoc test case, The statistics are updated and the + * outcome is reported to the output formatter. + */ + void tvec_endtest(struct tvec_state *tv) { unsigned out; - if (tv->f&TVSF_ACTIVE) out = TVOUT_WIN; - else out = (tv->f&TVSF_OUTMASK) >> TVSF_OUTSHIFT; + if (!(tv->f&TVSF_ACTIVE)) /* nothing to do */; + else if (tv->f&TVSF_XFAIL) set_outcome(tv, TVOUT_XFAIL); + else set_outcome(tv, TVOUT_WIN); + out = (tv->f&TVSF_OUTMASK) >> TVSF_OUTSHIFT; assert(out < TVOUT_LIMIT); tv->curr[out]++; tv->output->ops->etest(tv->output, out); tv->f &= ~TVSF_OPEN; } +/* --- @tvec_xfail@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: Mark the current test as an `expected failure'. That is, the + * behaviour -- if everything works as expected -- is known to + * be incorrect, perhaps as a result of a longstanding bug, so + * calling it a `success' would be inappropriate. A failure, as + * reported by @tvec_fail@, means that the behaviour is not as + * expected -- either the long-standing bug has been fixed, or a + * new bug has been introduced -- so investigation is required. + * + * An expected failure doesn't cause the test group or the + * session as a whole to fail, but it isn't counted as a win + * either. + */ + +void tvec_xfail(struct tvec_state *tv) + { tv->f |= TVSF_XFAIL; } + +/* --- @check@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @struct groupstate *g@ = private test group state + * + * Returns: --- + * + * Use: Run the current test. + * + * This function is called once the data for a test has been + * collected. It's responsible for checking that all of the + * necessary registers have been assigned values. It marks the + * output registers as live if the corresponding inputs are + * live. It calls the environment's @before@, @run@, and + * @after@ functions if provided; if there is no @run@ function, + * it calls @tvec_check@ to verify the output values. + */ + static void check(struct tvec_state *tv, struct groupstate *g) { const struct tvec_test *t = tv->test; - const struct tvec_env *env; + const struct tvec_env *env = t->env; const struct tvec_regdef *rd; if (!(tv->f&TVSF_OPEN)) return; @@ -360,21 +761,36 @@ static void check(struct tvec_state *tv, struct groupstate *g) if (!(tv->f&TVSF_SKIP)) { begin_test(tv); - env = t->env; - if (env && env->before && env->before(tv, g->ctx)) - tvec_skip(tv, "test setup failed"); + if (env && env->before) env->before(tv, g->ctx); + if (!(tv->f&TVSF_ACTIVE)) + /* setup forced a skip */; + else if (env && env->run) + env->run(tv, t->fn, g->ctx); else { - if (env && env->run) env->run(tv, t->fn, g->ctx); - else { t->fn(tv->in, tv->out, g->ctx); tvec_check(tv, 0); } + t->fn(tv->in, tv->out, g->ctx); + tvec_check(tv, 0); } - if (env && env->after) env->after(tv, g->ctx); tvec_endtest(tv); } + if (env && env->after) env->after(tv, g->ctx); end: - tv->f &= ~TVSF_OPEN; release_registers(tv); init_registers(tv); + tv->f &= ~TVSF_OPEN; tvec_releaseregs(tv); tvec_initregs(tv); } +/* --- @begin_test_group@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @struct groupstate *g@ = private test group state + * + * Returns: --- + * + * Use: Begins a test group. Expects @tv->test@ to have been set + * already. Calls the output driver, initializes the registers, + * clears the @tv->curr@ counters, allocates the environment + * context and calls the environment @setup@ function. + */ + static void begin_test_group(struct tvec_state *tv, struct groupstate *g) { const struct tvec_test *t = tv->test; @@ -382,17 +798,27 @@ static void begin_test_group(struct tvec_state *tv, struct groupstate *g) unsigned i; tv->output->ops->bgroup(tv->output); - tv->f &= ~TVSF_SKIP; - init_registers(tv); + tv->f &= ~(TVSF_SKIP | TVSF_MUFFLE); + tvec_initregs(tv); for (i = 0; i < TVOUT_LIMIT; i++) tv->curr[i] = 0; if (env && env->ctxsz) g->ctx = xmalloc(env->ctxsz); - if (env && env->setup && env->setup(tv, env, 0, g->ctx)) { - tvec_skipgroup(tv, "setup failed"); - xfree(g->ctx); g->ctx = 0; - } + if (env && env->setup) env->setup(tv, env, 0, g->ctx); } -void tvec_reportgroup(struct tvec_state *tv) +/* --- @report_group@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: Reports the result of the test group to the output driver. + * + * If all of the tests have been skipped then report this as a + * group skip. Otherwise, determine and report the group + * outcome. + */ + +static void report_group(struct tvec_state *tv) { unsigned i, out, nrun; @@ -404,10 +830,28 @@ void tvec_reportgroup(struct tvec_state *tv) else { if (tv->curr[TVOUT_LOSE]) out = TVOUT_LOSE; else out = TVOUT_WIN; - tv->grps[out]++; tv->output->ops->egroup(tv->output, out); + tv->grps[out]++; tv->output->ops->egroup(tv->output); } } +/* --- @end_test_group@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @struct groupstate *g@ = private test group state + * + * Returns: --- + * + * Use: Handles the end of a test group. Called at the end of the + * input file or when a new test group header is found. + * + * If a test is open, call @check@ to see whether it worked. If + * the test group is not being skipped, report the group + * result. Call the test environment @teardown@ function. Free + * the environment context and release the registers. + * + * If there's no test group active, then nothing happens. + */ + static void end_test_group(struct tvec_state *tv, struct groupstate *g) { const struct tvec_test *t = tv->test; @@ -415,11 +859,39 @@ static void end_test_group(struct tvec_state *tv, struct groupstate *g) if (!t) return; if (tv->f&TVSF_OPEN) check(tv, g); - if (!(tv->f&TVSF_SKIP)) tvec_reportgroup(tv); + if (!(tv->f&TVSF_SKIP)) report_group(tv); env = t->env; if (env && env->teardown) env->teardown(tv, g->ctx); - release_registers(tv); tv->test = 0; xfree(g->ctx); g->ctx = 0; + tvec_releaseregs(tv); tv->test = 0; xfree(g->ctx); g->ctx = 0; } +/* --- @tvec_read@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *infile@ = the name of the input file + * @FILE *fp@ = stream to read from + * + * Returns: Zero on success, @-1@ on error. + * + * Use: Read test vector data from @fp@ and exercise test functions. + * THe return code doesn't indicate test failures: it's only + * concerned with whether there were problems with the input + * file or with actually running the tests. + */ + +enum { WIN, XFAIL, NOUT }; +static const struct tvec_uassoc outcome_assoc[] = { + { "success", WIN }, + { "win", WIN }, + { "expected-failure", XFAIL }, + { "xfail", XFAIL }, + TVEC_ENDENUM +}; +static const struct tvec_urange outcome_range = { 0, NOUT - 1 }; +static const struct tvec_uenuminfo outcome_enum = + { "test-outcome", outcome_assoc, &outcome_range }; +static const struct tvec_regdef outcome_regdef = + { "outcome", 0, &tvty_uenum, 0, { &outcome_enum } }; + int tvec_read(struct tvec_state *tv, const char *infile, FILE *fp) { dstr d = DSTR_INIT; @@ -428,81 +900,165 @@ int tvec_read(struct tvec_state *tv, const char *infile, FILE *fp) const struct tvec_regdef *rd; struct tvec_reg *r; struct groupstate g = GROUPSTATE_INIT; + union tvec_regval rv; int ch, ret, rc = 0; + /* Set the initial location. */ tv->infile = infile; tv->lno = 1; tv->fp = fp; for (;;) { + + /* Get the next character and dispatch. Note that we're always at the + * start of a line here. + */ ch = getc(tv->fp); switch (ch) { case EOF: + /* End of the file. Exit the loop. */ + goto end; case '[': + /* A test group header. */ + + /* End the current group, if there is one. */ end_test_group(tv, &g); + + /* Read the group name. There may be leading and trailing + * whitespace. + */ tvec_skipspc(tv); DRESET(&d); tvec_readword(tv, &d, "];", "group name"); tvec_skipspc(tv); ch = getc(tv->fp); if (ch != ']') tvec_syntax(tv, ch, "`]'"); + + /* Find the matching test definition. */ for (test = tv->tests; test->name; test++) if (STRCMP(d.buf, ==, test->name)) goto found_test; - tvec_error(tv, "unknown test group `%s'", d.buf); goto flush_line; + + /* There wasn't one. Report the error. Muffle errors about the + * contents of this section because they won't be interesting. + */ + tvec_error(tv, "unknown test group `%s'", d.buf); + tv->f |= TVSF_MUFFLE; goto flush_line; + found_test: - tvec_flushtoeol(tv, 0); tv->test = test; begin_test_group(tv, &g); + /* Eat trailing whitespace and comments. */ + tvec_flushtoeol(tv, 0); + + /* Set up the new test group. */ + tv->test = test; begin_test_group(tv, &g); break; case '\n': + /* A newline, so this was a completely empty line. Advance the line + * counter, and run the current test. + */ + tv->lno++; if (tv->f&TVSF_OPEN) check(tv, &g); break; + case ';': + /* A semicolon. Skip the comment. */ + + tvec_flushtoeol(tv, TVFF_ALLOWANY); + break; + default: + /* Something else. */ + if (isspace(ch)) { - tvec_skipspc(tv); - ch = getc(tv->fp); + /* Whitespace. Skip and see what we find. */ + + tvec_skipspc(tv); ch = getc(tv->fp); + + /* If the file ends, then we're done. If we find a comment then we + * skip it. If there's some non-whitespace, then report an error. + * Otherwise the line was effectively blank, so run the test. + */ if (ch == EOF) goto end; else if (ch == ';') tvec_flushtoeol(tv, TVFF_ALLOWANY); else if (tvec_flushtoeol(tv, 0)) rc = -1; else check(tv, &g); - } else if (ch == ';') - tvec_flushtoeol(tv, TVFF_ALLOWANY); - else { + } else { + /* Some non-whitespace thing. */ + + /* Put the character back and read a word, which ought to be a + * register name. + */ ungetc(ch, tv->fp); DRESET(&d); if (tvec_readword(tv, &d, "=:;", "register name")) goto flush_line; + + /* Now there should be a separator. */ tvec_skipspc(tv); ch = getc(tv->fp); if (ch != '=' && ch != ':') { tvec_syntax(tv, ch, "`=' or `:'"); goto flush_line; } tvec_skipspc(tv); - if (!tv->test) - { tvec_error(tv, "no current test"); goto flush_line; } + + /* If there's no test, then report an error. Set the muffle flag, + * because there's no point in complaining about every assignment + * in this block. + */ + if (!tv->test) { + if (!(tv->f&TVSF_MUFFLE)) tvec_error(tv, "no current test"); + tv->f |= TVSF_MUFFLE; goto flush_line; + } + + /* Open the test. This is syntactically a stanza of settings, so + * it's fair to report on missing register assignments. + */ + open_test(tv); + if (d.buf[0] == '@') { + /* A special register assignment. */ env = tv->test->env; - if (!env || !env->set) ret = 0; - else ret = env->set(tv, d.buf, env, g.ctx); + + /* See if it's one of the core settings. */ + if (STRCMP(d.buf, ==, "@outcome")) { + + /* Parse the value. */ + if (tvty_uenum.parse(&rv, &outcome_regdef, tv)) + ret = -1; + else { + + /* Act on the result. */ + if (rv.u == XFAIL) tvec_xfail(tv); + ret = 1; + } + } + + /* If there's no environment, this is an unknown setting. */ + else if (!env || !env->set) ret = 0; + + /* Otherwise pass the setting on to the environment. */ + else ret = env->set(tv, d.buf, g.ctx); + + /* If it wasn't understood, report an error and flush. */ if (ret <= 0) { if (!ret) tvec_error(tv, "unknown special register `%s'", d.buf); goto flush_line; } - if (!(tv->f&TVSF_OPEN)) - { tv->test_lno = tv->lno; tv->f |= TVSF_OPEN; } } else { + /* A standard register. */ + + /* Find the definition. */ for (rd = tv->test->regs; rd->name; rd++) if (STRCMP(rd->name, ==, d.buf)) goto found_reg; tvec_error(tv, "unknown register `%s' for test `%s'", d.buf, tv->test->name); goto flush_line; + found_reg: - if (!(tv->f&TVSF_OPEN)) - { tv->test_lno = tv->lno; tv->f |= TVSF_OPEN; } - tvec_skipspc(tv); + /* Complain if the register is already set. */ r = TVEC_REG(tv, in, rd->i); - if (r->f&TVRF_LIVE) { - tvec_error(tv, "register `%s' already set", rd->name); - goto flush_line; - } + if (r->f&TVRF_LIVE) + { tvec_dupreg(tv, rd->name); goto flush_line; } + + /* Parse a value and mark the register as live. */ if (rd->ty->parse(&r->v, rd, tv)) goto flush_line; r->f |= TVRF_LIVE; } @@ -512,12 +1068,24 @@ int tvec_read(struct tvec_state *tv, const char *infile, FILE *fp) continue; flush_line: + /* This is a general parse-failure handler. Skip to the next line and + * remember that things didn't go so well. + */ tvec_flushtoeol(tv, TVFF_ALLOWANY); rc = -1; } + + /* We reached the end. If that was actually an I/O error then report it. + */ if (ferror(tv->fp)) { tvec_error(tv, "error reading input: %s", strerror(errno)); rc = -1; } + end: + /* Process the final test, if there was one, and wrap up the final + * group. + */ end_test_group(tv, &g); + + /* Clean up. */ tv->infile = 0; tv->fp = 0; dstr_destroy(&d); return (rc); @@ -525,6 +1093,17 @@ end: /*----- Session lifecycle -------------------------------------------------*/ +/* --- @tvec_begin@ --- * + * + * Arguments: @struct tvec_state *tv_out@ = state structure to fill in + * @const struct tvec_config *config@ = test configuration + * @struct tvec_output *o@ = output driver + * + * Returns: --- + * + * Use: Initialize a state structure ready to do some testing. + */ + void tvec_begin(struct tvec_state *tv_out, const struct tvec_config *config, struct tvec_output *o) @@ -548,15 +1127,25 @@ void tvec_begin(struct tvec_state *tv_out, tv_out->tests = config->tests; tv_out->test = 0; tv_out->infile = 0; tv_out->lno = 0; tv_out->fp = 0; - o->tv = tv_out; tv_out->output = o; - - tv_out->output->ops->bsession(tv_out->output); + tv_out->output = o; tv_out->output->ops->bsession(tv_out->output, tv_out); } +/* --- @tvec_end@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: A proposed exit code. + * + * Use: Conclude testing and suggests an exit code to be returned to + * the calling program. (The exit code comes from the output + * driver's @esession@ method.) + */ + int tvec_end(struct tvec_state *tv) { int rc = tv->output->ops->esession(tv->output); + if (tv->test) tvec_releaseregs(tv); tv->output->ops->destroy(tv->output); xfree(tv->in); xfree(tv->out); return (rc); @@ -564,6 +1153,25 @@ int tvec_end(struct tvec_state *tv) /*----- Serialization and deserialization ---------------------------------*/ +/* --- @tvec_serialize@ --- * + * + * Arguments: @const struct tvec_reg *rv@ = vector of registers + * @buf *b@ = buffer to write on + * @const struct tvec_regdef *regs@ = vector of register + * descriptions, terminated by an entry with a null + * @name@ slot + * @unsigned nr@ = number of entries in the @rv@ vector + * @size_t regsz@ = true size of each element of @rv@ + * + * Returns: Zero on success, @-1@ on failure. + * + * Use: Serialize a collection of register values. + * + * The serialized output is written to the buffer @b@. Failure + * can be caused by running out of buffer space, or a failing + * type handler. + */ + int tvec_serialize(const struct tvec_reg *rv, buf *b, const struct tvec_regdef *regs, unsigned nr, size_t regsz) @@ -582,12 +1190,36 @@ int tvec_serialize(const struct tvec_reg *rv, buf *b, for (rd = regs, i = 0; rd->name; rd++, i++) { if (rd->i >= nr) continue; r = TVEC_GREG(rv, rd->i, regsz); if (!(r->f&TVRF_LIVE)) continue; - bitmap = BBASE(b) + bitoff; bitmap[rd->i/8] |= 1 << rd->i%8; + bitmap = BBASE(b) + bitoff; bitmap[i/8] |= 1 << i%8; if (rd->ty->tobuf(b, &r->v, rd)) return (-1); } return (0); } +/* --- @tvec_deserialize@ --- * + * + * Arguments: @struct tvec_reg *rv@ = vector of registers + * @buf *b@ = buffer to write on + * @const struct tvec_regdef *regs@ = vector of register + * descriptions, terminated by an entry with a null + * @name@ slot + * @unsigned nr@ = number of entries in the @rv@ vector + * @size_t regsz@ = true size of each element of @rv@ + * + * Returns: Zero on success, @-1@ on failure. + * + * Use: Deserialize a collection of register values. + * + * The size of the register vector @nr@ and the register + * definitions @regs@ must match those used when producing the + * serialization. For each serialized register value, + * deserialize and store the value into the appropriate register + * slot, and set the @TVRF_LIVE@ flag on the register. See + * @tvec_serialize@ for a description of the format. + * + * Failure results only from a failing register type handler. + */ + int tvec_deserialize(struct tvec_reg *rv, buf *b, const struct tvec_regdef *regs, unsigned nr, size_t regsz) @@ -603,7 +1235,7 @@ int tvec_deserialize(struct tvec_reg *rv, buf *b, bitmap = buf_get(b, bitsz); if (!bitmap) return (-1); for (rd = regs, i = 0; rd->name; rd++, i++) { if (rd->i >= nr) continue; - if (!(bitmap[rd->i/8]&(1 << rd->i%8))) continue; + if (!(bitmap[i/8]&(1 << i%8))) continue; r = TVEC_GREG(rv, rd->i, regsz); if (rd->ty->frombuf(b, &r->v, rd)) return (-1); r->f |= TVRF_LIVE; @@ -618,12 +1250,42 @@ static const struct tvec_regdef no_regs = { 0, 0, 0, 0, { 0 } }; static void fakefn(const struct tvec_reg *in, struct tvec_reg *out, void *p) { assert(!"fake test function"); } +/* --- @tvec_adhoc@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @struct tvec_test *t@ = space for a test definition + * + * Returns: --- + * + * Use: Begin ad-hoc testing, i.e., without reading a file of + * test-vector data. + * + * The structure at @t@ will be used to record information about + * the tests underway, which would normally come from a static + * test definition. The other functions in this section assume + * that @tvec_adhoc@ has been called. + */ + void tvec_adhoc(struct tvec_state *tv, struct tvec_test *t) { t->name = ""; t->regs = &no_regs; t->env = 0; t->fn = fakefn; tv->tests = t; } +/* --- @tvec_begingroup@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *name@ = name for this test group + * @const char *file@, @unsigned @lno@ = calling file and line + * + * Returns: --- + * + * Use: Begin an ad-hoc test group with the given name. The @file@ + * and @lno@ can be anything, but it's usually best if they + * refer to the source code performing the test: the macro + * @TVEC_BEGINGROUP@ does this automatically. + */ + void tvec_begingroup(struct tvec_state *tv, const char *name, const char *file, unsigned lno) { @@ -634,16 +1296,39 @@ void tvec_begingroup(struct tvec_state *tv, const char *name, begin_test_group(tv, 0); } +/* --- @tvec_endgroup@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * + * Returns: --- + * + * Use: End an ad-hoc test group. The statistics are updated and the + * outcome is reported to the output formatter. + */ + void tvec_endgroup(struct tvec_state *tv) { - if (!(tv->f&TVSF_SKIP)) tvec_reportgroup(tv); + if (!(tv->f&TVSF_SKIP)) report_group(tv); tv->test = 0; } +/* --- @tvec_begintest@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @const char *file@, @unsigned @lno@ = calling file and line + * + * Returns: --- + * + * Use: Begin an ad-hoc test case. The @file@ and @lno@ can be + * anything, but it's usually best if they refer to the source + * code performing the test: the macro @TVEC_BEGINGROUP@ does + * this automatically. + */ + void tvec_begintest(struct tvec_state *tv, const char *file, unsigned lno) { tv->infile = file; tv->lno = tv->test_lno = lno; - begin_test(tv); tv->f |= TVSF_OPEN; + open_test(tv); begin_test(tv); } struct adhoc_claim { @@ -680,19 +1365,52 @@ static void adhoc_claim_teardown(struct tvec_state *tv, if (ck->f&ACF_FRESH) tvec_endtest(tv); } -int tvec_claim(struct tvec_state *tv, int ok, - const char *file, unsigned lno, const char *expr, ...) +/* --- @tvec_claim@, @tvec_claim_v@, @TVEC_CLAIM@ --- * + * + * Arguments: @struct tvec_state *tv@ = test-vector state + * @int ok@ = a flag + * @const char *file@, @unsigned @lno@ = calling file and line + * @const char *msg@, @va_list *ap@ = message to report on + * failure + * + * Returns: The value @ok@. + * + * Use: Check that a claimed condition holds, as (part of) a test. + * If no test case is underway (i.e., if @TVSF_OPEN@ is reset in + * @tv->f@), then a new test case is begun and ended. The + * @file@ and @lno@ are passed to the output formatter to be + * reported in case of a failure. If @ok@ is nonzero, then + * nothing else happens; so, in particular, if @tvec_claim@ + * established a new test case, then the test case succeeds. If + * @ok@ is zero, then a failure is reported, quoting @msg@. + * + * The @TVEC_CLAIM@ macro is similar, only it (a) identifies the + * file and line number of the call site automatically, and (b) + * implicitly quotes the source text of the @ok@ condition in + * the failure message. + */ + +int tvec_claim_v(struct tvec_state *tv, int ok, + const char *file, unsigned lno, + const char *msg, va_list *ap) { struct adhoc_claim ck; - va_list ap; adhoc_claim_setup(tv, &ck, 0, file, lno); - if (!ok) - { va_start(ap, expr); tvec_fail_v(tv, expr, &ap); va_end(ap); } + if (!ok) tvec_fail_v(tv, msg, ap); adhoc_claim_teardown(tv, &ck); return (ok); } +int tvec_claim(struct tvec_state *tv, int ok, + const char *file, unsigned lno, const char *msg, ...) +{ + va_list ap; + + va_start(ap, msg); tvec_claim_v(tv, ok, file, lno, msg, &ap); va_end(ap); + return (ok); +} + int tvec_claimeq(struct tvec_state *tv, const struct tvec_regty *ty, const union tvec_misc *arg, const char *file, unsigned lno, const char *expr) @@ -713,7 +1431,7 @@ int tvec_claimeq(struct tvec_state *tv, adhoc_claim_setup(tv, &ck, regs, file, lno); ok = ty->eq(&tv->in[0].v, &tv->out[0].v, ®s[0]); if (!ok) - { tvec_fail(tv, "%s", expr ); tvec_mismatch(tv, TVMF_IN | TVMF_OUT); } + { tvec_fail(tv, "%s", expr); tvec_mismatch(tv, TVMF_IN | TVMF_OUT); } adhoc_claim_teardown(tv, &ck); return (ok); }